-
Notifications
You must be signed in to change notification settings - Fork 734
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
aes-gcm: Enable AVX-512 implementation.
- Loading branch information
1 parent
ea2e22e
commit 159aa07
Showing
9 changed files
with
236 additions
and
82 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,93 @@ | ||
| // Copyright 2015-2025 Brian Smith. | ||
| // | ||
| // Permission to use, copy, modify, and/or distribute this software for any | ||
| // purpose with or without fee is hereby granted, provided that the above | ||
| // copyright notice and this permission notice appear in all copies. | ||
| // | ||
| // THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES | ||
| // WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF | ||
| // MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY | ||
| // SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES | ||
| // WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION | ||
| // OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN | ||
| // CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. | ||
|
|
||
| #![cfg(target_arch = "x86_64")] | ||
|
|
||
| use super::{aes, gcm, Counter, BLOCK_LEN}; | ||
| use crate::{aead::aes::Overlapping, c, polyfill::slice::AsChunksMut}; | ||
| use core::num::{NonZeroU32, NonZeroUsize}; | ||
|
|
||
| pub(super) fn seal_whole( | ||
| aes_key: &aes::hw::Key, | ||
| auth: &mut gcm::Context<gcm::vclmulavx512::Key>, | ||
| ctr: &mut Counter, | ||
| mut in_out: AsChunksMut<u8, BLOCK_LEN>, | ||
| ) { | ||
| prefixed_extern! { | ||
| fn aes_gcm_enc_update_vaes_avx10_512( | ||
| input: *const u8, | ||
| output: *mut u8, | ||
| len: c::NonZero_size_t, // TODO? zero OK? | ||
| key: &aes::AES_KEY, | ||
| ivec: &Counter, | ||
| Htable: &gcm::HTable, | ||
| Xi: &mut gcm::Xi); | ||
| } | ||
|
|
||
| let in_out = in_out.as_flattened_mut(); | ||
|
|
||
| // Precondition: Since we have a `gcm::Context` then the number of blocks | ||
| // must fit in `u32`. | ||
| let blocks = u32::try_from(in_out.len() / BLOCK_LEN).unwrap(); | ||
|
|
||
| if let Some(len) = NonZeroUsize::new(in_out.len()) { | ||
| let aes_key = aes_key.inner_less_safe(); | ||
| let (htable, xi) = auth.inner(); | ||
| let input = in_out.as_ptr(); | ||
| let output = in_out.as_mut_ptr(); | ||
| unsafe { aes_gcm_enc_update_vaes_avx10_512(input, output, len, aes_key, ctr, htable, xi) }; | ||
| let blocks = NonZeroU32::new(blocks).unwrap_or_else(|| { | ||
| unreachable!() // Due to previous checks. | ||
| }); | ||
| ctr.increment_by_less_safe(blocks); | ||
| } | ||
| } | ||
|
|
||
| pub(super) fn open_whole( | ||
| aes_key: &aes::hw::Key, | ||
| auth: &mut gcm::Context<gcm::vclmulavx512::Key>, | ||
| in_out: Overlapping, | ||
| ctr: &mut Counter, | ||
| ) { | ||
| prefixed_extern! { | ||
| fn aes_gcm_dec_update_vaes_avx10_512( | ||
| input: *const u8, | ||
| output: *mut u8, | ||
| len: c::NonZero_size_t, // TODO? zero OK? | ||
| key: &aes::AES_KEY, | ||
| ivec: &mut Counter, | ||
| Htable: &gcm::HTable, | ||
| Xi: &mut gcm::Xi); | ||
| } | ||
|
|
||
| // Precondition. TODO: Create an overlapping::AsChunks for this. | ||
| assert_eq!(in_out.len() % BLOCK_LEN, 0); | ||
| // Precondition: Since we have a `gcm::Context` then the number of blocks | ||
| // must fit in `u32`. | ||
| let blocks = u32::try_from(in_out.len() / BLOCK_LEN).unwrap(); | ||
|
|
||
| in_out.with_input_output_len(|input, output, len| { | ||
| if let Some(len) = NonZeroUsize::new(len) { | ||
| let aes_key = aes_key.inner_less_safe(); | ||
| let (htable, xi) = auth.inner(); | ||
| unsafe { | ||
| aes_gcm_dec_update_vaes_avx10_512(input, output, len, aes_key, ctr, htable, xi) | ||
| }; | ||
| let blocks = NonZeroU32::new(blocks).unwrap_or_else(|| { | ||
| unreachable!() // Due to previous checks. | ||
| }); | ||
| ctr.increment_by_less_safe(blocks); | ||
| } | ||
| }) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,49 @@ | ||
| // Copyright 2018-2025 Brian Smith. | ||
| // | ||
| // Permission to use, copy, modify, and/or distribute this software for any | ||
| // purpose with or without fee is hereby granted, provided that the above | ||
| // copyright notice and this permission notice appear in all copies. | ||
| // | ||
| // THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES | ||
| // WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF | ||
| // MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY | ||
| // SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES | ||
| // WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION | ||
| // OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN | ||
| // CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. | ||
|
|
||
| #![cfg(target_arch = "x86_64")] | ||
|
|
||
| use super::{ffi::KeyValue, HTable, UpdateBlock, Xi}; | ||
| use crate::{ | ||
| aead::gcm::ffi::BLOCK_LEN, | ||
| cpu::intel::{Avx2, Avx512_BW_VL_ZMM, Bmi2, VAesClmul}, | ||
| polyfill::slice::AsChunks, | ||
| }; | ||
|
|
||
| #[derive(Clone)] | ||
| pub struct Key { | ||
| h_table: HTable, | ||
| } | ||
|
|
||
| impl Key { | ||
| pub(in super::super) fn new( | ||
| value: KeyValue, | ||
| _cpu: (Avx2, Avx512_BW_VL_ZMM, Bmi2, VAesClmul), | ||
| ) -> Self { | ||
| Self { | ||
| h_table: unsafe { htable_new!(gcm_init_vpclmulqdq_avx10_512, value) }, | ||
| } | ||
| } | ||
|
|
||
| pub(super) fn inner(&self) -> &HTable { | ||
| &self.h_table | ||
| } | ||
| } | ||
|
|
||
| impl UpdateBlock for Key { | ||
| fn update_block(&self, xi: &mut Xi, a: [u8; BLOCK_LEN]) { | ||
| let input: AsChunks<u8, BLOCK_LEN> = (&a).into(); | ||
| unsafe { ghash!(gcm_ghash_vpclmulqdq_avx10_512_1, xi, &self.h_table, input) } | ||
| } | ||
| } |
Oops, something went wrong.