#!/usr/bin/env perl # Copyright 2024 The BoringSSL Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # #------------------------------------------------------------------------------ # # This is an AES-GCM implementation for x86_64 CPUs that support the following # CPU features: VAES && VPCLMULQDQ && AVX512BW && AVX512VL && BMI2. # # This file is based on aes-gcm-avx10-x86_64.S from the Linux kernel # (https://git.kernel.org/linus/b06affb1cb580e13). The following notable # changes have been made: # # - Relicensed under BoringSSL's preferred license. # # - Converted from GNU assembler to "perlasm". This was necessary for # compatibility with BoringSSL's Windows builds which use NASM instead of the # GNU assembler. It was also necessary for compatibility with the 'delocate' # tool used in BoringSSL's FIPS builds. # # - Added support for the Windows ABI. # # - Changed function prototypes to be compatible with what BoringSSL wants. # # - Removed the optimized finalization function, as BoringSSL doesn't want it. # # - Added a single-block GHASH multiplication function, as BoringSSL needs this. # # - Added optimization for large amounts of AAD. # # - Removed support for maximum vector lengths other than 512 bits. use strict; my $flavour = shift; my $output = shift; if ( $flavour =~ /\./ ) { $output = $flavour; undef $flavour; } my $win64; my @argregs; if ( $flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/ ) { $win64 = 1; @argregs = ( "%rcx", "%rdx", "%r8", "%r9" ); } else { $win64 = 0; @argregs = ( "%rdi", "%rsi", "%rdx", "%rcx", "%r8", "%r9" ); } $0 =~ m/(.*[\/\\])[^\/\\]+$/; my $dir = $1; my $xlate; ( $xlate = "${dir}x86_64-xlate.pl" and -f $xlate ) or ( $xlate = "${dir}../../../perlasm/x86_64-xlate.pl" and -f $xlate ) or die "can't locate x86_64-xlate.pl"; open OUT, "| \"$^X\" \"$xlate\" $flavour \"$output\""; *STDOUT = *OUT; my $g_cur_func_name; my $g_cur_func_uses_seh; my @g_cur_func_saved_gpregs; my @g_cur_func_saved_xmmregs; sub _begin_func { my ( $funcname, $uses_seh ) = @_; $g_cur_func_name = $funcname; $g_cur_func_uses_seh = $uses_seh; @g_cur_func_saved_gpregs = (); @g_cur_func_saved_xmmregs = (); return <<___; .globl $funcname .type $funcname,\@abi-omnipotent .align 32 $funcname: .cfi_startproc @{[ $uses_seh ? ".seh_startproc" : "" ]} _CET_ENDBR ___ } # Push a list of general purpose registers onto the stack. sub _save_gpregs { my @gpregs = @_; my $code = ""; die "_save_gpregs requires uses_seh" unless $g_cur_func_uses_seh; die "_save_gpregs can only be called once per function" if @g_cur_func_saved_gpregs; die "Order must be _save_gpregs, then _save_xmmregs" if @g_cur_func_saved_xmmregs; @g_cur_func_saved_gpregs = @gpregs; for my $reg (@gpregs) { $code .= "push $reg\n"; if ($win64) { $code .= ".seh_pushreg $reg\n"; } else { $code .= ".cfi_push $reg\n"; } } return $code; } # Push a list of xmm registers onto the stack if the target is Windows. sub _save_xmmregs { my @xmmregs = @_; my $num_xmmregs = scalar @xmmregs; my $code = ""; die "_save_xmmregs requires uses_seh" unless $g_cur_func_uses_seh; die "_save_xmmregs can only be called once per function" if @g_cur_func_saved_xmmregs; if ( $win64 and $num_xmmregs > 0 ) { @g_cur_func_saved_xmmregs = @xmmregs; my $is_misaligned = ( scalar @g_cur_func_saved_gpregs ) % 2 == 0; my $alloc_size = 16 * $num_xmmregs + ( $is_misaligned ? 8 : 0 ); $code .= "sub \$$alloc_size, %rsp\n"; $code .= ".seh_stackalloc $alloc_size\n"; for my $i ( 0 .. $num_xmmregs - 1 ) { my $reg_num = $xmmregs[$i]; my $pos = 16 * $i; $code .= "vmovdqa %xmm$reg_num, $pos(%rsp)\n"; $code .= ".seh_savexmm %xmm$reg_num, $pos\n"; } } return $code; } sub _end_func { my $code = ""; # Restore any xmm registers that were saved earlier. my $num_xmmregs = scalar @g_cur_func_saved_xmmregs; if ( $win64 and $num_xmmregs > 0 ) { my $need_alignment = ( scalar @g_cur_func_saved_gpregs ) % 2 == 0; my $alloc_size = 16 * $num_xmmregs + ( $need_alignment ? 8 : 0 ); for my $i ( 0 .. $num_xmmregs - 1 ) { my $reg_num = $g_cur_func_saved_xmmregs[$i]; my $pos = 16 * $i; $code .= "vmovdqa $pos(%rsp), %xmm$reg_num\n"; } $code .= "add \$$alloc_size, %rsp\n"; } # Restore any general purpose registers that were saved earlier. for my $reg ( reverse @g_cur_func_saved_gpregs ) { $code .= "pop $reg\n"; if ( !$win64 ) { $code .= ".cfi_pop $reg\n"; } } $code .= <<___; ret @{[ $g_cur_func_uses_seh ? ".seh_endproc" : "" ]} .cfi_endproc .size $g_cur_func_name, . - $g_cur_func_name ___ return $code; } my $code = <<___; .section .rodata .align 64 # A shuffle mask that reflects the bytes of 16-byte blocks .Lbswap_mask: .quad 0x08090a0b0c0d0e0f, 0x0001020304050607 # This is the GHASH reducing polynomial without its constant term, i.e. # x^128 + x^7 + x^2 + x, represented using the backwards mapping # between bits and polynomial coefficients. # # Alternatively, it can be interpreted as the naturally-ordered # representation of the polynomial x^127 + x^126 + x^121 + 1, i.e. the # "reversed" GHASH reducing polynomial without its x^128 term. .Lgfpoly: .quad 1, 0xc200000000000000 # Same as above, but with the (1 << 64) bit set. .Lgfpoly_and_internal_carrybit: .quad 1, 0xc200000000000001 # Values needed to prepare the initial vector of counter blocks. .Lctr_pattern: .quad 0, 0 .quad 1, 0 .quad 2, 0 .quad 3, 0 # The number of AES blocks per vector, as a 128-bit value. .Linc_4blocks: .quad 4, 0 .text ___ # Number of powers of the hash key stored in the key struct. The powers are # stored from highest (H^NUM_H_POWERS) to lowest (H^1). my $NUM_H_POWERS = 16; my $OFFSETOFEND_H_POWERS = $NUM_H_POWERS * 16; # Offset to 'rounds' in AES_KEY struct my $OFFSETOF_AES_ROUNDS = 240; # The _ghash_mul macro multiplies the 128-bit lanes of \a by the corresponding # 128-bit lanes of \b and stores the reduced products in \dst. \t0, \t1, and # \t2 are temporary registers of the same size as \a and \b. # # The multiplications are done in GHASH's representation of the finite field # GF(2^128). Elements of GF(2^128) are represented as binary polynomials # (i.e. polynomials whose coefficients are bits) modulo a reducing polynomial # G. The GCM specification uses G = x^128 + x^7 + x^2 + x + 1. Addition is # just XOR, while multiplication is more complex and has two parts: (a) do # carryless multiplication of two 128-bit input polynomials to get a 256-bit # intermediate product polynomial, and (b) reduce the intermediate product to # 128 bits by adding multiples of G that cancel out terms in it. (Adding # multiples of G doesn't change which field element the polynomial represents.) # # Unfortunately, the GCM specification maps bits to/from polynomial # coefficients backwards from the natural order. In each byte it specifies the # highest bit to be the lowest order polynomial coefficient, *not* the highest! # This makes it nontrivial to work with the GHASH polynomials. We could # reflect the bits, but x86 doesn't have an instruction that does that. # # Instead, we operate on the values without bit-reflecting them. This *mostly* # just works, since XOR and carryless multiplication are symmetric with respect # to bit order, but it has some consequences. First, due to GHASH's byte # order, by skipping bit reflection, *byte* reflection becomes necessary to # give the polynomial terms a consistent order. E.g., considering an N-bit # value interpreted using the G = x^128 + x^7 + x^2 + x + 1 convention, bits 0 # through N-1 of the byte-reflected value represent the coefficients of x^(N-1) # through x^0, whereas bits 0 through N-1 of the non-byte-reflected value # represent x^7...x^0, x^15...x^8, ..., x^(N-1)...x^(N-8) which can't be worked # with. Fortunately, x86's vpshufb instruction can do byte reflection. # # Second, forgoing the bit reflection causes an extra multiple of x (still # using the G = x^128 + x^7 + x^2 + x + 1 convention) to be introduced by each # multiplication. This is because an M-bit by N-bit carryless multiplication # really produces a (M+N-1)-bit product, but in practice it's zero-extended to # M+N bits. In the G = x^128 + x^7 + x^2 + x + 1 convention, which maps bits # to polynomial coefficients backwards, this zero-extension actually changes # the product by introducing an extra factor of x. Therefore, users of this # macro must ensure that one of the inputs has an extra factor of x^-1, i.e. # the multiplicative inverse of x, to cancel out the extra x. # # Third, the backwards coefficients convention is just confusing to work with, # since it makes "low" and "high" in the polynomial math mean the opposite of # their normal meaning in computer programming. This can be solved by using an # alternative interpretation: the polynomial coefficients are understood to be # in the natural order, and the multiplication is actually \a * \b * x^-128 mod # x^128 + x^127 + x^126 + x^121 + 1. This doesn't change the inputs, outputs, # or the implementation at all; it just changes the mathematical interpretation # of what each instruction is doing. Starting from here, we'll use this # alternative interpretation, as it's easier to understand the code that way. # # Moving onto the implementation, the vpclmulqdq instruction does 64 x 64 => # 128-bit carryless multiplication, so we break the 128 x 128 multiplication # into parts as follows (the _L and _H suffixes denote low and high 64 bits): # # LO = a_L * b_L # MI = (a_L * b_H) + (a_H * b_L) # HI = a_H * b_H # # The 256-bit product is x^128*HI + x^64*MI + LO. LO, MI, and HI are 128-bit. # Note that MI "overlaps" with LO and HI. We don't consolidate MI into LO and # HI right away, since the way the reduction works makes that unnecessary. # # For the reduction, we cancel out the low 128 bits by adding multiples of G = # x^128 + x^127 + x^126 + x^121 + 1. This is done by two iterations, each of # which cancels out the next lowest 64 bits. Consider a value x^64*A + B, # where A and B are 128-bit. Adding B_L*G to that value gives: # # x^64*A + B + B_L*G # = x^64*A + x^64*B_H + B_L + B_L*(x^128 + x^127 + x^126 + x^121 + 1) # = x^64*A + x^64*B_H + B_L + x^128*B_L + x^64*B_L*(x^63 + x^62 + x^57) + B_L # = x^64*A + x^64*B_H + x^128*B_L + x^64*B_L*(x^63 + x^62 + x^57) + B_L + B_L # = x^64*(A + B_H + x^64*B_L + B_L*(x^63 + x^62 + x^57)) # # So: if we sum A, B with its halves swapped, and the low half of B times x^63 # + x^62 + x^57, we get a 128-bit value C where x^64*C is congruent to the # original value x^64*A + B. I.e., the low 64 bits got canceled out. # # We just need to apply this twice: first to fold LO into MI, and second to # fold the updated MI into HI. # # The needed three-argument XORs are done using the vpternlogd instruction with # immediate 0x96, since this is faster than two vpxord instructions. # # A potential optimization, assuming that b is fixed per-key (if a is fixed # per-key it would work the other way around), is to use one iteration of the # reduction described above to precompute a value c such that x^64*c = b mod G, # and then multiply a_L by c (and implicitly by x^64) instead of by b: # # MI = (a_L * c_L) + (a_H * b_L) # HI = (a_L * c_H) + (a_H * b_H) # # This would eliminate the LO part of the intermediate product, which would # eliminate the need to fold LO into MI. This would save two instructions, # including a vpclmulqdq. However, we currently don't use this optimization # because it would require twice as many per-key precomputed values. # # Using Karatsuba multiplication instead of "schoolbook" multiplication # similarly would save a vpclmulqdq but does not seem to be worth it. sub _ghash_mul { my ( $a, $b, $dst, $gfpoly, $t0, $t1, $t2 ) = @_; return <<___; vpclmulqdq \$0x00, $a, $b, $t0 # LO = a_L * b_L vpclmulqdq \$0x01, $a, $b, $t1 # MI_0 = a_L * b_H vpclmulqdq \$0x10, $a, $b, $t2 # MI_1 = a_H * b_L vpxord $t2, $t1, $t1 # MI = MI_0 + MI_1 vpclmulqdq \$0x01, $t0, $gfpoly, $t2 # LO_L*(x^63 + x^62 + x^57) vpshufd \$0x4e, $t0, $t0 # Swap halves of LO vpternlogd \$0x96, $t2, $t0, $t1 # Fold LO into MI vpclmulqdq \$0x11, $a, $b, $dst # HI = a_H * b_H vpclmulqdq \$0x01, $t1, $gfpoly, $t0 # MI_L*(x^63 + x^62 + x^57) vpshufd \$0x4e, $t1, $t1 # Swap halves of MI vpternlogd \$0x96, $t0, $t1, $dst # Fold MI into HI ___ } # GHASH-multiply the 128-bit lanes of \a by the 128-bit lanes of \b and add the # *unreduced* products to \lo, \mi, and \hi. sub _ghash_mul_noreduce { my ( $a, $b, $lo, $mi, $hi, $t0, $t1, $t2, $t3 ) = @_; return <<___; vpclmulqdq \$0x00, $a, $b, $t0 # a_L * b_L vpclmulqdq \$0x01, $a, $b, $t1 # a_L * b_H vpclmulqdq \$0x10, $a, $b, $t2 # a_H * b_L vpclmulqdq \$0x11, $a, $b, $t3 # a_H * b_H vpxord $t0, $lo, $lo vpternlogd \$0x96, $t2, $t1, $mi vpxord $t3, $hi, $hi ___ } # Reduce the unreduced products from \lo, \mi, and \hi and store the 128-bit # reduced products in \hi. See _ghash_mul for explanation of reduction. sub _ghash_reduce { my ( $lo, $mi, $hi, $gfpoly, $t0 ) = @_; return <<___; vpclmulqdq \$0x01, $lo, $gfpoly, $t0 vpshufd \$0x4e, $lo, $lo vpternlogd \$0x96, $t0, $lo, $mi vpclmulqdq \$0x01, $mi, $gfpoly, $t0 vpshufd \$0x4e, $mi, $mi vpternlogd \$0x96, $t0, $mi, $hi ___ } # This is a specialized version of _ghash_mul that computes \a * \a, i.e. it # squares \a. It skips computing MI = (a_L * a_H) + (a_H * a_L) = 0. sub _ghash_square { my ( $a, $dst, $gfpoly, $t0, $t1 ) = @_; return <<___; vpclmulqdq \$0x00, $a, $a, $t0 # LO = a_L * a_L vpclmulqdq \$0x11, $a, $a, $dst # HI = a_H * a_H vpclmulqdq \$0x01, $t0, $gfpoly, $t1 # LO_L*(x^63 + x^62 + x^57) vpshufd \$0x4e, $t0, $t0 # Swap halves of LO vpxor $t0, $t1, $t1 # Fold LO into MI vpclmulqdq \$0x01, $t1, $gfpoly, $t0 # MI_L*(x^63 + x^62 + x^57) vpshufd \$0x4e, $t1, $t1 # Swap halves of MI vpternlogd \$0x96, $t0, $t1, $dst # Fold MI into HI ___ } # void gcm_init_vpclmulqdq_avx512(u128 Htable[16], const uint64_t H[2]); # # Initialize |Htable| with powers of the GHASH subkey |H|. # # The powers are stored in the order H^NUM_H_POWERS to H^1. $code .= _begin_func "gcm_init_vpclmulqdq_avx512", 0; { # Function arguments my ( $HTABLE, $H_PTR ) = @argregs[ 0 .. 1 ]; # Additional local variables. %rax is used as a temporary register. my ( $TMP0, $TMP0_YMM, $TMP0_XMM ) = ( "%zmm0", "%ymm0", "%xmm0" ); my ( $TMP1, $TMP1_YMM, $TMP1_XMM ) = ( "%zmm1", "%ymm1", "%xmm1" ); my ( $TMP2, $TMP2_YMM, $TMP2_XMM ) = ( "%zmm2", "%ymm2", "%xmm2" ); my $POWERS_PTR = "%r8"; my $RNDKEYLAST_PTR = "%r9"; my ( $H_CUR, $H_CUR_YMM, $H_CUR_XMM ) = ( "%zmm3", "%ymm3", "%xmm3" ); my ( $H_INC, $H_INC_YMM, $H_INC_XMM ) = ( "%zmm4", "%ymm4", "%xmm4" ); my ( $GFPOLY, $GFPOLY_YMM, $GFPOLY_XMM ) = ( "%zmm5", "%ymm5", "%xmm5" ); $code .= <<___; # Get pointer to lowest set of key powers (located at end of array). lea $OFFSETOFEND_H_POWERS-64($HTABLE), $POWERS_PTR # Load the byte-reflected hash subkey. BoringSSL provides it in # byte-reflected form except the two halves are in the wrong order. vpshufd \$0x4e, ($H_PTR), $H_CUR_XMM # Finish preprocessing the first key power, H^1. Since this GHASH # implementation operates directly on values with the backwards bit # order specified by the GCM standard, it's necessary to preprocess the # raw key as follows. First, reflect its bytes. Second, multiply it # by x^-1 mod x^128 + x^7 + x^2 + x + 1 (if using the backwards # interpretation of polynomial coefficients), which can also be # interpreted as multiplication by x mod x^128 + x^127 + x^126 + x^121 # + 1 using the alternative, natural interpretation of polynomial # coefficients. For details, see the comment above _ghash_mul. # # Either way, for the multiplication the concrete operation performed # is a left shift of the 128-bit value by 1 bit, then an XOR with (0xc2 # << 120) | 1 if a 1 bit was carried out. However, there's no 128-bit # wide shift instruction, so instead double each of the two 64-bit # halves and incorporate the internal carry bit into the value XOR'd. vpshufd \$0xd3, $H_CUR_XMM, $TMP0_XMM vpsrad \$31, $TMP0_XMM, $TMP0_XMM vpaddq $H_CUR_XMM, $H_CUR_XMM, $H_CUR_XMM # H_CUR_XMM ^= TMP0_XMM & gfpoly_and_internal_carrybit vpternlogd \$0x78, .Lgfpoly_and_internal_carrybit(%rip), $TMP0_XMM, $H_CUR_XMM # Load the gfpoly constant. vbroadcasti32x4 .Lgfpoly(%rip), $GFPOLY # Square H^1 to get H^2. # # Note that as with H^1, all higher key powers also need an extra # factor of x^-1 (or x using the natural interpretation). Nothing # special needs to be done to make this happen, though: H^1 * H^1 would # end up with two factors of x^-1, but the multiplication consumes one. # So the product H^2 ends up with the desired one factor of x^-1. @{[ _ghash_square $H_CUR_XMM, $H_INC_XMM, $GFPOLY_XMM, $TMP0_XMM, $TMP1_XMM ]} # Create H_CUR_YMM = [H^2, H^1] and H_INC_YMM = [H^2, H^2]. vinserti128 \$1, $H_CUR_XMM, $H_INC_YMM, $H_CUR_YMM vinserti128 \$1, $H_INC_XMM, $H_INC_YMM, $H_INC_YMM # Create H_CUR = [H^4, H^3, H^2, H^1] and H_INC = [H^4, H^4, H^4, H^4]. @{[ _ghash_mul $H_INC_YMM, $H_CUR_YMM, $H_INC_YMM, $GFPOLY_YMM, $TMP0_YMM, $TMP1_YMM, $TMP2_YMM ]} vinserti64x4 \$1, $H_CUR_YMM, $H_INC, $H_CUR vshufi64x2 \$0, $H_INC, $H_INC, $H_INC # Store the lowest set of key powers. vmovdqu8 $H_CUR, ($POWERS_PTR) # Compute and store the remaining key powers. # Repeatedly multiply [H^(i+3), H^(i+2), H^(i+1), H^i] by # [H^4, H^4, H^4, H^4] to get [H^(i+7), H^(i+6), H^(i+5), H^(i+4)]. mov \$3, %eax .Lprecompute_next: sub \$64, $POWERS_PTR @{[ _ghash_mul $H_INC, $H_CUR, $H_CUR, $GFPOLY, $TMP0, $TMP1, $TMP2 ]} vmovdqu8 $H_CUR, ($POWERS_PTR) dec %eax jnz .Lprecompute_next vzeroupper # This is needed after using ymm or zmm registers. ___ } $code .= _end_func; # XOR together the 128-bit lanes of \src (whose low lane is \src_xmm) and store # the result in \dst_xmm. This implicitly zeroizes the other lanes of dst. sub _horizontal_xor { my ( $src, $src_xmm, $dst_xmm, $t0_xmm, $t1_xmm, $t2_xmm ) = @_; return <<___; vextracti32x4 \$1, $src, $t0_xmm vextracti32x4 \$2, $src, $t1_xmm vextracti32x4 \$3, $src, $t2_xmm vpxord $t0_xmm, $src_xmm, $dst_xmm vpternlogd \$0x96, $t1_xmm, $t2_xmm, $dst_xmm ___ } # Do one step of the GHASH update of the data blocks given in the vector # registers GHASHDATA[0-3]. \i specifies the step to do, 0 through 9. The # division into steps allows users of this macro to optionally interleave the # computation with other instructions. This macro uses the vector register # GHASH_ACC as input/output; GHASHDATA[0-3] as inputs that are clobbered; # H_POW[4-1], GFPOLY, and BSWAP_MASK as inputs that aren't clobbered; and # GHASHTMP[0-2] as temporaries. This macro handles the byte-reflection of the # data blocks. The parameter registers must be preserved across steps. # # The GHASH update does: GHASH_ACC = H_POW4*(GHASHDATA0 + GHASH_ACC) + # H_POW3*GHASHDATA1 + H_POW2*GHASHDATA2 + H_POW1*GHASHDATA3, where the # operations are vectorized operations on 512-bit vectors of 128-bit blocks. # The vectorized terms correspond to the following non-vectorized terms: # # H_POW4*(GHASHDATA0 + GHASH_ACC) => H^16*(blk0 + GHASH_ACC_XMM), # H^15*(blk1 + 0), H^14*(blk2 + 0), and H^13*(blk3 + 0) # H_POW3*GHASHDATA1 => H^12*blk4, H^11*blk5, H^10*blk6, and H^9*blk7 # H_POW2*GHASHDATA2 => H^8*blk8, H^7*blk9, H^6*blk10, and H^5*blk11 # H_POW1*GHASHDATA3 => H^4*blk12, H^3*blk13, H^2*blk14, and H^1*blk15 # # More concretely, this code does: # - Do vectorized "schoolbook" multiplications to compute the intermediate # 256-bit product of each block and its corresponding hash key power. # - Sum (XOR) the intermediate 256-bit products across vectors. # - Do a vectorized reduction of these 256-bit intermediate values to 128-bits # each. # - Sum (XOR) these values and store the 128-bit result in GHASH_ACC_XMM. # # See _ghash_mul for the full explanation of the operations performed for each # individual finite field multiplication and reduction. sub _ghash_step_4x { my ( $i, $BSWAP_MASK, $GHASHDATA0, $GHASHDATA1, $GHASHDATA2, $GHASHDATA3, $GHASHDATA0_XMM, $GHASHDATA1_XMM, $GHASHDATA2_XMM, $GHASHDATA3_XMM, $H_POW4, $H_POW3, $H_POW2, $H_POW1, $GFPOLY, $GHASHTMP0, $GHASHTMP1, $GHASHTMP2, $GHASH_ACC, $GHASH_ACC_XMM ) = @_; if ( $i == 0 ) { return <<___; vpshufb $BSWAP_MASK, $GHASHDATA0, $GHASHDATA0 vpxord $GHASH_ACC, $GHASHDATA0, $GHASHDATA0 vpshufb $BSWAP_MASK, $GHASHDATA1, $GHASHDATA1 vpshufb $BSWAP_MASK, $GHASHDATA2, $GHASHDATA2 ___ } elsif ( $i == 1 ) { return <<___; vpshufb $BSWAP_MASK, $GHASHDATA3, $GHASHDATA3 vpclmulqdq \$0x00, $H_POW4, $GHASHDATA0, $GHASH_ACC # LO_0 vpclmulqdq \$0x00, $H_POW3, $GHASHDATA1, $GHASHTMP0 # LO_1 vpclmulqdq \$0x00, $H_POW2, $GHASHDATA2, $GHASHTMP1 # LO_2 ___ } elsif ( $i == 2 ) { return <<___; vpxord $GHASHTMP0, $GHASH_ACC, $GHASH_ACC # sum(LO_{1,0}) vpclmulqdq \$0x00, $H_POW1, $GHASHDATA3, $GHASHTMP2 # LO_3 vpternlogd \$0x96, $GHASHTMP2, $GHASHTMP1, $GHASH_ACC # LO = sum(LO_{3,2,1,0}) vpclmulqdq \$0x01, $H_POW4, $GHASHDATA0, $GHASHTMP0 # MI_0 ___ } elsif ( $i == 3 ) { return <<___; vpclmulqdq \$0x01, $H_POW3, $GHASHDATA1, $GHASHTMP1 # MI_1 vpclmulqdq \$0x01, $H_POW2, $GHASHDATA2, $GHASHTMP2 # MI_2 vpternlogd \$0x96, $GHASHTMP2, $GHASHTMP1, $GHASHTMP0 # sum(MI_{2,1,0}) vpclmulqdq \$0x01, $H_POW1, $GHASHDATA3, $GHASHTMP1 # MI_3 ___ } elsif ( $i == 4 ) { return <<___; vpclmulqdq \$0x10, $H_POW4, $GHASHDATA0, $GHASHTMP2 # MI_4 vpternlogd \$0x96, $GHASHTMP2, $GHASHTMP1, $GHASHTMP0 # sum(MI_{4,3,2,1,0}) vpclmulqdq \$0x10, $H_POW3, $GHASHDATA1, $GHASHTMP1 # MI_5 vpclmulqdq \$0x10, $H_POW2, $GHASHDATA2, $GHASHTMP2 # MI_6 ___ } elsif ( $i == 5 ) { return <<___; vpternlogd \$0x96, $GHASHTMP2, $GHASHTMP1, $GHASHTMP0 # sum(MI_{6,5,4,3,2,1,0}) vpclmulqdq \$0x01, $GHASH_ACC, $GFPOLY, $GHASHTMP2 # LO_L*(x^63 + x^62 + x^57) vpclmulqdq \$0x10, $H_POW1, $GHASHDATA3, $GHASHTMP1 # MI_7 vpxord $GHASHTMP1, $GHASHTMP0, $GHASHTMP0 # MI = sum(MI_{7,6,5,4,3,2,1,0}) ___ } elsif ( $i == 6 ) { return <<___; vpshufd \$0x4e, $GHASH_ACC, $GHASH_ACC # Swap halves of LO vpclmulqdq \$0x11, $H_POW4, $GHASHDATA0, $GHASHDATA0 # HI_0 vpclmulqdq \$0x11, $H_POW3, $GHASHDATA1, $GHASHDATA1 # HI_1 vpclmulqdq \$0x11, $H_POW2, $GHASHDATA2, $GHASHDATA2 # HI_2 ___ } elsif ( $i == 7 ) { return <<___; vpternlogd \$0x96, $GHASHTMP2, $GHASH_ACC, $GHASHTMP0 # Fold LO into MI vpclmulqdq \$0x11, $H_POW1, $GHASHDATA3, $GHASHDATA3 # HI_3 vpternlogd \$0x96, $GHASHDATA2, $GHASHDATA1, $GHASHDATA0 # sum(HI_{2,1,0}) vpclmulqdq \$0x01, $GHASHTMP0, $GFPOLY, $GHASHTMP1 # MI_L*(x^63 + x^62 + x^57) ___ } elsif ( $i == 8 ) { return <<___; vpxord $GHASHDATA3, $GHASHDATA0, $GHASH_ACC # HI = sum(HI_{3,2,1,0}) vpshufd \$0x4e, $GHASHTMP0, $GHASHTMP0 # Swap halves of MI vpternlogd \$0x96, $GHASHTMP1, $GHASHTMP0, $GHASH_ACC # Fold MI into HI ___ } elsif ( $i == 9 ) { return _horizontal_xor $GHASH_ACC, $GHASH_ACC_XMM, $GHASH_ACC_XMM, $GHASHDATA0_XMM, $GHASHDATA1_XMM, $GHASHDATA2_XMM; } } # Update GHASH with four vectors of data blocks. See _ghash_step_4x for full # explanation. sub _ghash_4x { my $code = ""; for my $i ( 0 .. 9 ) { $code .= _ghash_step_4x $i, @_; } return $code; } # void gcm_gmult_vpclmulqdq_avx512(uint8_t Xi[16], const u128 Htable[16]); $code .= _begin_func "gcm_gmult_vpclmulqdq_avx512", 1; { my ( $GHASH_ACC_PTR, $HTABLE ) = @argregs[ 0 .. 1 ]; my ( $GHASH_ACC, $BSWAP_MASK, $H_POW1, $GFPOLY, $T0, $T1, $T2 ) = map( "%xmm$_", ( 0 .. 6 ) ); $code .= <<___; @{[ _save_xmmregs (6) ]} .seh_endprologue vmovdqu ($GHASH_ACC_PTR), $GHASH_ACC vmovdqu .Lbswap_mask(%rip), $BSWAP_MASK vmovdqu $OFFSETOFEND_H_POWERS-16($HTABLE), $H_POW1 vmovdqu .Lgfpoly(%rip), $GFPOLY vpshufb $BSWAP_MASK, $GHASH_ACC, $GHASH_ACC @{[ _ghash_mul $H_POW1, $GHASH_ACC, $GHASH_ACC, $GFPOLY, $T0, $T1, $T2 ]} vpshufb $BSWAP_MASK, $GHASH_ACC, $GHASH_ACC vmovdqu $GHASH_ACC, ($GHASH_ACC_PTR) # No need for vzeroupper, since only xmm registers were used. ___ } $code .= _end_func; # void gcm_ghash_vpclmulqdq_avx512(uint8_t Xi[16], const u128 Htable[16], # const uint8_t *in, size_t len); # # Using the key |Htable|, update the GHASH accumulator |Xi| with the data given # by |in| and |len|. |len| must be a multiple of 16. # # This function handles large amounts of AAD efficiently, while also keeping the # overhead low for small amounts of AAD which is the common case. TLS uses less # than one block of AAD, but (uncommonly) other use cases may use much more. $code .= _begin_func "gcm_ghash_vpclmulqdq_avx512", 1; { # Function arguments my ( $GHASH_ACC_PTR, $HTABLE, $AAD, $AADLEN ) = @argregs[ 0 .. 3 ]; # Additional local variables my ( $GHASHDATA0, $GHASHDATA0_XMM ) = ( "%zmm0", "%xmm0" ); my ( $GHASHDATA1, $GHASHDATA1_XMM ) = ( "%zmm1", "%xmm1" ); my ( $GHASHDATA2, $GHASHDATA2_XMM ) = ( "%zmm2", "%xmm2" ); my ( $GHASHDATA3, $GHASHDATA3_XMM ) = ( "%zmm3", "%xmm3" ); my @GHASHDATA = ( $GHASHDATA0, $GHASHDATA1, $GHASHDATA2, $GHASHDATA3 ); my @GHASHDATA_XMM = ( $GHASHDATA0_XMM, $GHASHDATA1_XMM, $GHASHDATA2_XMM, $GHASHDATA3_XMM ); my ( $BSWAP_MASK, $BSWAP_MASK_XMM ) = ( "%zmm4", "%xmm4" ); my ( $GHASH_ACC, $GHASH_ACC_XMM ) = ( "%zmm5", "%xmm5" ); my ( $H_POW4, $H_POW3, $H_POW2 ) = ( "%zmm6", "%zmm7", "%zmm8" ); my ( $H_POW1, $H_POW1_XMM ) = ( "%zmm9", "%xmm9" ); my ( $GFPOLY, $GFPOLY_XMM ) = ( "%zmm10", "%xmm10" ); my ( $GHASHTMP0, $GHASHTMP1, $GHASHTMP2 ) = ( "%zmm11", "%zmm12", "%zmm13" ); $code .= <<___; @{[ _save_xmmregs (6 .. 13) ]} .seh_endprologue # Load the bswap_mask and gfpoly constants. Since AADLEN is usually small, # usually only 128-bit vectors will be used. So as an optimization, don't # broadcast these constants to all 128-bit lanes quite yet. vmovdqu .Lbswap_mask(%rip), $BSWAP_MASK_XMM vmovdqu .Lgfpoly(%rip), $GFPOLY_XMM # Load the GHASH accumulator. vmovdqu ($GHASH_ACC_PTR), $GHASH_ACC_XMM vpshufb $BSWAP_MASK_XMM, $GHASH_ACC_XMM, $GHASH_ACC_XMM # Optimize for AADLEN < 64 by checking for AADLEN < 64 before AADLEN < 256. cmp \$64, $AADLEN jb .Laad_blockbyblock # AADLEN >= 64, so we'll operate on full vectors. Broadcast bswap_mask and # gfpoly to all 128-bit lanes. vshufi64x2 \$0, $BSWAP_MASK, $BSWAP_MASK, $BSWAP_MASK vshufi64x2 \$0, $GFPOLY, $GFPOLY, $GFPOLY # Load the lowest set of key powers. vmovdqu8 $OFFSETOFEND_H_POWERS-1*64($HTABLE), $H_POW1 cmp \$256, $AADLEN jb .Laad_loop_1x # AADLEN >= 256. Load the higher key powers. vmovdqu8 $OFFSETOFEND_H_POWERS-4*64($HTABLE), $H_POW4 vmovdqu8 $OFFSETOFEND_H_POWERS-3*64($HTABLE), $H_POW3 vmovdqu8 $OFFSETOFEND_H_POWERS-2*64($HTABLE), $H_POW2 # Update GHASH with 256 bytes of AAD at a time. .Laad_loop_4x: vmovdqu8 0*64($AAD), $GHASHDATA0 vmovdqu8 1*64($AAD), $GHASHDATA1 vmovdqu8 2*64($AAD), $GHASHDATA2 vmovdqu8 3*64($AAD), $GHASHDATA3 @{[ _ghash_4x $BSWAP_MASK, @GHASHDATA, @GHASHDATA_XMM, $H_POW4, $H_POW3, $H_POW2, $H_POW1, $GFPOLY, $GHASHTMP0, $GHASHTMP1, $GHASHTMP2, $GHASH_ACC, $GHASH_ACC_XMM ]} add \$256, $AAD sub \$256, $AADLEN cmp \$256, $AADLEN jae .Laad_loop_4x # Update GHASH with 64 bytes of AAD at a time. cmp \$64, $AADLEN jb .Laad_large_done .Laad_loop_1x: vmovdqu8 ($AAD), $GHASHDATA0 vpshufb $BSWAP_MASK, $GHASHDATA0, $GHASHDATA0 vpxord $GHASHDATA0, $GHASH_ACC, $GHASH_ACC @{[ _ghash_mul $H_POW1, $GHASH_ACC, $GHASH_ACC, $GFPOLY, $GHASHDATA0, $GHASHDATA1, $GHASHDATA2 ]} @{[ _horizontal_xor $GHASH_ACC, $GHASH_ACC_XMM, $GHASH_ACC_XMM, $GHASHDATA0_XMM, $GHASHDATA1_XMM, $GHASHDATA2_XMM ]} add \$64, $AAD sub \$64, $AADLEN cmp \$64, $AADLEN jae .Laad_loop_1x .Laad_large_done: # GHASH the remaining data 16 bytes at a time, using xmm registers only. .Laad_blockbyblock: test $AADLEN, $AADLEN jz .Laad_done vmovdqu $OFFSETOFEND_H_POWERS-16($HTABLE), $H_POW1_XMM .Laad_loop_blockbyblock: vmovdqu ($AAD), $GHASHDATA0_XMM vpshufb $BSWAP_MASK_XMM, $GHASHDATA0_XMM, $GHASHDATA0_XMM vpxor $GHASHDATA0_XMM, $GHASH_ACC_XMM, $GHASH_ACC_XMM @{[ _ghash_mul $H_POW1_XMM, $GHASH_ACC_XMM, $GHASH_ACC_XMM, $GFPOLY_XMM, $GHASHDATA0_XMM, $GHASHDATA1_XMM, $GHASHDATA2_XMM ]} add \$16, $AAD sub \$16, $AADLEN jnz .Laad_loop_blockbyblock .Laad_done: # Store the updated GHASH accumulator back to memory. vpshufb $BSWAP_MASK_XMM, $GHASH_ACC_XMM, $GHASH_ACC_XMM vmovdqu $GHASH_ACC_XMM, ($GHASH_ACC_PTR) vzeroupper # This is needed after using ymm or zmm registers. ___ } $code .= _end_func; # Do one non-last round of AES encryption on the counter blocks in aesdata[0-3] # using the round key that has been broadcast to all 128-bit lanes of round_key. sub _vaesenc_4x { my ( $round_key, $aesdata0, $aesdata1, $aesdata2, $aesdata3 ) = @_; return <<___; vaesenc $round_key, $aesdata0, $aesdata0 vaesenc $round_key, $aesdata1, $aesdata1 vaesenc $round_key, $aesdata2, $aesdata2 vaesenc $round_key, $aesdata3, $aesdata3 ___ } # Start the AES encryption of four vectors of counter blocks. sub _ctr_begin_4x { my ( $le_ctr, $le_ctr_inc, $bswap_mask, $rndkey0, $aesdata0, $aesdata1, $aesdata2, $aesdata3 ) = @_; return <<___; # Increment le_ctr four times to generate four vectors of little-endian # counter blocks, swap each to big-endian, and store them in aesdata[0-3]. vpshufb $bswap_mask, $le_ctr, $aesdata0 vpaddd $le_ctr_inc, $le_ctr, $le_ctr vpshufb $bswap_mask, $le_ctr, $aesdata1 vpaddd $le_ctr_inc, $le_ctr, $le_ctr vpshufb $bswap_mask, $le_ctr, $aesdata2 vpaddd $le_ctr_inc, $le_ctr, $le_ctr vpshufb $bswap_mask, $le_ctr, $aesdata3 vpaddd $le_ctr_inc, $le_ctr, $le_ctr # AES "round zero": XOR in the zero-th round key. vpxord $rndkey0, $aesdata0, $aesdata0 vpxord $rndkey0, $aesdata1, $aesdata1 vpxord $rndkey0, $aesdata2, $aesdata2 vpxord $rndkey0, $aesdata3, $aesdata3 ___ } # Do the last AES round for four vectors of counter blocks, XOR four vectors of # source data with the resulting keystream blocks, and write the result to the # destination buffer and ghashdata[0-3]. The implementation differs slightly as # it takes advantage of the property vaesenclast(key, a) ^ b == # vaesenclast(key ^ b, a) to reduce latency, but it has the same effect. sub _aesenclast_and_xor_4x { my ( $src, $dst, $rndkeylast, $aesdata0, $aesdata1, $aesdata2, $aesdata3, $ghashdata0, $ghashdata1, $ghashdata2, $ghashdata3 ) = @_; return <<___; vpxord 0*64($src), $rndkeylast, $ghashdata0 vpxord 1*64($src), $rndkeylast, $ghashdata1 vpxord 2*64($src), $rndkeylast, $ghashdata2 vpxord 3*64($src), $rndkeylast, $ghashdata3 vaesenclast $ghashdata0, $aesdata0, $ghashdata0 vaesenclast $ghashdata1, $aesdata1, $ghashdata1 vaesenclast $ghashdata2, $aesdata2, $ghashdata2 vaesenclast $ghashdata3, $aesdata3, $ghashdata3 vmovdqu8 $ghashdata0, 0*64($dst) vmovdqu8 $ghashdata1, 1*64($dst) vmovdqu8 $ghashdata2, 2*64($dst) vmovdqu8 $ghashdata3, 3*64($dst) ___ } my $g_update_macro_expansion_count = 0; # void aes_gcm_{enc,dec}_update_vaes_avx512(const uint8_t *in, uint8_t *out, # size_t len, const AES_KEY *key, # const uint8_t ivec[16], # const u128 Htable[16], # uint8_t Xi[16]); # # This macro generates a GCM encryption or decryption update function with the # above prototype (with \enc selecting which one). The function computes the # next portion of the CTR keystream, XOR's it with |len| bytes from |in|, and # writes the resulting encrypted or decrypted data to |out|. It also updates # the GHASH accumulator |Xi| using the next |len| ciphertext bytes. # # |len| must be a multiple of 16, except on the last call where it can be any # length. The caller must do any buffering needed to ensure this. Both # in-place and out-of-place en/decryption are supported. # # |ivec| must give the current counter in big-endian format. This function # loads the counter from |ivec| and increments the loaded counter as needed, but # it does *not* store the updated counter back to |ivec|. The caller must # update |ivec| if any more data segments follow. Internally, only the low # 32-bit word of the counter is incremented, following the GCM standard. sub _aes_gcm_update { my $local_label_suffix = "__func" . ++$g_update_macro_expansion_count; my ($enc) = @_; my $code = ""; # Function arguments my ( $SRC, $DST, $DATALEN, $AESKEY, $BE_CTR_PTR, $HTABLE, $GHASH_ACC_PTR ) = $win64 ? ( @argregs[ 0 .. 3 ], "%rsi", "%rdi", "%r12" ) : ( @argregs[ 0 .. 5 ], "%r12" ); # Additional local variables. # %rax, %k1, and %k2 are used as temporary registers. BE_CTR_PTR is # also available as a temporary register after the counter is loaded. # AES key length in bytes my ( $AESKEYLEN, $AESKEYLEN64 ) = ( "%r10d", "%r10" ); # Pointer to the last AES round key for the chosen AES variant my $RNDKEYLAST_PTR = "%r11"; # AESDATA[0-3] hold the counter blocks that are being encrypted by AES. my ( $AESDATA0, $AESDATA0_XMM ) = ( "%zmm0", "%xmm0" ); my ( $AESDATA1, $AESDATA1_XMM ) = ( "%zmm1", "%xmm1" ); my ( $AESDATA2, $AESDATA2_XMM ) = ( "%zmm2", "%xmm2" ); my ( $AESDATA3, $AESDATA3_XMM ) = ( "%zmm3", "%xmm3" ); my @AESDATA = ( $AESDATA0, $AESDATA1, $AESDATA2, $AESDATA3 ); # GHASHDATA[0-3] hold the ciphertext blocks and GHASH input data. my ( $GHASHDATA0, $GHASHDATA0_XMM ) = ( "%zmm4", "%xmm4" ); my ( $GHASHDATA1, $GHASHDATA1_XMM ) = ( "%zmm5", "%xmm5" ); my ( $GHASHDATA2, $GHASHDATA2_XMM ) = ( "%zmm6", "%xmm6" ); my ( $GHASHDATA3, $GHASHDATA3_XMM ) = ( "%zmm7", "%xmm7" ); my @GHASHDATA = ( $GHASHDATA0, $GHASHDATA1, $GHASHDATA2, $GHASHDATA3 ); my @GHASHDATA_XMM = ( $GHASHDATA0_XMM, $GHASHDATA1_XMM, $GHASHDATA2_XMM, $GHASHDATA3_XMM ); # BSWAP_MASK is the shuffle mask for byte-reflecting 128-bit values # using vpshufb, copied to all 128-bit lanes. my ( $BSWAP_MASK, $BSWAP_MASK_XMM ) = ( "%zmm8", "%xmm8" ); # RNDKEY temporarily holds the next AES round key. my $RNDKEY = "%zmm9"; # GHASH_ACC is the accumulator variable for GHASH. When fully reduced, # only the lowest 128-bit lane can be nonzero. When not fully reduced, # more than one lane may be used, and they need to be XOR'd together. my ( $GHASH_ACC, $GHASH_ACC_XMM ) = ( "%zmm10", "%xmm10" ); # LE_CTR_INC is the vector of 32-bit words that need to be added to a # vector of little-endian counter blocks to advance it forwards. my $LE_CTR_INC = "%zmm11"; # LE_CTR contains the next set of little-endian counter blocks. my $LE_CTR = "%zmm12"; # RNDKEY0, RNDKEYLAST, and RNDKEY_M[9-1] contain cached AES round keys, # copied to all 128-bit lanes. RNDKEY0 is the zero-th round key, # RNDKEYLAST the last, and RNDKEY_M\i the one \i-th from the last. my ( $RNDKEY0, $RNDKEYLAST, $RNDKEY_M9, $RNDKEY_M8, $RNDKEY_M7, $RNDKEY_M6, $RNDKEY_M5, $RNDKEY_M4, $RNDKEY_M3, $RNDKEY_M2, $RNDKEY_M1 ) = ( "%zmm13", "%zmm14", "%zmm15", "%zmm16", "%zmm17", "%zmm18", "%zmm19", "%zmm20", "%zmm21", "%zmm22", "%zmm23" ); # GHASHTMP[0-2] are temporary variables used by _ghash_step_4x. These # cannot coincide with anything used for AES encryption, since for # performance reasons GHASH and AES encryption are interleaved. my ( $GHASHTMP0, $GHASHTMP1, $GHASHTMP2 ) = ( "%zmm24", "%zmm25", "%zmm26" ); # H_POW[4-1] contain the powers of the hash key H^16...H^1. The descending # numbering reflects the order of the key powers. my ( $H_POW4, $H_POW3, $H_POW2, $H_POW1 ) = ( "%zmm27", "%zmm28", "%zmm29", "%zmm30" ); # GFPOLY contains the .Lgfpoly constant, copied to all 128-bit lanes. my $GFPOLY = "%zmm31"; my @ghash_4x_args = ( $BSWAP_MASK, @GHASHDATA, @GHASHDATA_XMM, $H_POW4, $H_POW3, $H_POW2, $H_POW1, $GFPOLY, $GHASHTMP0, $GHASHTMP1, $GHASHTMP2, $GHASH_ACC, $GHASH_ACC_XMM ); if ($win64) { $code .= <<___; @{[ _save_gpregs $BE_CTR_PTR, $HTABLE, $GHASH_ACC_PTR ]} mov 64(%rsp), $BE_CTR_PTR # arg5 mov 72(%rsp), $HTABLE # arg6 mov 80(%rsp), $GHASH_ACC_PTR # arg7 @{[ _save_xmmregs (6 .. 15) ]} .seh_endprologue ___ } else { $code .= <<___; @{[ _save_gpregs $GHASH_ACC_PTR ]} mov 16(%rsp), $GHASH_ACC_PTR # arg7 ___ } if ($enc) { $code .= <<___; #ifdef BORINGSSL_DISPATCH_TEST .extern BORINGSSL_function_hit movb \$1,BORINGSSL_function_hit+7(%rip) #endif ___ } $code .= <<___; # Load some constants. vbroadcasti32x4 .Lbswap_mask(%rip), $BSWAP_MASK vbroadcasti32x4 .Lgfpoly(%rip), $GFPOLY # Load the GHASH accumulator and the starting counter. # BoringSSL passes these values in big endian format. vmovdqu ($GHASH_ACC_PTR), $GHASH_ACC_XMM vpshufb $BSWAP_MASK_XMM, $GHASH_ACC_XMM, $GHASH_ACC_XMM vbroadcasti32x4 ($BE_CTR_PTR), $LE_CTR vpshufb $BSWAP_MASK, $LE_CTR, $LE_CTR # Load the AES key length in bytes. BoringSSL stores number of rounds # minus 1, so convert using: AESKEYLEN = 4 * aeskey->rounds - 20. movl $OFFSETOF_AES_ROUNDS($AESKEY), $AESKEYLEN lea -20(,$AESKEYLEN,4), $AESKEYLEN # Make RNDKEYLAST_PTR point to the last AES round key. This is the # round key with index 10, 12, or 14 for AES-128, AES-192, or AES-256 # respectively. Then load the zero-th and last round keys. lea 6*16($AESKEY,$AESKEYLEN64,4), $RNDKEYLAST_PTR vbroadcasti32x4 ($AESKEY), $RNDKEY0 vbroadcasti32x4 ($RNDKEYLAST_PTR), $RNDKEYLAST # Finish initializing LE_CTR by adding [0, 1, 2, 3] to its low words. vpaddd .Lctr_pattern(%rip), $LE_CTR, $LE_CTR # Load 4 into all 128-bit lanes of LE_CTR_INC. vbroadcasti32x4 .Linc_4blocks(%rip), $LE_CTR_INC # If there are at least 256 bytes of data, then continue into the loop # that processes 256 bytes of data at a time. Otherwise skip it. cmp \$256, $DATALEN jb .Lcrypt_loop_4x_done$local_label_suffix # Load powers of the hash key. vmovdqu8 $OFFSETOFEND_H_POWERS-4*64($HTABLE), $H_POW4 vmovdqu8 $OFFSETOFEND_H_POWERS-3*64($HTABLE), $H_POW3 vmovdqu8 $OFFSETOFEND_H_POWERS-2*64($HTABLE), $H_POW2 vmovdqu8 $OFFSETOFEND_H_POWERS-1*64($HTABLE), $H_POW1 ___ # Main loop: en/decrypt and hash 4 vectors at a time. # # When possible, interleave the AES encryption of the counter blocks # with the GHASH update of the ciphertext blocks. This improves # performance on many CPUs because the execution ports used by the VAES # instructions often differ from those used by vpclmulqdq and other # instructions used in GHASH. For example, many Intel CPUs dispatch # vaesenc to ports 0 and 1 and vpclmulqdq to port 5. # # The interleaving is easiest to do during decryption, since during # decryption the ciphertext blocks are immediately available. For # encryption, instead encrypt the first set of blocks, then hash those # blocks while encrypting the next set of blocks, repeat that as # needed, and finally hash the last set of blocks. if ($enc) { $code .= <<___; # Encrypt the first 4 vectors of plaintext blocks. Leave the resulting # ciphertext in GHASHDATA[0-3] for GHASH. @{[ _ctr_begin_4x $LE_CTR, $LE_CTR_INC, $BSWAP_MASK, $RNDKEY0, @AESDATA ]} lea 16($AESKEY), %rax .Lvaesenc_loop_first_4_vecs$local_label_suffix: vbroadcasti32x4 (%rax), $RNDKEY @{[ _vaesenc_4x $RNDKEY, @AESDATA ]} add \$16, %rax cmp %rax, $RNDKEYLAST_PTR jne .Lvaesenc_loop_first_4_vecs$local_label_suffix @{[ _aesenclast_and_xor_4x $SRC, $DST, $RNDKEYLAST, @AESDATA, @GHASHDATA ]} add \$256, $SRC add \$256, $DST sub \$256, $DATALEN cmp \$256, $DATALEN jb .Lghash_last_ciphertext_4x$local_label_suffix ___ } $code .= <<___; # Cache as many additional AES round keys as possible. vbroadcasti32x4 -9*16($RNDKEYLAST_PTR), $RNDKEY_M9 vbroadcasti32x4 -8*16($RNDKEYLAST_PTR), $RNDKEY_M8 vbroadcasti32x4 -7*16($RNDKEYLAST_PTR), $RNDKEY_M7 vbroadcasti32x4 -6*16($RNDKEYLAST_PTR), $RNDKEY_M6 vbroadcasti32x4 -5*16($RNDKEYLAST_PTR), $RNDKEY_M5 vbroadcasti32x4 -4*16($RNDKEYLAST_PTR), $RNDKEY_M4 vbroadcasti32x4 -3*16($RNDKEYLAST_PTR), $RNDKEY_M3 vbroadcasti32x4 -2*16($RNDKEYLAST_PTR), $RNDKEY_M2 vbroadcasti32x4 -1*16($RNDKEYLAST_PTR), $RNDKEY_M1 .Lcrypt_loop_4x$local_label_suffix: ___ # If decrypting, load more ciphertext blocks into GHASHDATA[0-3]. If # encrypting, GHASHDATA[0-3] already contain the previous ciphertext. if ( !$enc ) { $code .= <<___; vmovdqu8 0*64($SRC), $GHASHDATA0 vmovdqu8 1*64($SRC), $GHASHDATA1 vmovdqu8 2*64($SRC), $GHASHDATA2 vmovdqu8 3*64($SRC), $GHASHDATA3 ___ } $code .= <<___; # Start the AES encryption of the counter blocks. @{[ _ctr_begin_4x $LE_CTR, $LE_CTR_INC, $BSWAP_MASK, $RNDKEY0, @AESDATA ]} cmp \$24, $AESKEYLEN jl .Laes128$local_label_suffix je .Laes192$local_label_suffix # AES-256 vbroadcasti32x4 -13*16($RNDKEYLAST_PTR), $RNDKEY @{[ _vaesenc_4x $RNDKEY, @AESDATA ]} vbroadcasti32x4 -12*16($RNDKEYLAST_PTR), $RNDKEY @{[ _vaesenc_4x $RNDKEY, @AESDATA ]} .Laes192$local_label_suffix: vbroadcasti32x4 -11*16($RNDKEYLAST_PTR), $RNDKEY @{[ _vaesenc_4x $RNDKEY, @AESDATA ]} vbroadcasti32x4 -10*16($RNDKEYLAST_PTR), $RNDKEY @{[ _vaesenc_4x $RNDKEY, @AESDATA ]} .Laes128$local_label_suffix: # Prefetch the source data 512 bytes ahead into the L1 data cache, to # improve performance when the hardware prefetcher is disabled. Assumes the # L1 data cache line size is 64 bytes (de facto standard on x86_64). prefetcht0 512+0*64($SRC) prefetcht0 512+1*64($SRC) prefetcht0 512+2*64($SRC) prefetcht0 512+3*64($SRC) # Finish the AES encryption of the counter blocks in AESDATA[0-3], # interleaved with the GHASH update of the ciphertext blocks in # GHASHDATA[0-3]. @{[ _ghash_step_4x 0, @ghash_4x_args ]} @{[ _vaesenc_4x $RNDKEY_M9, @AESDATA ]} @{[ _ghash_step_4x 1, @ghash_4x_args ]} @{[ _vaesenc_4x $RNDKEY_M8, @AESDATA ]} @{[ _ghash_step_4x 2, @ghash_4x_args ]} @{[ _vaesenc_4x $RNDKEY_M7, @AESDATA ]} @{[ _ghash_step_4x 3, @ghash_4x_args ]} @{[ _vaesenc_4x $RNDKEY_M6, @AESDATA ]} @{[ _ghash_step_4x 4, @ghash_4x_args ]} @{[ _vaesenc_4x $RNDKEY_M5, @AESDATA ]} @{[ _ghash_step_4x 5, @ghash_4x_args ]} @{[ _vaesenc_4x $RNDKEY_M4, @AESDATA ]} @{[ _ghash_step_4x 6, @ghash_4x_args ]} @{[ _vaesenc_4x $RNDKEY_M3, @AESDATA ]} @{[ _ghash_step_4x 7, @ghash_4x_args ]} @{[ _vaesenc_4x $RNDKEY_M2, @AESDATA ]} @{[ _ghash_step_4x 8, @ghash_4x_args ]} @{[ _vaesenc_4x $RNDKEY_M1, @AESDATA ]} @{[ _ghash_step_4x 9, @ghash_4x_args ]} @{[ _aesenclast_and_xor_4x $SRC, $DST, $RNDKEYLAST, @AESDATA, @GHASHDATA ]} add \$256, $SRC add \$256, $DST sub \$256, $DATALEN cmp \$256, $DATALEN jae .Lcrypt_loop_4x$local_label_suffix ___ if ($enc) { # Update GHASH with the last set of ciphertext blocks. $code .= <<___; .Lghash_last_ciphertext_4x$local_label_suffix: @{[ _ghash_4x @ghash_4x_args ]} ___ } my $POWERS_PTR = $BE_CTR_PTR; # BE_CTR_PTR is free to be reused. $code .= <<___; .Lcrypt_loop_4x_done$local_label_suffix: # Check whether any data remains. test $DATALEN, $DATALEN jz .Ldone$local_label_suffix # The data length isn't a multiple of 256 bytes. Process the remaining # data of length 1 <= DATALEN < 256, up to one 64-byte vector at a time. # Going one vector at a time may seem inefficient compared to having # separate code paths for each possible number of vectors remaining. # However, using a loop keeps the code size down, and it performs # surprising well; modern CPUs will start executing the next iteration # before the previous one finishes and also predict the number of loop # iterations. For a similar reason, we roll up the AES rounds. # # On the last iteration, the remaining length may be less than 64 bytes. # Handle this using masking. # # Since there are enough key powers available for all remaining data, # there is no need to do a GHASH reduction after each iteration. # Instead, multiply each remaining block by its own key power, and only # do a GHASH reduction at the very end. # Make POWERS_PTR point to the key powers [H^N, H^(N-1), ...] where N # is the number of blocks that remain. mov $DATALEN, %rax neg %rax and \$-16, %rax # -round_up(DATALEN, 16) lea $OFFSETOFEND_H_POWERS($HTABLE,%rax), $POWERS_PTR ___ # Start collecting the unreduced GHASH intermediate value LO, MI, HI. my ( $LO, $LO_XMM ) = ( $GHASHDATA0, $GHASHDATA0_XMM ); my ( $MI, $MI_XMM ) = ( $GHASHDATA1, $GHASHDATA1_XMM ); my ( $HI, $HI_XMM ) = ( $GHASHDATA2, $GHASHDATA2_XMM ); $code .= <<___; vpxor $LO_XMM, $LO_XMM, $LO_XMM vpxor $MI_XMM, $MI_XMM, $MI_XMM vpxor $HI_XMM, $HI_XMM, $HI_XMM cmp \$64, $DATALEN jb .Lpartial_vec$local_label_suffix .Lcrypt_loop_1x$local_label_suffix: # Process a full 64-byte vector. # Encrypt a vector of counter blocks. vpshufb $BSWAP_MASK, $LE_CTR, $AESDATA0 vpaddd $LE_CTR_INC, $LE_CTR, $LE_CTR vpxord $RNDKEY0, $AESDATA0, $AESDATA0 lea 16($AESKEY), %rax .Lvaesenc_loop_tail_full_vec$local_label_suffix: vbroadcasti32x4 (%rax), $RNDKEY vaesenc $RNDKEY, $AESDATA0, $AESDATA0 add \$16, %rax cmp %rax, $RNDKEYLAST_PTR jne .Lvaesenc_loop_tail_full_vec$local_label_suffix vaesenclast $RNDKEYLAST, $AESDATA0, $AESDATA0 # XOR the data with the vector of keystream blocks. vmovdqu8 ($SRC), $AESDATA1 vpxord $AESDATA1, $AESDATA0, $AESDATA0 vmovdqu8 $AESDATA0, ($DST) # Update GHASH with the ciphertext blocks, without reducing. vmovdqu8 ($POWERS_PTR), $H_POW1 vpshufb $BSWAP_MASK, @{[ $enc ? $AESDATA0 : $AESDATA1 ]}, $AESDATA0 vpxord $GHASH_ACC, $AESDATA0, $AESDATA0 @{[ _ghash_mul_noreduce $H_POW1, $AESDATA0, $LO, $MI, $HI, $GHASHDATA3, $AESDATA1, $AESDATA2, $AESDATA3 ]} vpxor $GHASH_ACC_XMM, $GHASH_ACC_XMM, $GHASH_ACC_XMM add \$64, $POWERS_PTR add \$64, $SRC add \$64, $DST sub \$64, $DATALEN cmp \$64, $DATALEN jae .Lcrypt_loop_1x$local_label_suffix test $DATALEN, $DATALEN jz .Lreduce$local_label_suffix .Lpartial_vec$local_label_suffix: # Process a partial vector of length 1 <= DATALEN < 64. # Set the data mask %k1 to DATALEN 1's. # Set the key powers mask %k2 to round_up(DATALEN, 16) 1's. mov \$-1, %rax bzhi $DATALEN, %rax, %rax kmovq %rax, %k1 add \$15, $DATALEN and \$-16, $DATALEN mov \$-1, %rax bzhi $DATALEN, %rax, %rax kmovq %rax, %k2 # Encrypt one last vector of counter blocks. This does not need to be # masked. The counter does not need to be incremented here. vpshufb $BSWAP_MASK, $LE_CTR, $AESDATA0 vpxord $RNDKEY0, $AESDATA0, $AESDATA0 lea 16($AESKEY), %rax .Lvaesenc_loop_tail_partialvec$local_label_suffix: vbroadcasti32x4 (%rax), $RNDKEY vaesenc $RNDKEY, $AESDATA0, $AESDATA0 add \$16, %rax cmp %rax, $RNDKEYLAST_PTR jne .Lvaesenc_loop_tail_partialvec$local_label_suffix vaesenclast $RNDKEYLAST, $AESDATA0, $AESDATA0 # XOR the data with the appropriate number of keystream bytes. vmovdqu8 ($SRC), $AESDATA1\{%k1}{z} vpxord $AESDATA1, $AESDATA0, $AESDATA0 vmovdqu8 $AESDATA0, ($DST){%k1} # Update GHASH with the ciphertext block(s), without reducing. # # In the case of DATALEN < 64, the ciphertext is zero-padded to 64 # bytes. (If decrypting, it's done by the above masked load. If # encrypting, it's done by the below masked register-to-register move.) # Note that if DATALEN <= 48, there will be additional padding beyond # the padding of the last block specified by GHASH itself; i.e., there # may be whole block(s) that get processed by the GHASH multiplication # and reduction instructions but should not actually be included in the # GHASH. However, any such blocks are all-zeroes, and the values that # they're multiplied with are also all-zeroes. Therefore they just add # 0 * 0 = 0 to the final GHASH result, which makes no difference. vmovdqu8 ($POWERS_PTR), $H_POW1\{%k2}{z} @{[ $enc ? "vmovdqu8 $AESDATA0, $AESDATA1\{%k1}{z}" : "" ]} vpshufb $BSWAP_MASK, $AESDATA1, $AESDATA0 vpxord $GHASH_ACC, $AESDATA0, $AESDATA0 @{[ _ghash_mul_noreduce $H_POW1, $AESDATA0, $LO, $MI, $HI, $GHASHDATA3, $AESDATA1, $AESDATA2, $AESDATA3 ]} .Lreduce$local_label_suffix: # Finally, do the GHASH reduction. @{[ _ghash_reduce $LO, $MI, $HI, $GFPOLY, $AESDATA0 ]} @{[ _horizontal_xor $HI, $HI_XMM, $GHASH_ACC_XMM, $AESDATA0_XMM, $AESDATA1_XMM, $AESDATA2_XMM ]} .Ldone$local_label_suffix: # Store the updated GHASH accumulator back to memory. vpshufb $BSWAP_MASK_XMM, $GHASH_ACC_XMM, $GHASH_ACC_XMM vmovdqu $GHASH_ACC_XMM, ($GHASH_ACC_PTR) vzeroupper # This is needed after using ymm or zmm registers. ___ return $code; } $code .= _begin_func "aes_gcm_enc_update_vaes_avx512", 1; $code .= _aes_gcm_update 1; $code .= _end_func; $code .= _begin_func "aes_gcm_dec_update_vaes_avx512", 1; $code .= _aes_gcm_update 0; $code .= _end_func; print $code; close STDOUT or die "error closing STDOUT: $!"; exit 0;