Crypto++  5.6.4
Free C++ class library of cryptographic schemes
gcm.cpp
1 // gcm.cpp - written and placed in the public domain by Wei Dai
2 
3 // use "cl /EP /P /DCRYPTOPP_GENERATE_X64_MASM gcm.cpp" to generate MASM code
4 
5 #include "pch.h"
6 #include "config.h"
7 
8 #if CRYPTOPP_MSC_VERSION
9 # pragma warning(disable: 4189)
10 #endif
11 
12 #ifndef CRYPTOPP_IMPORTS
13 #ifndef CRYPTOPP_GENERATE_X64_MASM
14 
15 // Clang 3.3 integrated assembler crash on Linux. MacPorts GCC compile error. SunCC crash under SunCC 5.14 and below.
16 #if (defined(CRYPTOPP_LLVM_CLANG_VERSION) && (CRYPTOPP_LLVM_CLANG_VERSION < 30400)) || defined(CRYPTOPP_CLANG_INTEGRATED_ASSEMBLER) || (defined(__SUNPRO_CC) && __SUNPRO_CC <= 0x5140)
17 # undef CRYPTOPP_X86_ASM_AVAILABLE
18 # undef CRYPTOPP_X32_ASM_AVAILABLE
19 # undef CRYPTOPP_X64_ASM_AVAILABLE
20 # undef CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE
21 # undef CRYPTOPP_BOOL_SSSE3_ASM_AVAILABLE
22 # undef CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE
23 # define CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE 0
24 # define CRYPTOPP_BOOL_SSSE3_ASM_AVAILABLE 0
25 # define CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE 0
26 #endif
27 
28 #include "gcm.h"
29 #include "cpu.h"
30 
31 NAMESPACE_BEGIN(CryptoPP)
32 
33 word16 GCM_Base::s_reductionTable[256];
34 volatile bool GCM_Base::s_reductionTableInitialized = false;
35 
36 void GCM_Base::GCTR::IncrementCounterBy256()
37 {
38  IncrementCounterByOne(m_counterArray+BlockSize()-4, 3);
39 }
40 
41 #if 0
42 // preserved for testing
43 void gcm_gf_mult(const unsigned char *a, const unsigned char *b, unsigned char *c)
44 {
45  word64 Z0=0, Z1=0, V0, V1;
46 
48  Block::Get(a)(V0)(V1);
49 
50  for (int i=0; i<16; i++)
51  {
52  for (int j=0x80; j!=0; j>>=1)
53  {
54  int x = b[i] & j;
55  Z0 ^= x ? V0 : 0;
56  Z1 ^= x ? V1 : 0;
57  x = (int)V1 & 1;
58  V1 = (V1>>1) | (V0<<63);
59  V0 = (V0>>1) ^ (x ? W64LIT(0xe1) << 56 : 0);
60  }
61  }
62  Block::Put(NULL, c)(Z0)(Z1);
63 }
64 
65 __m128i _mm_clmulepi64_si128(const __m128i &a, const __m128i &b, int i)
66 {
67  word64 A[1] = {ByteReverse(((word64*)&a)[i&1])};
68  word64 B[1] = {ByteReverse(((word64*)&b)[i>>4])};
69 
70  PolynomialMod2 pa((byte *)A, 8);
71  PolynomialMod2 pb((byte *)B, 8);
72  PolynomialMod2 c = pa*pb;
73 
74  __m128i output;
75  for (int i=0; i<16; i++)
76  ((byte *)&output)[i] = c.GetByte(i);
77  return output;
78 }
79 #endif
80 
81 #if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE || CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE
82 inline static void SSE2_Xor16(byte *a, const byte *b, const byte *c)
83 {
84 #if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE
85  assert(IsAlignedOn(a,GetAlignmentOf<__m128i>()));
86  assert(IsAlignedOn(b,GetAlignmentOf<__m128i>()));
87  assert(IsAlignedOn(c,GetAlignmentOf<__m128i>()));
88  *(__m128i *)(void *)a = _mm_xor_si128(*(__m128i *)(void *)b, *(__m128i *)(void *)c);
89 #else
90  asm ("movdqa %1, %%xmm0; pxor %2, %%xmm0; movdqa %%xmm0, %0;" : "=m" (a[0]) : "m"(b[0]), "m"(c[0]));
91 #endif
92 }
93 #endif
94 
95 #if CRYPTOPP_BOOL_NEON_INTRINSICS_AVAILABLE
96 inline static void NEON_Xor16(byte *a, const byte *b, const byte *c)
97 {
98  assert(IsAlignedOn(a,GetAlignmentOf<uint64x2_t>()));
99  assert(IsAlignedOn(b,GetAlignmentOf<uint64x2_t>()));
100  assert(IsAlignedOn(c,GetAlignmentOf<uint64x2_t>()));
101  *(uint64x2_t*)a = veorq_u64(*(uint64x2_t*)b, *(uint64x2_t*)c);
102 }
103 #endif
104 
105 inline static void Xor16(byte *a, const byte *b, const byte *c)
106 {
107  assert(IsAlignedOn(a,GetAlignmentOf<word64>()));
108  assert(IsAlignedOn(b,GetAlignmentOf<word64>()));
109  assert(IsAlignedOn(c,GetAlignmentOf<word64>()));
110  ((word64 *)(void *)a)[0] = ((word64 *)(void *)b)[0] ^ ((word64 *)(void *)c)[0];
111  ((word64 *)(void *)a)[1] = ((word64 *)(void *)b)[1] ^ ((word64 *)(void *)c)[1];
112 }
113 
114 #if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE
115 CRYPTOPP_ALIGN_DATA(16)
116 static const word64 s_clmulConstants64[] = {
117  W64LIT(0xe100000000000000), W64LIT(0xc200000000000000),
118  W64LIT(0x08090a0b0c0d0e0f), W64LIT(0x0001020304050607),
119  W64LIT(0x0001020304050607), W64LIT(0x08090a0b0c0d0e0f)};
120 
121 static const __m128i *s_clmulConstants = (const __m128i *)(const void *)s_clmulConstants64;
122 static const unsigned int s_clmulTableSizeInBlocks = 8;
123 
124 inline __m128i CLMUL_Reduce(__m128i c0, __m128i c1, __m128i c2, const __m128i &r)
125 {
126  /*
127  The polynomial to be reduced is c0 * x^128 + c1 * x^64 + c2. c0t below refers to the most
128  significant half of c0 as a polynomial, which, due to GCM's bit reflection, are in the
129  rightmost bit positions, and the lowest byte addresses.
130 
131  c1 ^= c0t * 0xc200000000000000
132  c2t ^= c0t
133  t = shift (c1t ^ c0b) left 1 bit
134  c2 ^= t * 0xe100000000000000
135  c2t ^= c1b
136  shift c2 left 1 bit and xor in lowest bit of c1t
137  */
138 #if 0 // MSVC 2010 workaround: see http://connect.microsoft.com/VisualStudio/feedback/details/575301
139  c2 = _mm_xor_si128(c2, _mm_move_epi64(c0));
140 #else
141  c1 = _mm_xor_si128(c1, _mm_slli_si128(c0, 8));
142 #endif
143  c1 = _mm_xor_si128(c1, _mm_clmulepi64_si128(c0, r, 0x10));
144  c0 = _mm_srli_si128(c0, 8);
145  c0 = _mm_xor_si128(c0, c1);
146  c0 = _mm_slli_epi64(c0, 1);
147  c0 = _mm_clmulepi64_si128(c0, r, 0);
148  c2 = _mm_xor_si128(c2, c0);
149  c2 = _mm_xor_si128(c2, _mm_srli_si128(c1, 8));
150  c1 = _mm_unpacklo_epi64(c1, c2);
151  c1 = _mm_srli_epi64(c1, 63);
152  c2 = _mm_slli_epi64(c2, 1);
153  return _mm_xor_si128(c2, c1);
154 }
155 
156 inline __m128i CLMUL_GF_Mul(const __m128i &x, const __m128i &h, const __m128i &r)
157 {
158  const __m128i c0 = _mm_clmulepi64_si128(x,h,0);
159  const __m128i c1 = _mm_xor_si128(_mm_clmulepi64_si128(x,h,1), _mm_clmulepi64_si128(x,h,0x10));
160  const __m128i c2 = _mm_clmulepi64_si128(x,h,0x11);
161 
162  return CLMUL_Reduce(c0, c1, c2, r);
163 }
164 #endif
165 
166 #if CRYPTOPP_BOOL_ARM_CRYPTO_INTRINSICS_AVAILABLE
167 
168 CRYPTOPP_ALIGN_DATA(16)
169 static const word64 s_clmulConstants64[] = {
170  W64LIT(0xe100000000000000), W64LIT(0xc200000000000000), // Used for ARM and x86; polynomial coefficients
171  W64LIT(0x08090a0b0c0d0e0f), W64LIT(0x0001020304050607), // Unused for ARM; used for x86 _mm_shuffle_epi8
172  W64LIT(0x0001020304050607), W64LIT(0x08090a0b0c0d0e0f) // Unused for ARM; used for x86 _mm_shuffle_epi8
173 };
174 
175 static const uint64x2_t *s_clmulConstants = (const uint64x2_t *)s_clmulConstants64;
176 static const unsigned int s_clmulTableSizeInBlocks = 8;
177 
178 inline uint64x2_t PMULL_Reduce(uint64x2_t c0, uint64x2_t c1, uint64x2_t c2, const uint64x2_t &r)
179 {
180  // See comments fo CLMUL_Reduce
181 
182  c1 = veorq_u64(c1, (uint64x2_t)vextq_u8(vdupq_n_u8(0), (uint8x16_t)c0, 8));
183  c1 = veorq_u64(c1, (uint64x2_t)vmull_p64(vgetq_lane_u64(c0, 0), vgetq_lane_u64(r, 1)));
184  c0 = (uint64x2_t)vextq_u8((uint8x16_t)c0, vdupq_n_u8(0), 8);
185  c0 = veorq_u64(c0, c1);
186  c0 = vshlq_n_u64(c0, 1);
187  c0 = (uint64x2_t)vmull_p64(vgetq_lane_u64(c0, 0), vgetq_lane_u64(r, 0));
188  c2 = veorq_u64(c2, c0);
189  c2 = veorq_u64(c2, (uint64x2_t)vextq_u8((uint8x16_t)c1, vdupq_n_u8(0), 8));
190  c1 = vcombine_u64(vget_low_u64(c1), vget_low_u64(c2));
191  c1 = vshrq_n_u64(c1, 63);
192  c2 = vshlq_n_u64(c2, 1);
193 
194  return veorq_u64(c2, c1);
195 }
196 
197 inline uint64x2_t PMULL_GF_Mul(const uint64x2_t &x, const uint64x2_t &h, const uint64x2_t &r)
198 {
199  const uint64x2_t c0 = (uint64x2_t)vmull_p64(vgetq_lane_u64(x, 0), vgetq_lane_u64(h, 0));
200  const uint64x2_t c1 = veorq_u64((uint64x2_t)vmull_p64(vgetq_lane_u64(x, 1), vgetq_lane_u64(h,0)),
201  (uint64x2_t)vmull_p64(vgetq_lane_u64(x, 0), vgetq_lane_u64(h, 1)));
202  const uint64x2_t c2 = (uint64x2_t)vmull_high_p64((poly64x2_t)x, (poly64x2_t)h);
203 
204  return PMULL_Reduce(c0, c1, c2, r);
205 }
206 #endif
207 
208 void GCM_Base::SetKeyWithoutResync(const byte *userKey, size_t keylength, const NameValuePairs &params)
209 {
210  BlockCipher &blockCipher = AccessBlockCipher();
211  blockCipher.SetKey(userKey, keylength, params);
212 
213  if (blockCipher.BlockSize() != REQUIRED_BLOCKSIZE)
214  throw InvalidArgument(AlgorithmName() + ": block size of underlying block cipher is not 16");
215 
216  int tableSize, i, j, k;
217 
218 #if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE
219  if (HasCLMUL())
220  {
221  // Avoid "parameter not used" error and suppress Coverity finding
222  (void)params.GetIntValue(Name::TableSize(), tableSize);
223  tableSize = s_clmulTableSizeInBlocks * REQUIRED_BLOCKSIZE;
224  }
225  else
226 #elif CRYPTOPP_BOOL_ARM_CRYPTO_INTRINSICS_AVAILABLE
227  if (HasPMULL())
228  {
229  // Avoid "parameter not used" error and suppress Coverity finding
230  (void)params.GetIntValue(Name::TableSize(), tableSize);
231  tableSize = s_clmulTableSizeInBlocks * REQUIRED_BLOCKSIZE;
232  }
233  else
234 #endif
235  {
236  if (params.GetIntValue(Name::TableSize(), tableSize))
237  tableSize = (tableSize >= 64*1024) ? 64*1024 : 2*1024;
238  else
239  tableSize = (GetTablesOption() == GCM_64K_Tables) ? 64*1024 : 2*1024;
240 
241 #if defined(_MSC_VER) && (_MSC_VER >= 1300 && _MSC_VER < 1400)
242  // VC 2003 workaround: compiler generates bad code for 64K tables
243  tableSize = 2*1024;
244 #endif
245  }
246 
247  m_buffer.resize(3*REQUIRED_BLOCKSIZE + tableSize);
248  byte *table = MulTable();
249  byte *hashKey = HashKey();
250  memset(hashKey, 0, REQUIRED_BLOCKSIZE);
251  blockCipher.ProcessBlock(hashKey);
252 
253 #if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE
254  if (HasCLMUL())
255  {
256  const __m128i r = s_clmulConstants[0];
257  __m128i h0 = _mm_shuffle_epi8(_mm_load_si128((__m128i *)(void *)hashKey), s_clmulConstants[1]);
258  __m128i h = h0;
259 
260  for (i=0; i<tableSize; i+=32)
261  {
262  __m128i h1 = CLMUL_GF_Mul(h, h0, r);
263  _mm_storel_epi64((__m128i *)(void *)(table+i), h);
264  _mm_storeu_si128((__m128i *)(void *)(table+i+16), h1);
265  _mm_storeu_si128((__m128i *)(void *)(table+i+8), h);
266  _mm_storel_epi64((__m128i *)(void *)(table+i+8), h1);
267  h = CLMUL_GF_Mul(h1, h0, r);
268  }
269 
270  return;
271  }
272 #elif CRYPTOPP_BOOL_ARM_CRYPTO_INTRINSICS_AVAILABLE
273  if (HasPMULL())
274  {
275  const uint64x2_t r = s_clmulConstants[0];
276  const uint64x2_t t = vld1q_u64((uint64_t *)hashKey);
277  const uint64x2_t h0 = (uint64x2_t)vrev64q_u8((uint8x16_t)vcombine_u64(vget_high_u64(t), vget_low_u64(t)));
278 
279  uint64x2_t h = h0;
280  for (i=0; i<tableSize-32; i+=32)
281  {
282  const uint64x2_t h1 = PMULL_GF_Mul(h, h0, r);
283  vst1_u64((uint64_t *)(table+i), vget_low_u64(h));
284  vst1q_u64((uint64_t *)(table+i+16), h1);
285  vst1q_u64((uint64_t *)(table+i+8), h);
286  vst1_u64((uint64_t *)(table+i+8), vget_low_u64(h1));
287  h = PMULL_GF_Mul(h1, h0, r);
288  }
289 
290  const uint64x2_t h1 = PMULL_GF_Mul(h, h0, r);
291  vst1_u64((uint64_t *)(table+i), vget_low_u64(h));
292  vst1q_u64((uint64_t *)(table+i+16), h1);
293  vst1q_u64((uint64_t *)(table+i+8), h);
294  vst1_u64((uint64_t *)(table+i+8), vget_low_u64(h1));
295 
296  return;
297  }
298 #endif
299 
300  word64 V0, V1;
301  typedef BlockGetAndPut<word64, BigEndian> Block;
302  Block::Get(hashKey)(V0)(V1);
303 
304  if (tableSize == 64*1024)
305  {
306  for (i=0; i<128; i++)
307  {
308  k = i%8;
309  Block::Put(NULL, table+(i/8)*256*16+(size_t(1)<<(11-k)))(V0)(V1);
310 
311  int x = (int)V1 & 1;
312  V1 = (V1>>1) | (V0<<63);
313  V0 = (V0>>1) ^ (x ? W64LIT(0xe1) << 56 : 0);
314  }
315 
316  for (i=0; i<16; i++)
317  {
318  memset(table+i*256*16, 0, 16);
319 #if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE || CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE
320  if (HasSSE2())
321  for (j=2; j<=0x80; j*=2)
322  for (k=1; k<j; k++)
323  SSE2_Xor16(table+i*256*16+(j+k)*16, table+i*256*16+j*16, table+i*256*16+k*16);
324  else
325 #elif CRYPTOPP_BOOL_NEON_INTRINSICS_AVAILABLE
326  if (HasNEON())
327  for (j=2; j<=0x80; j*=2)
328  for (k=1; k<j; k++)
329  NEON_Xor16(table+i*256*16+(j+k)*16, table+i*256*16+j*16, table+i*256*16+k*16);
330  else
331 #endif
332  for (j=2; j<=0x80; j*=2)
333  for (k=1; k<j; k++)
334  Xor16(table+i*256*16+(j+k)*16, table+i*256*16+j*16, table+i*256*16+k*16);
335  }
336  }
337  else
338  {
339  if (!s_reductionTableInitialized)
340  {
341  s_reductionTable[0] = 0;
342  word16 x = 0x01c2;
343  s_reductionTable[1] = ByteReverse(x);
344  for (unsigned int ii=2; ii<=0x80; ii*=2)
345  {
346  x <<= 1;
347  s_reductionTable[ii] = ByteReverse(x);
348  for (unsigned int jj=1; jj<ii; jj++)
349  s_reductionTable[ii+jj] = s_reductionTable[ii] ^ s_reductionTable[jj];
350  }
351  s_reductionTableInitialized = true;
352  }
353 
354  for (i=0; i<128-24; i++)
355  {
356  k = i%32;
357  if (k < 4)
358  Block::Put(NULL, table+1024+(i/32)*256+(size_t(1)<<(7-k)))(V0)(V1);
359  else if (k < 8)
360  Block::Put(NULL, table+(i/32)*256+(size_t(1)<<(11-k)))(V0)(V1);
361 
362  int x = (int)V1 & 1;
363  V1 = (V1>>1) | (V0<<63);
364  V0 = (V0>>1) ^ (x ? W64LIT(0xe1) << 56 : 0);
365  }
366 
367  for (i=0; i<4; i++)
368  {
369  memset(table+i*256, 0, 16);
370  memset(table+1024+i*256, 0, 16);
371 #if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE || CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE
372  if (HasSSE2())
373  for (j=2; j<=8; j*=2)
374  for (k=1; k<j; k++)
375  {
376  SSE2_Xor16(table+i*256+(j+k)*16, table+i*256+j*16, table+i*256+k*16);
377  SSE2_Xor16(table+1024+i*256+(j+k)*16, table+1024+i*256+j*16, table+1024+i*256+k*16);
378  }
379  else
380 #elif CRYPTOPP_BOOL_NEON_INTRINSICS_AVAILABLE
381  if (HasNEON())
382  for (j=2; j<=8; j*=2)
383  for (k=1; k<j; k++)
384  {
385  NEON_Xor16(table+i*256+(j+k)*16, table+i*256+j*16, table+i*256+k*16);
386  NEON_Xor16(table+1024+i*256+(j+k)*16, table+1024+i*256+j*16, table+1024+i*256+k*16);
387  }
388  else
389 #endif
390  for (j=2; j<=8; j*=2)
391  for (k=1; k<j; k++)
392  {
393  Xor16(table+i*256+(j+k)*16, table+i*256+j*16, table+i*256+k*16);
394  Xor16(table+1024+i*256+(j+k)*16, table+1024+i*256+j*16, table+1024+i*256+k*16);
395  }
396  }
397  }
398 }
399 
400 inline void GCM_Base::ReverseHashBufferIfNeeded()
401 {
402 #if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE
403  if (HasCLMUL())
404  {
405  __m128i &x = *(__m128i *)(void *)HashBuffer();
406  x = _mm_shuffle_epi8(x, s_clmulConstants[1]);
407  }
408 #elif CRYPTOPP_BOOL_ARM_CRYPTO_INTRINSICS_AVAILABLE
409  if (HasPMULL())
410  {
412  {
413  const uint8x16_t x = vrev64q_u8(vld1q_u8(HashBuffer()));
414  vst1q_u8(HashBuffer(), (uint8x16_t)vcombine_u64(vget_high_u64((uint64x2_t)x), vget_low_u64((uint64x2_t)x)));
415  }
416  }
417 #endif
418 }
419 
420 void GCM_Base::Resync(const byte *iv, size_t len)
421 {
422  BlockCipher &cipher = AccessBlockCipher();
423  byte *hashBuffer = HashBuffer();
424 
425  if (len == 12)
426  {
427  memcpy(hashBuffer, iv, len);
428  memset(hashBuffer+len, 0, 3);
429  hashBuffer[len+3] = 1;
430  }
431  else
432  {
433  size_t origLen = len;
434  memset(hashBuffer, 0, HASH_BLOCKSIZE);
435 
436  if (len >= HASH_BLOCKSIZE)
437  {
438  len = GCM_Base::AuthenticateBlocks(iv, len);
439  iv += (origLen - len);
440  }
441 
442  if (len > 0)
443  {
444  memcpy(m_buffer, iv, len);
445  memset(m_buffer+len, 0, HASH_BLOCKSIZE-len);
446  GCM_Base::AuthenticateBlocks(m_buffer, HASH_BLOCKSIZE);
447  }
448 
449  PutBlock<word64, BigEndian, true>(NULL, m_buffer)(0)(origLen*8);
450  GCM_Base::AuthenticateBlocks(m_buffer, HASH_BLOCKSIZE);
451 
452  ReverseHashBufferIfNeeded();
453  }
454 
455  if (m_state >= State_IVSet)
456  m_ctr.Resynchronize(hashBuffer, REQUIRED_BLOCKSIZE);
457  else
458  m_ctr.SetCipherWithIV(cipher, hashBuffer);
459 
460  m_ctr.Seek(HASH_BLOCKSIZE);
461 
462  memset(hashBuffer, 0, HASH_BLOCKSIZE);
463 }
464 
465 unsigned int GCM_Base::OptimalDataAlignment() const
466 {
467  return
468 #if CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE || defined(CRYPTOPP_X64_MASM_AVAILABLE)
469  HasSSE2() ? 16 :
470 #elif CRYPTOPP_BOOL_NEON_INTRINSICS_AVAILABLE
471  HasNEON() ? 16 :
472 #endif
473  GetBlockCipher().OptimalDataAlignment();
474 }
475 
476 #if CRYPTOPP_MSC_VERSION
477 # pragma warning(disable: 4731) // frame pointer register 'ebp' modified by inline assembly code
478 #endif
479 
480 #endif // #ifndef CRYPTOPP_GENERATE_X64_MASM
481 
482 #ifdef CRYPTOPP_X64_MASM_AVAILABLE
483 extern "C" {
484 void GCM_AuthenticateBlocks_2K(const byte *data, size_t blocks, word64 *hashBuffer, const word16 *reductionTable);
485 void GCM_AuthenticateBlocks_64K(const byte *data, size_t blocks, word64 *hashBuffer);
486 }
487 #endif
488 
489 #ifndef CRYPTOPP_GENERATE_X64_MASM
490 
491 size_t GCM_Base::AuthenticateBlocks(const byte *data, size_t len)
492 {
493 #if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE
494  if (HasCLMUL())
495  {
496  const __m128i *table = (const __m128i *)(const void *)MulTable();
497  __m128i x = _mm_load_si128((__m128i *)(void *)HashBuffer());
498  const __m128i r = s_clmulConstants[0], mask1 = s_clmulConstants[1], mask2 = s_clmulConstants[2];
499 
500  while (len >= 16)
501  {
502  size_t s = UnsignedMin(len/16, s_clmulTableSizeInBlocks), i=0;
503  __m128i d, d2 = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)(const void *)(data+(s-1)*16)), mask2);;
504  __m128i c0 = _mm_setzero_si128();
505  __m128i c1 = _mm_setzero_si128();
506  __m128i c2 = _mm_setzero_si128();
507 
508  while (true)
509  {
510  __m128i h0 = _mm_load_si128(table+i);
511  __m128i h1 = _mm_load_si128(table+i+1);
512  __m128i h2 = _mm_xor_si128(h0, h1);
513 
514  if (++i == s)
515  {
516  d = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)(const void *)data), mask1);
517  d = _mm_xor_si128(d, x);
518  c0 = _mm_xor_si128(c0, _mm_clmulepi64_si128(d, h0, 0));
519  c2 = _mm_xor_si128(c2, _mm_clmulepi64_si128(d, h1, 1));
520  d = _mm_xor_si128(d, _mm_shuffle_epi32(d, _MM_SHUFFLE(1, 0, 3, 2)));
521  c1 = _mm_xor_si128(c1, _mm_clmulepi64_si128(d, h2, 0));
522  break;
523  }
524 
525  d = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)(const void *)(data+(s-i)*16-8)), mask2);
526  c0 = _mm_xor_si128(c0, _mm_clmulepi64_si128(d2, h0, 1));
527  c2 = _mm_xor_si128(c2, _mm_clmulepi64_si128(d, h1, 1));
528  d2 = _mm_xor_si128(d2, d);
529  c1 = _mm_xor_si128(c1, _mm_clmulepi64_si128(d2, h2, 1));
530 
531  if (++i == s)
532  {
533  d = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)(const void *)data), mask1);
534  d = _mm_xor_si128(d, x);
535  c0 = _mm_xor_si128(c0, _mm_clmulepi64_si128(d, h0, 0x10));
536  c2 = _mm_xor_si128(c2, _mm_clmulepi64_si128(d, h1, 0x11));
537  d = _mm_xor_si128(d, _mm_shuffle_epi32(d, _MM_SHUFFLE(1, 0, 3, 2)));
538  c1 = _mm_xor_si128(c1, _mm_clmulepi64_si128(d, h2, 0x10));
539  break;
540  }
541 
542  d2 = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)(const void *)(data+(s-i)*16-8)), mask1);
543  c0 = _mm_xor_si128(c0, _mm_clmulepi64_si128(d, h0, 0x10));
544  c2 = _mm_xor_si128(c2, _mm_clmulepi64_si128(d2, h1, 0x10));
545  d = _mm_xor_si128(d, d2);
546  c1 = _mm_xor_si128(c1, _mm_clmulepi64_si128(d, h2, 0x10));
547  }
548  data += s*16;
549  len -= s*16;
550 
551  c1 = _mm_xor_si128(_mm_xor_si128(c1, c0), c2);
552  x = CLMUL_Reduce(c0, c1, c2, r);
553  }
554 
555  _mm_store_si128((__m128i *)(void *)HashBuffer(), x);
556  return len;
557  }
558 #elif CRYPTOPP_BOOL_ARM_CRYPTO_INTRINSICS_AVAILABLE
559  if (HasPMULL())
560  {
561  const uint64x2_t *table = (const uint64x2_t *)MulTable();
562  uint64x2_t x = vld1q_u64((const uint64_t*)HashBuffer());
563  const uint64x2_t r = s_clmulConstants[0];
564 
565  while (len >= 16)
566  {
567  size_t s = UnsignedMin(len/16, s_clmulTableSizeInBlocks), i=0;
568  uint64x2_t d, d2 = (uint64x2_t)vrev64q_u8((uint8x16_t)vld1q_u64((const uint64_t *)(data+(s-1)*16)));
569  uint64x2_t c0 = vdupq_n_u64(0);
570  uint64x2_t c1 = vdupq_n_u64(0);
571  uint64x2_t c2 = vdupq_n_u64(0);
572 
573  while (true)
574  {
575  const uint64x2_t h0 = vld1q_u64((const uint64_t*)(table+i));
576  const uint64x2_t h1 = vld1q_u64((const uint64_t*)(table+i+1));
577  const uint64x2_t h2 = veorq_u64(h0, h1);
578 
579  if (++i == s)
580  {
581  const uint64x2_t t1 = vld1q_u64((const uint64_t *)data);
582  d = veorq_u64((uint64x2_t)vrev64q_u8((uint8x16_t)vcombine_u64(vget_high_u64(t1), vget_low_u64(t1))), x);
583  c0 = veorq_u64(c0, (uint64x2_t)vmull_p64(vgetq_lane_u64(d, 0), vgetq_lane_u64(h0, 0)));
584  c2 = veorq_u64(c2, (uint64x2_t)vmull_p64(vgetq_lane_u64(d, 1), vgetq_lane_u64(h1, 0)));
585  d = veorq_u64(d, (uint64x2_t)vcombine_u32(vget_high_u32((uint32x4_t)d), vget_low_u32((uint32x4_t)d)));
586  c1 = veorq_u64(c1, (uint64x2_t)vmull_p64(vgetq_lane_u64(d, 0), vgetq_lane_u64(h2, 0)));
587 
588  break;
589  }
590 
591  d = (uint64x2_t)vrev64q_u8((uint8x16_t)vld1q_u64((const uint64_t *)(data+(s-i)*16-8)));
592  c0 = veorq_u64(c0, (uint64x2_t)vmull_p64(vgetq_lane_u64(d2, 1), vgetq_lane_u64(h0, 0)));
593  c2 = veorq_u64(c2, (uint64x2_t)vmull_p64(vgetq_lane_u64(d, 1), vgetq_lane_u64(h1, 0)));
594  d2 = veorq_u64(d2, d);
595  c1 = veorq_u64(c1, (uint64x2_t)vmull_p64(vgetq_lane_u64(d2, 1), vgetq_lane_u64(h2, 0)));
596 
597  if (++i == s)
598  {
599 
600  const uint64x2_t t2 = vld1q_u64((const uint64_t *)data);
601  d = veorq_u64((uint64x2_t)vrev64q_u8((uint8x16_t)vcombine_u64(vget_high_u64(t2), vget_low_u64(t2))), x);
602  c0 = veorq_u64(c0, (uint64x2_t)vmull_p64(vgetq_lane_u64(d, 0), vgetq_lane_u64(h0, 1)));
603  c2 = veorq_u64(c2, (uint64x2_t)vmull_high_p64((poly64x2_t)d, (poly64x2_t)h1));
604  d = veorq_u64(d, (uint64x2_t)vcombine_u32(vget_high_u32((uint32x4_t)d), vget_low_u32((uint32x4_t)d)));
605  c1 = veorq_u64(c1, (uint64x2_t)vmull_p64(vgetq_lane_u64(d, 0), vgetq_lane_u64(h2, 1)));
606 
607  break;
608  }
609 
610  const uint64x2_t t3 = vld1q_u64((uint64_t *)(data+(s-i)*16-8));
611  d2 = (uint64x2_t)vrev64q_u8((uint8x16_t)vcombine_u64(vget_high_u64(t3), vget_low_u64(t3)));
612  c0 = veorq_u64(c0, (uint64x2_t)vmull_p64(vgetq_lane_u64(d, 0), vgetq_lane_u64(h0, 1)));
613  c2 = veorq_u64(c2, (uint64x2_t)vmull_p64(vgetq_lane_u64(d2, 0), vgetq_lane_u64(h1, 1)));
614  d = veorq_u64(d, d2);
615  c1 = veorq_u64(c1, (uint64x2_t)vmull_p64(vgetq_lane_u64(d, 0), vgetq_lane_u64(h2, 1)));
616  }
617  data += s*16;
618  len -= s*16;
619 
620  c1 = veorq_u64(veorq_u64(c1, c0), c2);
621  x = PMULL_Reduce(c0, c1, c2, r);
622  }
623 
624  vst1q_u64((uint64_t *)HashBuffer(), x);
625  return len;
626 }
627 #endif
628 
630  word64 *hashBuffer = (word64 *)(void *)HashBuffer();
631  assert(IsAlignedOn(hashBuffer,GetAlignmentOf<word64>()));
632 
633  switch (2*(m_buffer.size()>=64*1024)
634 #if CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE || defined(CRYPTOPP_X64_MASM_AVAILABLE)
635  + HasSSE2()
636 //#elif CRYPTOPP_BOOL_NEON_INTRINSICS_AVAILABLE
637 // + HasNEON()
638 #endif
639  )
640  {
641  case 0: // non-SSE2 and 2K tables
642  {
643  byte *table = MulTable();
644  word64 x0 = hashBuffer[0], x1 = hashBuffer[1];
645 
646  do
647  {
648  word64 y0, y1, a0, a1, b0, b1, c0, c1, d0, d1;
649  Block::Get(data)(y0)(y1);
650  x0 ^= y0;
651  x1 ^= y1;
652 
653  data += HASH_BLOCKSIZE;
654  len -= HASH_BLOCKSIZE;
655 
656  #define READ_TABLE_WORD64_COMMON(a, b, c, d) *(word64 *)(void *)(table+(a*1024)+(b*256)+c+d*8)
657 
658  #ifdef IS_LITTLE_ENDIAN
659  #if CRYPTOPP_BOOL_SLOW_WORD64
660  word32 z0 = (word32)x0;
661  word32 z1 = (word32)(x0>>32);
662  word32 z2 = (word32)x1;
663  word32 z3 = (word32)(x1>>32);
664  #define READ_TABLE_WORD64(a, b, c, d, e) READ_TABLE_WORD64_COMMON((d%2), c, (d?(z##c>>((d?d-1:0)*4))&0xf0:(z##c&0xf)<<4), e)
665  #else
666  #define READ_TABLE_WORD64(a, b, c, d, e) READ_TABLE_WORD64_COMMON((d%2), c, ((d+8*b)?(x##a>>(((d+8*b)?(d+8*b)-1:1)*4))&0xf0:(x##a&0xf)<<4), e)
667  #endif
668  #define GF_MOST_SIG_8BITS(a) (a##1 >> 7*8)
669  #define GF_SHIFT_8(a) a##1 = (a##1 << 8) ^ (a##0 >> 7*8); a##0 <<= 8;
670  #else
671  #define READ_TABLE_WORD64(a, b, c, d, e) READ_TABLE_WORD64_COMMON((1-d%2), c, ((15-d-8*b)?(x##a>>(((15-d-8*b)?(15-d-8*b)-1:0)*4))&0xf0:(x##a&0xf)<<4), e)
672  #define GF_MOST_SIG_8BITS(a) (a##1 & 0xff)
673  #define GF_SHIFT_8(a) a##1 = (a##1 >> 8) ^ (a##0 << 7*8); a##0 >>= 8;
674  #endif
675 
676  #define GF_MUL_32BY128(op, a, b, c) \
677  a0 op READ_TABLE_WORD64(a, b, c, 0, 0) ^ READ_TABLE_WORD64(a, b, c, 1, 0);\
678  a1 op READ_TABLE_WORD64(a, b, c, 0, 1) ^ READ_TABLE_WORD64(a, b, c, 1, 1);\
679  b0 op READ_TABLE_WORD64(a, b, c, 2, 0) ^ READ_TABLE_WORD64(a, b, c, 3, 0);\
680  b1 op READ_TABLE_WORD64(a, b, c, 2, 1) ^ READ_TABLE_WORD64(a, b, c, 3, 1);\
681  c0 op READ_TABLE_WORD64(a, b, c, 4, 0) ^ READ_TABLE_WORD64(a, b, c, 5, 0);\
682  c1 op READ_TABLE_WORD64(a, b, c, 4, 1) ^ READ_TABLE_WORD64(a, b, c, 5, 1);\
683  d0 op READ_TABLE_WORD64(a, b, c, 6, 0) ^ READ_TABLE_WORD64(a, b, c, 7, 0);\
684  d1 op READ_TABLE_WORD64(a, b, c, 6, 1) ^ READ_TABLE_WORD64(a, b, c, 7, 1);\
685 
686  GF_MUL_32BY128(=, 0, 0, 0)
687  GF_MUL_32BY128(^=, 0, 1, 1)
688  GF_MUL_32BY128(^=, 1, 0, 2)
689  GF_MUL_32BY128(^=, 1, 1, 3)
690 
691  word32 r = (word32)s_reductionTable[GF_MOST_SIG_8BITS(d)] << 16;
692  GF_SHIFT_8(d)
693  c0 ^= d0; c1 ^= d1;
694  r ^= (word32)s_reductionTable[GF_MOST_SIG_8BITS(c)] << 8;
695  GF_SHIFT_8(c)
696  b0 ^= c0; b1 ^= c1;
697  r ^= s_reductionTable[GF_MOST_SIG_8BITS(b)];
698  GF_SHIFT_8(b)
699  a0 ^= b0; a1 ^= b1;
700  a0 ^= ConditionalByteReverse<word64>(LITTLE_ENDIAN_ORDER, r);
701  x0 = a0; x1 = a1;
702  }
703  while (len >= HASH_BLOCKSIZE);
704 
705  hashBuffer[0] = x0; hashBuffer[1] = x1;
706  return len;
707  }
708 
709  case 2: // non-SSE2 and 64K tables
710  {
711  byte *table = MulTable();
712  word64 x0 = hashBuffer[0], x1 = hashBuffer[1];
713 
714  do
715  {
716  word64 y0, y1, a0, a1;
717  Block::Get(data)(y0)(y1);
718  x0 ^= y0;
719  x1 ^= y1;
720 
721  data += HASH_BLOCKSIZE;
722  len -= HASH_BLOCKSIZE;
723 
724  #undef READ_TABLE_WORD64_COMMON
725  #undef READ_TABLE_WORD64
726 
727  #define READ_TABLE_WORD64_COMMON(a, c, d) *(word64 *)(void *)(table+(a)*256*16+(c)+(d)*8)
728 
729  #ifdef IS_LITTLE_ENDIAN
730  #if CRYPTOPP_BOOL_SLOW_WORD64
731  word32 z0 = (word32)x0;
732  word32 z1 = (word32)(x0>>32);
733  word32 z2 = (word32)x1;
734  word32 z3 = (word32)(x1>>32);
735  #define READ_TABLE_WORD64(b, c, d, e) READ_TABLE_WORD64_COMMON(c*4+d, (d?(z##c>>((d?d:1)*8-4))&0xff0:(z##c&0xff)<<4), e)
736  #else
737  #define READ_TABLE_WORD64(b, c, d, e) READ_TABLE_WORD64_COMMON(c*4+d, ((d+4*(c%2))?(x##b>>(((d+4*(c%2))?(d+4*(c%2)):1)*8-4))&0xff0:(x##b&0xff)<<4), e)
738  #endif
739  #else
740  #define READ_TABLE_WORD64(b, c, d, e) READ_TABLE_WORD64_COMMON(c*4+d, ((7-d-4*(c%2))?(x##b>>(((7-d-4*(c%2))?(7-d-4*(c%2)):1)*8-4))&0xff0:(x##b&0xff)<<4), e)
741  #endif
742 
743  #define GF_MUL_8BY128(op, b, c, d) \
744  a0 op READ_TABLE_WORD64(b, c, d, 0);\
745  a1 op READ_TABLE_WORD64(b, c, d, 1);\
746 
747  GF_MUL_8BY128(=, 0, 0, 0)
748  GF_MUL_8BY128(^=, 0, 0, 1)
749  GF_MUL_8BY128(^=, 0, 0, 2)
750  GF_MUL_8BY128(^=, 0, 0, 3)
751  GF_MUL_8BY128(^=, 0, 1, 0)
752  GF_MUL_8BY128(^=, 0, 1, 1)
753  GF_MUL_8BY128(^=, 0, 1, 2)
754  GF_MUL_8BY128(^=, 0, 1, 3)
755  GF_MUL_8BY128(^=, 1, 2, 0)
756  GF_MUL_8BY128(^=, 1, 2, 1)
757  GF_MUL_8BY128(^=, 1, 2, 2)
758  GF_MUL_8BY128(^=, 1, 2, 3)
759  GF_MUL_8BY128(^=, 1, 3, 0)
760  GF_MUL_8BY128(^=, 1, 3, 1)
761  GF_MUL_8BY128(^=, 1, 3, 2)
762  GF_MUL_8BY128(^=, 1, 3, 3)
763 
764  x0 = a0; x1 = a1;
765  }
766  while (len >= HASH_BLOCKSIZE);
767 
768  hashBuffer[0] = x0; hashBuffer[1] = x1;
769  return len;
770  }
771 #endif // #ifndef CRYPTOPP_GENERATE_X64_MASM
772 
773 #ifdef CRYPTOPP_X64_MASM_AVAILABLE
774  case 1: // SSE2 and 2K tables
775  GCM_AuthenticateBlocks_2K(data, len/16, hashBuffer, s_reductionTable);
776  return len % 16;
777  case 3: // SSE2 and 64K tables
778  GCM_AuthenticateBlocks_64K(data, len/16, hashBuffer);
779  return len % 16;
780 #endif
781 
782 #if CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE
783  case 1: // SSE2 and 2K tables
784  {
785  #ifdef __GNUC__
786  __asm__ __volatile__
787  (
788  INTEL_NOPREFIX
789  #elif defined(CRYPTOPP_GENERATE_X64_MASM)
790  ALIGN 8
791  GCM_AuthenticateBlocks_2K PROC FRAME
792  rex_push_reg rsi
793  push_reg rdi
794  push_reg rbx
795  .endprolog
796  mov rsi, r8
797  mov r11, r9
798  #else
799  AS2( mov WORD_REG(cx), data )
800  AS2( mov WORD_REG(dx), len )
801  AS2( mov WORD_REG(si), hashBuffer )
802  AS2( shr WORD_REG(dx), 4 )
803  #endif
804 
805  #if CRYPTOPP_BOOL_X32
806  AS1(push rbx)
807  AS1(push rbp)
808  #else
809  AS_PUSH_IF86( bx)
810  AS_PUSH_IF86( bp)
811  #endif
812 
813  #ifdef __GNUC__
814  AS2( mov AS_REG_7, WORD_REG(di))
815  #elif CRYPTOPP_BOOL_X86
816  AS2( lea AS_REG_7, s_reductionTable)
817  #endif
818 
819  AS2( movdqa xmm0, [WORD_REG(si)] )
820 
821  #define MUL_TABLE_0 WORD_REG(si) + 32
822  #define MUL_TABLE_1 WORD_REG(si) + 32 + 1024
823  #define RED_TABLE AS_REG_7
824 
825  ASL(0)
826  AS2( movdqu xmm4, [WORD_REG(cx)] )
827  AS2( pxor xmm0, xmm4 )
828 
829  AS2( movd ebx, xmm0 )
830  AS2( mov eax, AS_HEX(f0f0f0f0) )
831  AS2( and eax, ebx )
832  AS2( shl ebx, 4 )
833  AS2( and ebx, AS_HEX(f0f0f0f0) )
834  AS2( movzx edi, ah )
835  AS2( movdqa xmm5, XMMWORD_PTR [MUL_TABLE_1 + WORD_REG(di)] )
836  AS2( movzx edi, al )
837  AS2( movdqa xmm4, XMMWORD_PTR [MUL_TABLE_1 + WORD_REG(di)] )
838  AS2( shr eax, 16 )
839  AS2( movzx edi, ah )
840  AS2( movdqa xmm3, XMMWORD_PTR [MUL_TABLE_1 + WORD_REG(di)] )
841  AS2( movzx edi, al )
842  AS2( movdqa xmm2, XMMWORD_PTR [MUL_TABLE_1 + WORD_REG(di)] )
843 
844  #define SSE2_MUL_32BITS(i) \
845  AS2( psrldq xmm0, 4 )\
846  AS2( movd eax, xmm0 )\
847  AS2( and eax, AS_HEX(f0f0f0f0) )\
848  AS2( movzx edi, bh )\
849  AS2( pxor xmm5, XMMWORD_PTR [MUL_TABLE_0 + (i-1)*256 + WORD_REG(di)] )\
850  AS2( movzx edi, bl )\
851  AS2( pxor xmm4, XMMWORD_PTR [MUL_TABLE_0 + (i-1)*256 + WORD_REG(di)] )\
852  AS2( shr ebx, 16 )\
853  AS2( movzx edi, bh )\
854  AS2( pxor xmm3, XMMWORD_PTR [MUL_TABLE_0 + (i-1)*256 + WORD_REG(di)] )\
855  AS2( movzx edi, bl )\
856  AS2( pxor xmm2, XMMWORD_PTR [MUL_TABLE_0 + (i-1)*256 + WORD_REG(di)] )\
857  AS2( movd ebx, xmm0 )\
858  AS2( shl ebx, 4 )\
859  AS2( and ebx, AS_HEX(f0f0f0f0) )\
860  AS2( movzx edi, ah )\
861  AS2( pxor xmm5, XMMWORD_PTR [MUL_TABLE_1 + i*256 + WORD_REG(di)] )\
862  AS2( movzx edi, al )\
863  AS2( pxor xmm4, XMMWORD_PTR [MUL_TABLE_1 + i*256 + WORD_REG(di)] )\
864  AS2( shr eax, 16 )\
865  AS2( movzx edi, ah )\
866  AS2( pxor xmm3, XMMWORD_PTR [MUL_TABLE_1 + i*256 + WORD_REG(di)] )\
867  AS2( movzx edi, al )\
868  AS2( pxor xmm2, XMMWORD_PTR [MUL_TABLE_1 + i*256 + WORD_REG(di)] )\
869 
870  SSE2_MUL_32BITS(1)
871  SSE2_MUL_32BITS(2)
872  SSE2_MUL_32BITS(3)
873 
874  AS2( movzx edi, bh )
875  AS2( pxor xmm5, XMMWORD_PTR [MUL_TABLE_0 + 3*256 + WORD_REG(di)] )
876  AS2( movzx edi, bl )
877  AS2( pxor xmm4, XMMWORD_PTR [MUL_TABLE_0 + 3*256 + WORD_REG(di)] )
878  AS2( shr ebx, 16 )
879  AS2( movzx edi, bh )
880  AS2( pxor xmm3, XMMWORD_PTR [MUL_TABLE_0 + 3*256 + WORD_REG(di)] )
881  AS2( movzx edi, bl )
882  AS2( pxor xmm2, XMMWORD_PTR [MUL_TABLE_0 + 3*256 + WORD_REG(di)] )
883 
884  AS2( movdqa xmm0, xmm3 )
885  AS2( pslldq xmm3, 1 )
886  AS2( pxor xmm2, xmm3 )
887  AS2( movdqa xmm1, xmm2 )
888  AS2( pslldq xmm2, 1 )
889  AS2( pxor xmm5, xmm2 )
890 
891  AS2( psrldq xmm0, 15 )
892 #if (CRYPTOPP_LLVM_CLANG_VERSION >= 30600) || (CRYPTOPP_APPLE_CLANG_VERSION >= 70000)
893  AS2( movd edi, xmm0 )
894 #elif (defined(CRYPTOPP_LLVM_CLANG_VERSION) || defined(CRYPTOPP_APPLE_CLANG_VERSION)) && defined(CRYPTOPP_X64_ASM_AVAILABLE)
895  AS2( mov WORD_REG(di), xmm0 )
896 #else // GNU Assembler
897  AS2( movd WORD_REG(di), xmm0 )
898 #endif
899  AS2( movzx eax, WORD PTR [RED_TABLE + WORD_REG(di)*2] )
900  AS2( shl eax, 8 )
901 
902  AS2( movdqa xmm0, xmm5 )
903  AS2( pslldq xmm5, 1 )
904  AS2( pxor xmm4, xmm5 )
905 
906  AS2( psrldq xmm1, 15 )
907 #if (CRYPTOPP_LLVM_CLANG_VERSION >= 30600) || (CRYPTOPP_APPLE_CLANG_VERSION >= 70000)
908  AS2( movd edi, xmm1 )
909 #elif (defined(CRYPTOPP_LLVM_CLANG_VERSION) || defined(CRYPTOPP_APPLE_CLANG_VERSION)) && defined(CRYPTOPP_X64_ASM_AVAILABLE)
910  AS2( mov WORD_REG(di), xmm1 )
911 #else
912  AS2( movd WORD_REG(di), xmm1 )
913 #endif
914  AS2( xor ax, WORD PTR [RED_TABLE + WORD_REG(di)*2] )
915  AS2( shl eax, 8 )
916 
917  AS2( psrldq xmm0, 15 )
918 #if (CRYPTOPP_LLVM_CLANG_VERSION >= 30600) || (CRYPTOPP_APPLE_CLANG_VERSION >= 70000)
919  AS2( movd edi, xmm0 )
920 #elif (defined(CRYPTOPP_LLVM_CLANG_VERSION) || defined(CRYPTOPP_APPLE_CLANG_VERSION)) && defined(CRYPTOPP_X64_ASM_AVAILABLE)
921  AS2( mov WORD_REG(di), xmm0 )
922 #else
923  AS2( movd WORD_REG(di), xmm0 )
924 #endif
925  AS2( xor ax, WORD PTR [RED_TABLE + WORD_REG(di)*2] )
926 
927  AS2( movd xmm0, eax )
928  AS2( pxor xmm0, xmm4 )
929 
930  AS2( add WORD_REG(cx), 16 )
931  AS2( sub WORD_REG(dx), 1 )
932  ATT_NOPREFIX
933  ASJ( jnz, 0, b )
934  INTEL_NOPREFIX
935  AS2( movdqa [WORD_REG(si)], xmm0 )
936 
937  #if CRYPTOPP_BOOL_X32
938  AS1(pop rbp)
939  AS1(pop rbx)
940  #else
941  AS_POP_IF86( bp)
942  AS_POP_IF86( bx)
943  #endif
944 
945  #ifdef __GNUC__
946  ATT_PREFIX
947  :
948  : "c" (data), "d" (len/16), "S" (hashBuffer), "D" (s_reductionTable)
949  : "memory", "cc", "%eax"
950  #if CRYPTOPP_BOOL_X64
951  , "%ebx", "%r11"
952  #endif
953  );
954  #elif defined(CRYPTOPP_GENERATE_X64_MASM)
955  pop rbx
956  pop rdi
957  pop rsi
958  ret
959  GCM_AuthenticateBlocks_2K ENDP
960  #endif
961 
962  return len%16;
963  }
964  case 3: // SSE2 and 64K tables
965  {
966  #ifdef __GNUC__
967  __asm__ __volatile__
968  (
969  INTEL_NOPREFIX
970  #elif defined(CRYPTOPP_GENERATE_X64_MASM)
971  ALIGN 8
972  GCM_AuthenticateBlocks_64K PROC FRAME
973  rex_push_reg rsi
974  push_reg rdi
975  .endprolog
976  mov rsi, r8
977  #else
978  AS2( mov WORD_REG(cx), data )
979  AS2( mov WORD_REG(dx), len )
980  AS2( mov WORD_REG(si), hashBuffer )
981  AS2( shr WORD_REG(dx), 4 )
982  #endif
983 
984  AS2( movdqa xmm0, [WORD_REG(si)] )
985 
986  #undef MUL_TABLE
987  #define MUL_TABLE(i,j) WORD_REG(si) + 32 + (i*4+j)*256*16
988 
989  ASL(1)
990  AS2( movdqu xmm1, [WORD_REG(cx)] )
991  AS2( pxor xmm1, xmm0 )
992  AS2( pxor xmm0, xmm0 )
993 
994  #undef SSE2_MUL_32BITS
995  #define SSE2_MUL_32BITS(i) \
996  AS2( movd eax, xmm1 )\
997  AS2( psrldq xmm1, 4 )\
998  AS2( movzx edi, al )\
999  AS2( add WORD_REG(di), WORD_REG(di) )\
1000  AS2( pxor xmm0, [MUL_TABLE(i,0) + WORD_REG(di)*8] )\
1001  AS2( movzx edi, ah )\
1002  AS2( add WORD_REG(di), WORD_REG(di) )\
1003  AS2( pxor xmm0, [MUL_TABLE(i,1) + WORD_REG(di)*8] )\
1004  AS2( shr eax, 16 )\
1005  AS2( movzx edi, al )\
1006  AS2( add WORD_REG(di), WORD_REG(di) )\
1007  AS2( pxor xmm0, [MUL_TABLE(i,2) + WORD_REG(di)*8] )\
1008  AS2( movzx edi, ah )\
1009  AS2( add WORD_REG(di), WORD_REG(di) )\
1010  AS2( pxor xmm0, [MUL_TABLE(i,3) + WORD_REG(di)*8] )\
1011 
1012  SSE2_MUL_32BITS(0)
1013  SSE2_MUL_32BITS(1)
1014  SSE2_MUL_32BITS(2)
1015  SSE2_MUL_32BITS(3)
1016 
1017  AS2( add WORD_REG(cx), 16 )
1018  AS2( sub WORD_REG(dx), 1 )
1019  ATT_NOPREFIX
1020  ASJ( jnz, 1, b )
1021  INTEL_NOPREFIX
1022  AS2( movdqa [WORD_REG(si)], xmm0 )
1023 
1024  #ifdef __GNUC__
1025  ATT_PREFIX
1026  :
1027  : "c" (data), "d" (len/16), "S" (hashBuffer)
1028  : "memory", "cc", "%edi", "%eax"
1029  );
1030  #elif defined(CRYPTOPP_GENERATE_X64_MASM)
1031  pop rdi
1032  pop rsi
1033  ret
1034  GCM_AuthenticateBlocks_64K ENDP
1035  #endif
1036 
1037  return len%16;
1038  }
1039 #endif
1040 #ifndef CRYPTOPP_GENERATE_X64_MASM
1041  }
1042 
1043  return len%16;
1044 }
1045 
1046 void GCM_Base::AuthenticateLastHeaderBlock()
1047 {
1048  if (m_bufferedDataLength > 0)
1049  {
1050  memset(m_buffer+m_bufferedDataLength, 0, HASH_BLOCKSIZE-m_bufferedDataLength);
1051  m_bufferedDataLength = 0;
1052  GCM_Base::AuthenticateBlocks(m_buffer, HASH_BLOCKSIZE);
1053  }
1054 }
1055 
1056 void GCM_Base::AuthenticateLastConfidentialBlock()
1057 {
1058  GCM_Base::AuthenticateLastHeaderBlock();
1059  PutBlock<word64, BigEndian, true>(NULL, m_buffer)(m_totalHeaderLength*8)(m_totalMessageLength*8);
1060  GCM_Base::AuthenticateBlocks(m_buffer, HASH_BLOCKSIZE);
1061 }
1062 
1063 void GCM_Base::AuthenticateLastFooterBlock(byte *mac, size_t macSize)
1064 {
1065  m_ctr.Seek(0);
1066  ReverseHashBufferIfNeeded();
1067  m_ctr.ProcessData(mac, HashBuffer(), macSize);
1068 }
1069 
1070 NAMESPACE_END
1071 
1072 #endif // #ifndef CRYPTOPP_GENERATE_X64_MASM
1073 #endif
GCM block cipher base implementation.
Definition: gcm.h:27
An invalid argument was detected.
Definition: cryptlib.h:182
virtual void SetKey(const byte *key, size_t length, const NameValuePairs &params=g_nullNameValuePairs)
Sets or reset the key of this object.
Definition: cryptlib.cpp:101
unsigned int OptimalDataAlignment() const
Provides input and output data alignment for optimal performance.
Definition: gcm.cpp:465
void IncrementCounterByOne(byte *inout, unsigned int size)
Performs an addition with carry on a block of bytes.
Definition: misc.h:1013
Library configuration file.
Access a block of memory.
Definition: misc.h:2246
virtual unsigned int OptimalDataAlignment() const
Provides input and output data alignment for optimal performance.
Definition: cryptlib.cpp:233
byte order is little-endian
Definition: cryptlib.h:125
Polynomial with Coefficients in GF(2)
Definition: gf2n.h:18
Interface for one direction (encryption or decryption) of a block cipher.
Definition: cryptlib.h:1091
Use a table with 64K entries.
Definition: gcm.h:21
virtual unsigned int BlockSize() const =0
Provides the block size of the cipher.
bool IsAlignedOn(const void *ptr, unsigned int alignment)
Determines whether ptr is aligned to a minimum value.
Definition: misc.h:907
const char * TableSize()
int, in bytes
Definition: argnames.h:80
bool HasCLMUL()
Determines Carryless Multiply availability.
Definition: cpu.h:281
void ProcessBlock(const byte *inBlock, byte *outBlock) const
Encrypt or decrypt a block.
Definition: cryptlib.h:751
const T1 UnsignedMin(const T1 &a, const T2 &b)
Safe comparison of values that could be neagtive and incorrectly promoted.
Definition: misc.h:503
std::string AlgorithmName() const
Provides the name of this algorithm.
Definition: gcm.h:31
byte order is big-endian
Definition: cryptlib.h:127
const char * BlockSize()
int, in bytes
Definition: argnames.h:26
Functions for CPU features and intrinsics.
bool HasSSE2()
Determines SSE2 availability.
Definition: cpu.h:236
GCM block cipher mode of operation.
ByteOrder GetNativeByteOrder()
Returns NativeByteOrder as an enumerated ByteOrder value.
Definition: misc.h:940
byte GetByte(size_t n) const
return the n-th byte
Definition: gf2n.cpp:77
Access a block of memory.
Definition: misc.h:2208
Crypto++ library namespace.
bool GetIntValue(const char *name, int &value) const
Get a named value with type int.
Definition: cryptlib.h:371
byte ByteReverse(byte value)
Reverses bytes in a 8-bit value.
Definition: misc.h:1705
Interface for retrieving values given their names.
Definition: cryptlib.h:277