Added SHA3/SHAKE XOF functionality (aws#1839)

FIPS 203/204 require the SHAKE family of extensible-output-functions (XOF) used as a deterministic random number generator. SHAKE as a streaming XOF requires changes to the SHA3/SHAKE functionality in AWS-LC. Within the current architecture, the SHA3_Squeeze function will always calls the Keccak state permute function KeccakF1600(A). However, to support SHAKE as a streaming XOF, SHA3_Squeeze is called upon multiple times, to continually squeeze additional output from the Keccak state.

crypto/fipsmodule/sha/asm/keccak1600-armv8.pl

@@ -458,6 +458,8 @@
 	mov	$out,x1
 	mov	$len,x2
 	mov	$bsz,x3
+	cmp	x4, #0				// x4 = 'padded' argument; if !=0, perform Keccak first
+	bne .Lnext_block
 .Loop_squeeze:
 	ldr	x4,[x0],#8
 	cmp	$len,#8
@@ -470,6 +472,7 @@
 	beq	.Lsqueeze_done
 	subs	x3,x3,#8
 	bhi	.Loop_squeeze
+.Lnext_block:
 	mov	x0,$A_flat
 	bl	KeccakF1600
 	mov	x0,$A_flat

crypto/fipsmodule/sha/internal.h

@@ -80,7 +80,8 @@ struct keccak_st {
   size_t md_size;                                  // output length, variable in XOF (SHAKE)
   size_t buf_load;                                 // used bytes in below buffer
   uint8_t buf[SHA3_MAX_BLOCKSIZE];                 // should have at least the max data block size bytes
-  uint8_t pad;
+  uint8_t pad;                                     // padding character
+  uint8_t padded;                                  // denotes if padding has been performed
 };
 // Define SHA{n}[_{variant}]_ASM if sha{n}_block_data_order[_{variant}] is
 // defined in assembly.
@@ -377,9 +378,15 @@ OPENSSL_EXPORT int SHA3_Final(uint8_t *md, KECCAK1600_CTX *ctx);
 OPENSSL_EXPORT size_t SHA3_Absorb(uint64_t A[SHA3_ROWS][SHA3_ROWS],
                                   const uint8_t *data, size_t len, size_t r);
 
-// SHA3_Squeeze generate |out| hash value of |len| bytes.
+// SHA3_Squeeze generates |out| value of |len| bytes (per call). It can be called
+// multiple times when used as eXtendable Output Function. |padded| indicates
+// whether it is the first call to SHA3_Squeeze; i.e., if the current block has
+// been already processed and padded right after the last call to SHA3_Absorb.
+// Squeezes full blocks of |r| bytes each. When performing multiple squeezes, any
+// left over bytes from previous squeezes are not consumed, and |len| must be a
+// multiple of the block size (except on the final squeeze).
 OPENSSL_EXPORT void SHA3_Squeeze(uint64_t A[SHA3_ROWS][SHA3_ROWS],
-                                 uint8_t *out, size_t len, size_t r);
+                                 uint8_t *out, size_t len, size_t r, int padded);
 
 #if defined(__cplusplus)
 }  // extern "C"

crypto/fipsmodule/sha/keccak1600.c

@@ -356,16 +356,19 @@ size_t SHA3_Absorb(uint64_t A[SHA3_ROWS][SHA3_ROWS], const uint8_t *inp, size_t
     return len;
 }
 
- // SHA3_Squeeze is called once at the end to generate |out| hash value
- // of |len| bytes.
-void SHA3_Squeeze(uint64_t A[SHA3_ROWS][SHA3_ROWS], uint8_t *out, size_t len, size_t r)
+void SHA3_Squeeze(uint64_t A[SHA3_ROWS][SHA3_ROWS], uint8_t *out, size_t len, size_t r, int padded)
+// SHA3_Squeeze can be called multiple times to incrementally 
 {
     uint64_t *A_flat = (uint64_t *)A;
     size_t i, w = r / 8;
 
     assert(r < (25 * sizeof(A[0][0])) && (r % 8) == 0);
 
     while (len != 0) {
+        if (padded) {
+            KeccakF1600(A); 
+        }
+        padded = 1;
         for (i = 0; i < w && len != 0; i++) {
             uint64_t Ai = BitDeinterleave(A_flat[i]);
 
@@ -388,9 +391,6 @@ void SHA3_Squeeze(uint64_t A[SHA3_ROWS][SHA3_ROWS], uint8_t *out, size_t len, si
             out += 8;
             len -= 8;
         }
-        if (len != 0) {
-            KeccakF1600(A);
-        }
     }
 }
 
@@ -405,10 +405,10 @@ size_t SHA3_Absorb(uint64_t A[SHA3_ROWS][SHA3_ROWS], const uint8_t *inp, size_t
 }
 
 size_t SHA3_Squeeze_hw(uint64_t A[SHA3_ROWS][SHA3_ROWS], const uint8_t *out, size_t len,
-                        size_t r);
+                        size_t r, int padded);
 
-void SHA3_Squeeze(uint64_t A[SHA3_ROWS][SHA3_ROWS], uint8_t *out, size_t len, size_t r) {
-    SHA3_Squeeze_hw(A, out, len, r);
+void SHA3_Squeeze(uint64_t A[SHA3_ROWS][SHA3_ROWS], uint8_t *out, size_t len, size_t r, int padded) {
+    SHA3_Squeeze_hw(A, out, len, r, padded);
 }
 
 #endif // !KECCAK1600_ASM

crypto/fipsmodule/sha/sha3.c

@@ -10,7 +10,6 @@
 #include "internal.h"
 #include <string.h>
 
-
 uint8_t *SHA3_224(const uint8_t *data, size_t len,
                   uint8_t out[SHA3_224_DIGEST_LENGTH]) {
   FIPS_service_indicator_lock_state();
@@ -126,6 +125,7 @@ int SHAKE_Final(uint8_t *md, KECCAK1600_CTX *ctx, size_t len) {
 void SHA3_Reset(KECCAK1600_CTX *ctx) {
   memset(ctx->A, 0, sizeof(ctx->A));
   ctx->buf_load = 0;
+  ctx->padded = 0;
 }
 
 int SHA3_Init(KECCAK1600_CTX *ctx, uint8_t pad, size_t bit_len) {
@@ -144,7 +144,8 @@ int SHA3_Init(KECCAK1600_CTX *ctx, uint8_t pad, size_t bit_len) {
   } else {
     return 0;
   }
-
+  ctx->padded = 0;
+
   if (block_size <= sizeof(ctx->buf)) {
     SHA3_Reset(ctx);
     ctx->block_size = block_size;
@@ -209,18 +210,21 @@ int SHA3_Final(uint8_t *md, KECCAK1600_CTX *ctx) {
     return 1;
   }
 
-   // Pad the data with 10*1. Note that |num| can be |block_size - 1|
-   // in which case both byte operations below are performed on
-   // the same byte.
-  memset(ctx->buf + num, 0, block_size - num);
-  ctx->buf[num] = ctx->pad;
-  ctx->buf[block_size - 1] |= 0x80;
+  if (ctx->padded == 0) {
+    // Pad the data with 10*1. Note that |num| can be |block_size - 1|
+    // in which case both byte operations below are performed on
+    // the same byte.
+    memset(ctx->buf + num, 0, block_size - num);
+    ctx->buf[num] = ctx->pad;
+    ctx->buf[block_size - 1] |= 0x80;
 
-  if (SHA3_Absorb(ctx->A, ctx->buf, block_size, block_size) != 0) {
-    return 0;
+    if (SHA3_Absorb(ctx->A, ctx->buf, block_size, block_size) != 0) {
+      return 0;
+    }
   }
 
-  SHA3_Squeeze(ctx->A, md, ctx->md_size, block_size);
+  SHA3_Squeeze(ctx->A, md, ctx->md_size, block_size, ctx->padded);
+  ctx->padded = 1;
 
   FIPS_service_indicator_update_state();
 

crypto/fipsmodule/sha/sha3_test.cc

@@ -134,8 +134,38 @@ TEST(SHA3Test, NISTTestVectors) {
                   const EVP_MD *algorithm = EVP_sha3_512();
                   test_vec.NISTTestVectors(algorithm);
                 });
+
+  FileTestGTest("crypto/fipsmodule/sha/testvectors/SHA3_224LongMsg.txt",
+                [](FileTest *t) {
+                  SHA3TestVector test_vec;
+                  EXPECT_TRUE(test_vec.ReadFromFileTest(t));
+                  const EVP_MD *algorithm = EVP_sha3_224();
+                  test_vec.NISTTestVectors(algorithm);
+                });
+  FileTestGTest("crypto/fipsmodule/sha/testvectors/SHA3_256LongMsg.txt",
+                [](FileTest *t) {
+                  SHA3TestVector test_vec;
+                  EXPECT_TRUE(test_vec.ReadFromFileTest(t));
+                  const EVP_MD *algorithm = EVP_sha3_256();
+                  test_vec.NISTTestVectors(algorithm);
+                });
+  FileTestGTest("crypto/fipsmodule/sha/testvectors/SHA3_384LongMsg.txt",
+                [](FileTest *t) {
+                  SHA3TestVector test_vec;
+                  EXPECT_TRUE(test_vec.ReadFromFileTest(t));
+                  const EVP_MD *algorithm = EVP_sha3_384();
+                  test_vec.NISTTestVectors(algorithm);
+                });
+  FileTestGTest("crypto/fipsmodule/sha/testvectors/SHA3_512LongMsg.txt",
+                [](FileTest *t) {
+                  SHA3TestVector test_vec;
+                  EXPECT_TRUE(test_vec.ReadFromFileTest(t));
+                  const EVP_MD *algorithm = EVP_sha3_512();
+                  test_vec.NISTTestVectors(algorithm);
+                });
 }
 
+
 TEST(SHA3Test, NISTTestVectors_SingleShot) {
   FileTestGTest("crypto/fipsmodule/sha/testvectors/SHA3_224ShortMsg.txt",
                 [](FileTest *t) {
@@ -181,7 +211,7 @@ TEST(KeccakInternalTest, SqueezeOutputBufferOverflow) {
     EXPECT_TRUE(SHA3_Init(&ctx, SHA3_PAD_CHAR, SHA3_384_DIGEST_BITLENGTH));
     out.resize(out_len + canary.size());
     std::copy(canary.begin(), canary.end(), out.end() - canary.size());
-    SHA3_Squeeze(ctx.A, out.data(), out_len, ctx.block_size);
+    SHA3_Squeeze(ctx.A, out.data(), out_len, ctx.block_size, 1);
     EXPECT_TRUE(std::equal(out.end() - canary.size(), out.end(),
                            canary.begin()) == true);
   }