crypto: x86/sha256-ni - convert to use rounds macros

To avoid source code duplication, do the SHA-256 rounds using macros.
This reduces the length of sha256_ni_asm.S by 153 lines while still
producing the exact same object file.

Signed-off-by: Eric Biggers <ebiggers@google.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

diff --git a/arch/x86/crypto/sha256_ni_asm.S b/arch/x86/crypto/sha256_ni_asm.S
index 537b6dcd7ed80ef1f285019e87f27cdba0634e58..498f67727b94c91ee81c349c84ce0e3a00642a46 100644 (file)
--- a/arch/x86/crypto/sha256_ni_asm.S
+++ b/arch/x86/crypto/sha256_ni_asm.S
@@ -76,6 +76,29 @@
 #define ABEF_SAVE      %xmm9
 #define CDGH_SAVE      %xmm10
 
+.macro do_4rounds      i, m0, m1, m2, m3
+.if \i < 16
+       movdqu          \i*4(DATA_PTR), MSG
+       pshufb          SHUF_MASK, MSG
+       movdqa          MSG, \m0
+.else
+       movdqa          \m0, MSG
+.endif
+       paddd           \i*4(SHA256CONSTANTS), MSG
+       sha256rnds2     STATE0, STATE1
+.if \i >= 12 && \i < 60
+       movdqa          \m0, MSGTMP4
+       palignr         $4, \m3, MSGTMP4
+       paddd           MSGTMP4, \m1
+       sha256msg2      \m0, \m1
+.endif
+       pshufd          $0x0E, MSG, MSG
+       sha256rnds2     STATE1, STATE0
+.if \i >= 4 && \i < 52
+       sha256msg1      \m0, \m3
+.endif
+.endm
+
 /*
  * Intel SHA Extensions optimized implementation of a SHA-256 update function
  *
@@ -86,9 +109,6 @@
  * store partial blocks.  All message padding and hash value initialization must
  * be done outside the update function.
  *
- * The indented lines in the loop are instructions related to rounds processing.
- * The non-indented lines are instructions related to the message schedule.
- *
  * void sha256_ni_transform(uint32_t *digest, const void *data,
                uint32_t numBlocks);
  * digest : pointer to digest
@@ -125,185 +145,12 @@ SYM_TYPED_FUNC_START(sha256_ni_transform)
        movdqa          STATE0, ABEF_SAVE
        movdqa          STATE1, CDGH_SAVE
 
-       /* Rounds 0-3 */
-       movdqu          0*16(DATA_PTR), MSG
-       pshufb          SHUF_MASK, MSG
-       movdqa          MSG, MSGTMP0
-               paddd           0*16(SHA256CONSTANTS), MSG
-               sha256rnds2     STATE0, STATE1
-               pshufd          $0x0E, MSG, MSG
-               sha256rnds2     STATE1, STATE0
-
-       /* Rounds 4-7 */
-       movdqu          1*16(DATA_PTR), MSG
-       pshufb          SHUF_MASK, MSG
-       movdqa          MSG, MSGTMP1
-               paddd           1*16(SHA256CONSTANTS), MSG
-               sha256rnds2     STATE0, STATE1
-               pshufd          $0x0E, MSG, MSG
-               sha256rnds2     STATE1, STATE0
-       sha256msg1      MSGTMP1, MSGTMP0
-
-       /* Rounds 8-11 */
-       movdqu          2*16(DATA_PTR), MSG
-       pshufb          SHUF_MASK, MSG
-       movdqa          MSG, MSGTMP2
-               paddd           2*16(SHA256CONSTANTS), MSG
-               sha256rnds2     STATE0, STATE1
-               pshufd          $0x0E, MSG, MSG
-               sha256rnds2     STATE1, STATE0
-       sha256msg1      MSGTMP2, MSGTMP1
-
-       /* Rounds 12-15 */
-       movdqu          3*16(DATA_PTR), MSG
-       pshufb          SHUF_MASK, MSG
-       movdqa          MSG, MSGTMP3
-               paddd           3*16(SHA256CONSTANTS), MSG
-               sha256rnds2     STATE0, STATE1
-       movdqa          MSGTMP3, MSGTMP4
-       palignr         $4, MSGTMP2, MSGTMP4
-       paddd           MSGTMP4, MSGTMP0
-       sha256msg2      MSGTMP3, MSGTMP0
-               pshufd          $0x0E, MSG, MSG
-               sha256rnds2     STATE1, STATE0
-       sha256msg1      MSGTMP3, MSGTMP2
-
-       /* Rounds 16-19 */
-       movdqa          MSGTMP0, MSG
-               paddd           4*16(SHA256CONSTANTS), MSG
-               sha256rnds2     STATE0, STATE1
-       movdqa          MSGTMP0, MSGTMP4
-       palignr         $4, MSGTMP3, MSGTMP4
-       paddd           MSGTMP4, MSGTMP1
-       sha256msg2      MSGTMP0, MSGTMP1
-               pshufd          $0x0E, MSG, MSG
-               sha256rnds2     STATE1, STATE0
-       sha256msg1      MSGTMP0, MSGTMP3
-
-       /* Rounds 20-23 */
-       movdqa          MSGTMP1, MSG
-               paddd           5*16(SHA256CONSTANTS), MSG
-               sha256rnds2     STATE0, STATE1
-       movdqa          MSGTMP1, MSGTMP4
-       palignr         $4, MSGTMP0, MSGTMP4
-       paddd           MSGTMP4, MSGTMP2
-       sha256msg2      MSGTMP1, MSGTMP2
-               pshufd          $0x0E, MSG, MSG
-               sha256rnds2     STATE1, STATE0
-       sha256msg1      MSGTMP1, MSGTMP0
-
-       /* Rounds 24-27 */
-       movdqa          MSGTMP2, MSG
-               paddd           6*16(SHA256CONSTANTS), MSG
-               sha256rnds2     STATE0, STATE1
-       movdqa          MSGTMP2, MSGTMP4
-       palignr         $4, MSGTMP1, MSGTMP4
-       paddd           MSGTMP4, MSGTMP3
-       sha256msg2      MSGTMP2, MSGTMP3
-               pshufd          $0x0E, MSG, MSG
-               sha256rnds2     STATE1, STATE0
-       sha256msg1      MSGTMP2, MSGTMP1
-
-       /* Rounds 28-31 */
-       movdqa          MSGTMP3, MSG
-               paddd           7*16(SHA256CONSTANTS), MSG
-               sha256rnds2     STATE0, STATE1
-       movdqa          MSGTMP3, MSGTMP4
-       palignr         $4, MSGTMP2, MSGTMP4
-       paddd           MSGTMP4, MSGTMP0
-       sha256msg2      MSGTMP3, MSGTMP0
-               pshufd          $0x0E, MSG, MSG
-               sha256rnds2     STATE1, STATE0
-       sha256msg1      MSGTMP3, MSGTMP2
-
-       /* Rounds 32-35 */
-       movdqa          MSGTMP0, MSG
-               paddd           8*16(SHA256CONSTANTS), MSG
-               sha256rnds2     STATE0, STATE1
-       movdqa          MSGTMP0, MSGTMP4
-       palignr         $4, MSGTMP3, MSGTMP4
-       paddd           MSGTMP4, MSGTMP1
-       sha256msg2      MSGTMP0, MSGTMP1
-               pshufd          $0x0E, MSG, MSG
-               sha256rnds2     STATE1, STATE0
-       sha256msg1      MSGTMP0, MSGTMP3
-
-       /* Rounds 36-39 */
-       movdqa          MSGTMP1, MSG
-               paddd           9*16(SHA256CONSTANTS), MSG
-               sha256rnds2     STATE0, STATE1
-       movdqa          MSGTMP1, MSGTMP4
-       palignr         $4, MSGTMP0, MSGTMP4
-       paddd           MSGTMP4, MSGTMP2
-       sha256msg2      MSGTMP1, MSGTMP2
-               pshufd          $0x0E, MSG, MSG
-               sha256rnds2     STATE1, STATE0
-       sha256msg1      MSGTMP1, MSGTMP0
-
-       /* Rounds 40-43 */
-       movdqa          MSGTMP2, MSG
-               paddd           10*16(SHA256CONSTANTS), MSG
-               sha256rnds2     STATE0, STATE1
-       movdqa          MSGTMP2, MSGTMP4
-       palignr         $4, MSGTMP1, MSGTMP4
-       paddd           MSGTMP4, MSGTMP3
-       sha256msg2      MSGTMP2, MSGTMP3
-               pshufd          $0x0E, MSG, MSG
-               sha256rnds2     STATE1, STATE0
-       sha256msg1      MSGTMP2, MSGTMP1
-
-       /* Rounds 44-47 */
-       movdqa          MSGTMP3, MSG
-               paddd           11*16(SHA256CONSTANTS), MSG
-               sha256rnds2     STATE0, STATE1
-       movdqa          MSGTMP3, MSGTMP4
-       palignr         $4, MSGTMP2, MSGTMP4
-       paddd           MSGTMP4, MSGTMP0
-       sha256msg2      MSGTMP3, MSGTMP0
-               pshufd          $0x0E, MSG, MSG
-               sha256rnds2     STATE1, STATE0
-       sha256msg1      MSGTMP3, MSGTMP2
-
-       /* Rounds 48-51 */
-       movdqa          MSGTMP0, MSG
-               paddd           12*16(SHA256CONSTANTS), MSG
-               sha256rnds2     STATE0, STATE1
-       movdqa          MSGTMP0, MSGTMP4
-       palignr         $4, MSGTMP3, MSGTMP4
-       paddd           MSGTMP4, MSGTMP1
-       sha256msg2      MSGTMP0, MSGTMP1
-               pshufd          $0x0E, MSG, MSG
-               sha256rnds2     STATE1, STATE0
-       sha256msg1      MSGTMP0, MSGTMP3
-
-       /* Rounds 52-55 */
-       movdqa          MSGTMP1, MSG
-               paddd           13*16(SHA256CONSTANTS), MSG
-               sha256rnds2     STATE0, STATE1
-       movdqa          MSGTMP1, MSGTMP4
-       palignr         $4, MSGTMP0, MSGTMP4
-       paddd           MSGTMP4, MSGTMP2
-       sha256msg2      MSGTMP1, MSGTMP2
-               pshufd          $0x0E, MSG, MSG
-               sha256rnds2     STATE1, STATE0
-
-       /* Rounds 56-59 */
-       movdqa          MSGTMP2, MSG
-               paddd           14*16(SHA256CONSTANTS), MSG
-               sha256rnds2     STATE0, STATE1
-       movdqa          MSGTMP2, MSGTMP4
-       palignr         $4, MSGTMP1, MSGTMP4
-       paddd           MSGTMP4, MSGTMP3
-       sha256msg2      MSGTMP2, MSGTMP3
-               pshufd          $0x0E, MSG, MSG
-               sha256rnds2     STATE1, STATE0
-
-       /* Rounds 60-63 */
-       movdqa          MSGTMP3, MSG
-               paddd           15*16(SHA256CONSTANTS), MSG
-               sha256rnds2     STATE0, STATE1
-               pshufd          $0x0E, MSG, MSG
-               sha256rnds2     STATE1, STATE0
+.irp i, 0, 16, 32, 48
+       do_4rounds      (\i + 0),  MSGTMP0, MSGTMP1, MSGTMP2, MSGTMP3
+       do_4rounds      (\i + 4),  MSGTMP1, MSGTMP2, MSGTMP3, MSGTMP0
+       do_4rounds      (\i + 8),  MSGTMP2, MSGTMP3, MSGTMP0, MSGTMP1
+       do_4rounds      (\i + 12), MSGTMP3, MSGTMP0, MSGTMP1, MSGTMP2
+.endr
 
        /* Add current hash values with previously saved */
        paddd           ABEF_SAVE, STATE0