v3.9.9.1

simd-utils/simd-128.h

@@ -298,30 +298,38 @@ static inline void memcpy_128( __m128i *dst, const __m128i *src, const int n )
 // 64 and 32 bit elements.
 
 // compiler doesn't like when a variable is used for the last arg of
-// _mm_rol_epi32, must be "8 bit immediate".
+// _mm_rol_epi32, must be "8 bit immediate". Therefore use rol_var where
+// necessary.
 // sm3-hash-4way.c fails to compile.
+
+#define mm128_ror_var_64( v, c ) \
+   _mm_or_si128( _mm_srli_epi64( v, c ), _mm_slli_epi64( v, 64-(c) ) )
+
+#define mm128_rol_var_64( v, c ) \
+   _mm_or_si128( _mm_slli_epi64( v, c ), _mm_srli_epi64( v, 64-(c) ) )
+
+#define mm128_ror_var_32( v, c ) \
+   _mm_or_si128( _mm_srli_epi32( v, c ), _mm_slli_epi32( v, 32-(c) ) )
+
+#define mm128_rol_var_32( v, c ) \
+   _mm_or_si128( _mm_slli_epi32( v, c ), _mm_srli_epi32( v, 32-(c) ) )
+
+
 /*
 #if defined(__AVX512F__) && defined(__AVX512VL__) && defined(__AVX512DQ__) && defined(__AVX512BW__)
 
-#define mm128_ror_64( v, c )    _mm_ror_epi64( v, c )
-#define mm128_rol_64( v, c )    _mm_rol_epi64( v, c )
-#define mm128_ror_32( v, c )    _mm_ror_epi32( v, c )
-#define mm128_rol_32( v, c )    _mm_rol_epi32( v, c )
+#define mm128_ror_64    _mm_ror_epi64
+#define mm128_rol_64    _mm_rol_epi64
+#define mm128_ror_32    _mm_ror_epi32
+#define mm128_rol_32    _mm_rol_epi32
 
 #else
 */
 
-#define mm128_ror_64( v, c ) \
-   _mm_or_si128( _mm_srli_epi64( v, c ), _mm_slli_epi64( v, 64-(c) ) )
-
-#define mm128_rol_64( v, c ) \
-   _mm_or_si128( _mm_slli_epi64( v, c ), _mm_srli_epi64( v, 64-(c) ) )
-
-#define mm128_ror_32( v, c ) \
-   _mm_or_si128( _mm_srli_epi32( v, c ), _mm_slli_epi32( v, 32-(c) ) )
-
-#define mm128_rol_32( v, c ) \
-   _mm_or_si128( _mm_slli_epi32( v, c ), _mm_srli_epi32( v, 32-(c) ) )
+#define mm128_ror_64   mm128_ror_var_64
+#define mm128_rol_64   mm128_rol_var_64
+#define mm128_ror_32   mm128_ror_var_32
+#define mm128_rol_32   mm128_rol_var_32
 
 //#endif   // AVX512 else
 

simd-utils/simd-256.h

@@ -367,38 +367,49 @@ static inline void memcpy_256( __m256i *dst, const __m256i *src, const int n )
 //
 // AVX512 has bit rotate for 256 bit vectors with 64 or 32 bit elements
 
+
 // compiler doesn't like when a variable is used for the last arg of
-// _mm_rol_epi32, must be "8 bit immediate".
+// _mm_rol_epi32, must be "8 bit immediate". Therefore use rol_var where
+// necessary. 
+
+#define mm256_ror_var_64( v, c ) \
+   _mm256_or_si256( _mm256_srli_epi64( v, c ), \
+                    _mm256_slli_epi64( v, 64-(c) ) )
+
+#define mm256_rol_var_64( v, c ) \
+   _mm256_or_si256( _mm256_slli_epi64( v, c ), \
+                    _mm256_srli_epi64( v, 64-(c) ) )
+
+#define mm256_ror_var_32( v, c ) \
+   _mm256_or_si256( _mm256_srli_epi32( v, c ), \
+                    _mm256_slli_epi32( v, 32-(c) ) )
+
+#define mm256_rol_var_32( v, c ) \
+   _mm256_or_si256( _mm256_slli_epi32( v, c ), \
+                    _mm256_srli_epi32( v, 32-(c) ) )
+
 /*
 #if defined(__AVX512F__) && defined(__AVX512VL__) && defined(__AVX512DQ__) && defined(__AVX512BW__)
 
-#define mm256_ror_64( v, c )    _mm256_ror_epi64( v, c )
-#define mm256_rol_64( v, c )    _mm256_rol_epi64( v, c )
-#define mm256_ror_32( v, c )    _mm256_ror_epi32( v, c )
-#define mm256_rol_32( v, c )    _mm256_rol_epi32( v, c )
+// AVX512, control must be 8 bit immediate.
+
+#define mm256_ror_64    _mm256_ror_epi64
+#define mm256_rol_64    _mm256_rol_epi64
+#define mm256_ror_32    _mm256_ror_epi32
+#define mm256_rol_32    _mm256_rol_epi32
 
 #else
 */
 
-#define mm256_ror_64( v, c ) \
-   _mm256_or_si256( _mm256_srli_epi64( v, c ), \
-                    _mm256_slli_epi64( v, 64-(c) ) )
+// No AVX512, use fallback.
 
-#define mm256_rol_64( v, c ) \
-   _mm256_or_si256( _mm256_slli_epi64( v, c ), \
-                    _mm256_srli_epi64( v, 64-(c) ) )
-
-#define mm256_ror_32( v, c ) \
-   _mm256_or_si256( _mm256_srli_epi32( v, c ), \
-                    _mm256_slli_epi32( v, 32-(c) ) )
-
-#define mm256_rol_32( v, c ) \
-   _mm256_or_si256( _mm256_slli_epi32( v, c ), \
-                    _mm256_srli_epi32( v, 32-(c) ) )
+#define mm256_ror_64    mm256_ror_var_64 
+#define mm256_rol_64    mm256_rol_var_64
+#define mm256_ror_32    mm256_ror_var_32
+#define mm256_rol_32    mm256_rol_var_32
 
 // #endif     // AVX512 else
 
-
 #define  mm256_ror_16( v, c ) \
    _mm256_or_si256( _mm256_srli_epi16( v, c ), \
                     _mm256_slli_epi16( v, 16-(c) ) )

simd-utils/simd-512.h

@@ -278,7 +278,7 @@ static inline __m512i mm512_neg1_fn()
 // Horizontal vector testing
 
 #define mm512_allbits0( a )    _mm512_cmpeq_epi64_mask( a, m512_zero )
-#define mm256_allbits1( a )    _mm512_cmpeq_epi64_mask( a, m512_neg1 )
+#define mm512_allbits1( a )    _mm512_cmpeq_epi64_mask( a, m512_neg1 )
 #define mm512_anybits0( a )    _mm512_cmpneq_epi64_mask( a, m512_neg1 )
 #define mm512_anybits1( a )    _mm512_cmpneq_epi64_mask( a, m512_zero )
 
@@ -287,11 +287,30 @@ static inline __m512i mm512_neg1_fn()
 // Bit rotations.
 
 // AVX512F has built-in fixed and variable bit rotation for 64 & 32 bit
-// elements and can be called directly.
+// elements and can be called directly. But they only accept immediate 8
+// for control arg. 
 //
 // _mm512_rol_epi64,  _mm512_ror_epi64,  _mm512_rol_epi32,  _mm512_ror_epi32
 // _mm512_rolv_epi64, _mm512_rorv_epi64, _mm512_rolv_epi32, _mm512_rorv_epi32
 //
+
+#define mm512_ror_var_64( v, c ) \
+   _mm512_or_si512( _mm512_srli_epi64( v, c ), \
+                    _mm512_slli_epi64( v, 64-(c) ) )
+
+#define mm512_rol_var_64( v, c ) \
+   _mm512_or_si512( _mm512_slli_epi64( v, c ), \
+                    _mm512_srli_epi64( v, 64-(c) ) )
+
+#define mm512_ror_var_32( v, c ) \
+   _mm512_or_si512( _mm512_srli_epi32( v, c ), \
+                    _mm512_slli_epi32( v, 32-(c) ) )
+
+#define mm512_rol_var_32( v, c ) \
+   _mm512_or_si512( _mm512_slli_epi32( v, c ), \
+                    _mm512_srli_epi32( v, 32-(c) ) )
+
+
 // Here is a fixed bit rotate for 16 bit elements:
 #define mm512_ror_16( v, c ) \
     _mm512_or_si512( _mm512_srli_epi16( v, c ), \
@@ -300,6 +319,8 @@ static inline __m512i mm512_neg1_fn()
     _mm512_or_si512( _mm512_slli_epi16( v, c ), \
                      _mm512_srli_epi16( v, 16-(c) )
 
+
+
 // Rotations using a vector control index are very slow due to overhead
 // to generate the index vector. Repeated rotations using the same index
 // are better handled by the calling function where the index only needs