v3.10.1

2025-09-17 23:44:27 +00:00 · 2019-12-05 19:09:23 -05:00
parent 40039386a0
commit 73430b13b1
52 changed files with 4515 additions and 874 deletions
--- a/simd-utils/intrlv.h
+++ b/simd-utils/intrlv.h
@@ -1991,7 +1991,7 @@ static inline void rintrlv_4x64_4x32( void *dst, const void *src,
   d[ 0] = s[ 0];  d[ 1] = s[ 2];  d[ 2] = s[ 4];  d[ 3] = s[ 6]; \
   d[ 4] = s[ 8];  d[ 5] = s[10];  d[ 6] = s[12];  d[ 7] = s[14]; \
   d[ 8] = s[ 1];  d[ 9] = s[ 3];  d[10] = s[ 5];  d[11] = s[ 7]; \
-   d[12] = s[ 9];  d[13] = s[11];  d[14] = s[13];  d[16] = s[15]; \
+   d[12] = s[ 9];  d[13] = s[11];  d[14] = s[13];  d[15] = s[15]; \
 } while(0)


@@ -2002,47 +2002,18 @@ static inline void rintrlv_8x64_8x32( void *dst, const void *src,
 {
   RLEAVE_8x64_8x32(   0 );   RLEAVE_8x64_8x32(  16 );
   RLEAVE_8x64_8x32(  32 );   RLEAVE_8x64_8x32(  48 );
+
+   if ( bit_len <= 256 ) return;
+
   RLEAVE_8x64_8x32(  64 );   RLEAVE_8x64_8x32(  80 );
   RLEAVE_8x64_8x32(  96 );   RLEAVE_8x64_8x32( 112 );

+   if ( bit_len <= 512 ) return;
+   
   RLEAVE_8x64_8x32( 128 );   RLEAVE_8x64_8x32( 144 );
   RLEAVE_8x64_8x32( 160 );   RLEAVE_8x64_8x32( 176 );
   RLEAVE_8x64_8x32( 192 );   RLEAVE_8x64_8x32( 208 );
   RLEAVE_8x64_8x32( 224 );   RLEAVE_8x64_8x32( 240 );
-   
-   if ( bit_len <= 256 ) return;
-
-   RLEAVE_8x64_8x32( 256 );   RLEAVE_8x64_8x32( 272 );
-   RLEAVE_8x64_8x32( 288 );   RLEAVE_8x64_8x32( 304 );
-   RLEAVE_8x64_8x32( 320 );   RLEAVE_8x64_8x32( 336 );
-   RLEAVE_8x64_8x32( 352 );   RLEAVE_8x64_8x32( 368 );
-
-   RLEAVE_8x64_8x32( 384 );   RLEAVE_8x64_8x32( 400 );
-   RLEAVE_8x64_8x32( 416 );   RLEAVE_8x64_8x32( 432 );
-   RLEAVE_8x64_8x32( 448 );   RLEAVE_8x64_8x32( 464 );
-   RLEAVE_8x64_8x32( 480 );   RLEAVE_8x64_8x32( 496 );
-
-   if ( bit_len <= 512 ) return;
-
-   RLEAVE_8x64_8x32( 512 );   RLEAVE_8x64_8x32( 528 );
-   RLEAVE_8x64_8x32( 544 );   RLEAVE_8x64_8x32( 560 );
-   RLEAVE_8x64_8x32( 576 );   RLEAVE_8x64_8x32( 592 );
-   RLEAVE_8x64_8x32( 608 );   RLEAVE_8x64_8x32( 624 );
-
-   RLEAVE_8x64_8x32( 640 );   RLEAVE_8x64_8x32( 656 );
-   RLEAVE_8x64_8x32( 672 );   RLEAVE_8x64_8x32( 688 );
-   RLEAVE_8x64_8x32( 704 );   RLEAVE_8x64_8x32( 720 );
-   RLEAVE_8x64_8x32( 736 );   RLEAVE_8x64_8x32( 752 );
-
-   RLEAVE_8x64_8x32( 768 );   RLEAVE_8x64_8x32( 784 );
-   RLEAVE_8x64_8x32( 800 );   RLEAVE_8x64_8x32( 816 );
-   RLEAVE_8x64_8x32( 832 );   RLEAVE_8x64_8x32( 848 );
-   RLEAVE_8x64_8x32( 864 );   RLEAVE_8x64_8x32( 880 );
-
-   RLEAVE_8x64_8x32( 896 );   RLEAVE_8x64_8x32( 912 );
-   RLEAVE_8x64_8x32( 928 );   RLEAVE_8x64_8x32( 944 );
-   RLEAVE_8x64_8x32( 960 );   RLEAVE_8x64_8x32( 976 );
-   RLEAVE_8x64_8x32( 992 );   RLEAVE_8x64_8x32(1008 );
 }

 #undef RLEAVE_8x64_8x32
@@ -2308,5 +2279,17 @@ do { \
 #define mm512_intrlv_blend_32( hi, lo ) \
   _mm512_mask_blend_epi32( 0x5555, hi, lo )

+#define mm512_blend_hash_8x64( dst, a, b, mask ) \
+do { \
+    dst[0] = _mm512_mask_blend_epi64( mask, a[0], b[0] ); \
+    dst[1] = _mm512_mask_blend_epi64( mask, a[1], b[1] ); \
+    dst[2] = _mm512_mask_blend_epi64( mask, a[2], b[2] ); \
+    dst[3] = _mm512_mask_blend_epi64( mask, a[3], b[3] ); \
+    dst[4] = _mm512_mask_blend_epi64( mask, a[4], b[4] ); \
+    dst[5] = _mm512_mask_blend_epi64( mask, a[5], b[5] ); \
+    dst[6] = _mm512_mask_blend_epi64( mask, a[6], b[6] ); \
+    dst[7] = _mm512_mask_blend_epi64( mask, a[7], b[7] ); \
+} while(0)
+
 #endif // AVX512
 #endif // INTERLEAVE_H__
--- a/simd-utils/simd-128.h
+++ b/simd-utils/simd-128.h
@@ -162,9 +162,10 @@ static inline __m128i mm128_neg1_fn()

 #define mm128_allbits0( a )    _mm_testz_si128(   a, a )
 #define mm128_allbits1( a )    _mm_testc_si128(   a, m128_neg1 )
-#define mm128_allbitsne( a )   _mm_testnzc_si128( a, m128_neg1 )
-#define mm128_anybits0         mm128_allbitsne
-#define mm128_anybits1         mm128_allbitsne
+// probably broken, avx2 is
+//#define mm128_allbitsne( a )   _mm_testnzc_si128( a, m128_neg1 )
+#define mm128_anybits0( a )    mm128_allbits1( a )
+#define mm128_anybits1( a )    mm128_allbits0( a )

 #else   // SSE2

--- a/simd-utils/simd-256.h
+++ b/simd-utils/simd-256.h
@@ -123,9 +123,10 @@ do { \

 #define mm256_allbits0( a )    _mm256_testz_si256(   a, a )
 #define mm256_allbits1( a )    _mm256_testc_si256(   a, m256_neg1 )
-#define mm256_allbitsne( a )   _mm256_testnzc_si256( a, m256_neg1 )
-#define mm256_anybits0         mm256_allbitsne
-#define mm256_anybits1         mm256_allbitsne
+//broken
+//#define mm256_allbitsne( a )   _mm256_testnzc_si256( a, m256_neg1 )
+#define mm256_anybits0( a )    !mm256_allbits1( a )
+#define mm256_anybits1( a )   !mm256_allbits0( a )


 // Parallel AES, for when x is expected to be in a 256 bit register.
--- a/simd-utils/simd-512.h
+++ b/simd-utils/simd-512.h
@@ -110,10 +110,12 @@ static inline __m512i m512_const4_64( const uint64_t i3, const uint64_t i2,
                          m128_const_64( i1, i0 ) ), 0x44 )

 // Equivalent of set1, broadcast 64 bit constant to all 64 bit elements.
+#define m512_const1_256( i )   _mm512_broadcast_i64x4( i )
+#define m512_const1_128( i )   _mm512_broadcast_i64x2( i )
 #define m512_const1_64( i ) _mm512_broadcastq_epi64( mm128_mov64_128( i ) )
 #define m512_const1_32( i ) _mm512_broadcastd_epi32( mm128_mov32_128( i ) )
 #define m512_const1_16( i ) _mm512_broadcastw_epi16( mm128_mov32_128( i ) )
-#define m512_const1_8 ( i ) _mm512_broadcastb_epi8 ( mm128_mov32_128( i ) )
+#define m512_const1_8( i ) _mm512_broadcastb_epi8 ( mm128_mov32_128( i ) )


 //
@@ -277,8 +279,8 @@ static inline void memcpy_512( __m512i *dst, const __m512i *src, const int n )
   _mm512_shuffle_epi8( v, \
               m512_const_64( 0x3c3d3e3f38393a3b, 0x3435363730313233, \
                              0x2c2d2e2f28292a2b, 0x2425262720212223, \
-                              0x0c0d0e0f08090a0b, 0x0405060700010203, \
-                              0x1c1d1e1f18191a1b, 0x1415161710111213 ) )
+                              0x1c1d1e1f18191a1b, 0x1415161710111213, \
+                              0x0c0d0e0f08090a0b, 0x0405060700010203 ) )

 #define mm512_bswap_16( v ) \
   _mm512_shuffle_epi8( v, \
@@ -415,38 +417,49 @@ static inline void memcpy_512( __m512i *dst, const __m512i *src, const int n )
 #define mm512_rol1x64_256( v )   _mm512_permutex_epi64( v, 0x93 )


-/*  Need to fix 
 // Rotate 256 bit lanes by one 32 bit element
 #define mm512_ror1x32_256( v ) \
-   _mm512_permutexvar_epi32( m512_const4_64( \
+   _mm512_permutexvar_epi32( m512_const_64( \
+                      0x000000080000000f, 0x0000000e0000000d, \
+                      0x0000000c0000000b, 0x0000000a00000009, \
                      0x0000000000000007, 0x0000000600000005, \
                      0x0000000400000003, 0x0000000200000001 ), v )

 #define mm512_rol1x32_256( v ) \
-   _mm512_permutexvar_epi32( m512_const4_64( \
+   _mm512_permutexvar_epi32( m512_const_64( \
+                      0x0000000e0000000d, 0x0000000c0000000b, \
+                      0x0000000a00000009, 0x000000080000000f, \
                      0x0000000600000005, 0x0000000400000003, \
                      0x0000000200000001, 0x0000000000000007 ), v )

 #define mm512_ror1x16_256( v ) \
-   _mm512_permutexvar_epi16( m512_const4_64( \
+   _mm512_permutexvar_epi16( m512_const_64( \
+                     0x00100001001e001d, 0x001c001b001a0019, \
+                     0x0018001700160015, 0x0014001300120011, \
                     0x0000000f000e000d, 0x000c000b000a0009, \
                     0x0008000700060005, 0x0004000300020001 ), v )

 #define mm512_rol1x16_256( v ) \
-   _mm512_permutexvar_epi16( m512_const4_64( \
+   _mm512_permutexvar_epi16( m512_const_64( \
+                     0x001e001d001c001b, 0x001a001900180017, \
+                     0x0016001500140013, 0x001200110010001f, \
                     0x000e000d000c000b, 0x000a000900080007, \
                     0x0006000500040003, 0x000200010000000f ), v )

 #define mm512_ror1x8_256( v ) \
-   _mm512_shuffle_epi8( v, m512_const4_64( \
+   _mm512_shuffle_epi8( v, m512_const_64( \
+                     0x203f3e3d3c3b3a39, 0x3837363534333231, \
+                     0x302f2e2d2c2b2a29, 0x2827262524232221, \
                     0x001f1e1d1c1b1a19, 0x1817161514131211, \
                     0x100f0e0d0c0b0a09, 0x0807060504030201 ), v )

 #define mm512_rol1x8_256( v ) \
-    _mm512_shuffle_epi8( v, m512_const4_64( \
+   _mm512_shuffle_epi8( v, m512_const_64( \
+                     0x3e3d3c3b3a393837, 0x363534333231302f, \
+                     0x2e2d2c2b2a292827, 0x262524232221203f, \
                     0x1e1d1c1b1a191817, 0x161514131211100f, \
                     0x0e0d0c0b0a090807, 0x060504030201001f ), v )
-*/
+
 //
 // Rotate elements within 128 bit lanes of 512 bit vector.

@@ -457,23 +470,33 @@ static inline void memcpy_512( __m512i *dst, const __m512i *src, const int n )
 #define mm512_ror1x32_128( v )   _mm512_shuffle_epi32( v, 0x39 )
 #define mm512_rol1x32_128( v )   _mm512_shuffle_epi32( v, 0x93 )

-/*
 #define mm512_ror1x16_128( v ) \
-    _mm512_permutexvar_epi16( m512_const2_64( \
+    _mm512_permutexvar_epi16( m512_const_64( \
+                     0x0018001f001e001d, 0x001c001b001a0019, \
+                     0x0010001700160015, 0x0014001300120011, \
+                     0x0008000f000e000d, 0x000c000b000a0009, \
                     0x0000000700060005, 0x0004000300020001 ), v ) 

 #define mm512_rol1x16_128( v ) \
-    _mm512_permutexvar_epi16( m512_const2_64( \
+    _mm512_permutexvar_epi16( m512_const_64( \
+                     0x001e001d001c001b, 0x001a00190018001f, \
+                     0x0016001500140013, 0x0012001100100017, \
+                     0x000e000d000c000b, 0x000a00090008000f, \
                     0x0006000500040003, 0x0002000100000007 ), v ) 

 #define mm512_ror1x8_128( v ) \
-    _mm512_shuffle_epi8( v, m512_const2_64( \
+    _mm512_shuffle_epi8( v, m512_const_64( \
+                     0x303f3e3d3c3b3a39, 0x3837363534333231, \
+                     0x202f2e2d2c2b2a29, 0x2827262524232221, \
+                     0x101f1e1d1c1b1a19, 0x1817161514131211, \
                     0x000f0e0d0c0b0a09, 0x0807060504030201 ) )

 #define mm512_rol1x8_128( v ) \
-    _mm512_shuffle_epi8( v, m512_const2_64( \
+    _mm512_shuffle_epi8( v, m512_const_64( \
+                     0x3e3d3c3b3a393837, 0x363534333231303f, \
+                     0x2e2d2c2b2a292827, 0x262524232221202f, \
+                     0x1e1d1c1b1a191817, 0x161514131211101f, \
                     0x0e0d0c0b0a090807, 0x060504030201000f ) )
-*/

 // Rotate 128 bit lanes by c bytes.  
 #define mm512_bror_128( v, c ) \