mirror of
https://github.com/JayDDee/cpuminer-opt.git
synced 2025-09-17 23:44:27 +00:00
v3.9.10
This commit is contained in:
Jay D Dee
1385
simd-utils/intrlv.h
1385
simd-utils/intrlv.h
File diff suppressed because it is too large
Load Diff
@@ -3,183 +3,132 @@
|
||||
|
||||
#if defined(__SSE2__)
|
||||
|
||||
//////////////////////////////////////////////////////////////////
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// 128 bit SSE vectors
|
||||
//
|
||||
// SSE2 is generally required for full 128 bit support. Some functions
|
||||
// are also optimized with SSSE3 or SSE4.1.
|
||||
//
|
||||
// Do not call intrinsic _mm_extract directly, it isn't supported in SSE2.
|
||||
// Use mm128_extr macro instead, it will select the appropriate implementation.
|
||||
//
|
||||
// 128 bit operations are enhanced with uint128 which adds 128 bit integer
|
||||
// support for arithmetic and other operations. Casting to uint128_t is not
|
||||
// free but is sometimes the only way for certain operations.
|
||||
// SSE2 is required for 128 bit integer support. Some functions are also
|
||||
// optimized with SSSE3, SSE4.1 or AVX. Some of these more optimized
|
||||
// functions don't have SSE2 equivalents and their use would break SSE2
|
||||
// compatibility.
|
||||
//
|
||||
// Constants are an issue with simd. Simply put, immediate constants don't
|
||||
// exist. All simd constants either reside in memory or a register and
|
||||
// must be loaded from memory or generated using instructions at run time.
|
||||
// must be loaded from memory or generated at run time.
|
||||
//
|
||||
// Due to the cost of generating constants it is often more efficient to
|
||||
// Due to the cost of generating constants it is more efficient to
|
||||
// define a local const for repeated references to the same constant.
|
||||
//
|
||||
// Some constant values can be generated using shortcuts. Zero for example
|
||||
// is as simple as XORing any register with itself, and is implemented
|
||||
// iby the setzero instrinsic. These shortcuts must be implemented using ASM
|
||||
// due to doing things the compiler would complain about. Another single
|
||||
// instruction constant is -1, defined below. Others may be added as the need
|
||||
// arises. Even single instruction constants are less efficient than local
|
||||
// register variables so the advice above stands. These pseudo-constants
|
||||
// do not perform any memory accesses
|
||||
// One common use for simd constants is as a control index for vector
|
||||
// instructions like blend and shuffle. Alhough the ultimate instruction
|
||||
// may execute in a single clock cycle, generating the control index adds
|
||||
// several more cycles to the entire operation.
|
||||
//
|
||||
// One common use for simd constants is as a control index for some simd
|
||||
// instructions like blend and shuffle. The utilities below do not take this
|
||||
// into account. Those that generate a simd constant should not be used
|
||||
// repeatedly. It may be better for the application to reimplement the
|
||||
// utility to better suit its usage.
|
||||
// All of the utilities here assume all data is in registers except
|
||||
// in rare cases where arguments are pointers.
|
||||
//
|
||||
// Intrinsics automatically promote from REX to VEX when AVX is available
|
||||
// but ASM needs to be done manually.
|
||||
//
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
// Efficient and convenient moving bwtween GP & low bits of XMM.
|
||||
// Use VEX when available to give access to xmm8-15 and zero extend for
|
||||
// larger vectors.
|
||||
|
||||
static inline __m128i mm128_mov64_128( const uint64_t n )
|
||||
{
|
||||
__m128i a;
|
||||
#if defined(__AVX__)
|
||||
asm( "vmovq %1, %0\n\t" : "=x"(a) : "r"(n) );
|
||||
#else
|
||||
asm( "movq %1, %0\n\t" : "=x"(a) : "r"(n) );
|
||||
#endif
|
||||
return a;
|
||||
}
|
||||
|
||||
static inline __m128i mm128_mov32_128( const uint32_t n )
|
||||
{
|
||||
__m128i a;
|
||||
#if defined(__AVX__)
|
||||
asm( "vmovd %1, %0\n\t" : "=x"(a) : "r"(n) );
|
||||
#else
|
||||
asm( "movd %1, %0\n\t" : "=x"(a) : "r"(n) );
|
||||
#endif
|
||||
return a;
|
||||
}
|
||||
|
||||
static inline uint64_t mm128_mov128_64( const __m128i a )
|
||||
{
|
||||
uint64_t n;
|
||||
#if defined(__AVX__)
|
||||
asm( "vmovq %1, %0\n\t" : "=r"(n) : "x"(a) );
|
||||
#else
|
||||
asm( "movq %1, %0\n\t" : "=r"(n) : "x"(a) );
|
||||
#endif
|
||||
return n;
|
||||
}
|
||||
|
||||
static inline uint32_t mm128_mov128_32( const __m128i a )
|
||||
{
|
||||
uint32_t n;
|
||||
#if defined(__AVX__)
|
||||
asm( "vmovd %1, %0\n\t" : "=r"(n) : "x"(a) );
|
||||
#else
|
||||
asm( "movd %1, %0\n\t" : "=r"(n) : "x"(a) );
|
||||
#endif
|
||||
return n;
|
||||
}
|
||||
|
||||
// Pseudo constants
|
||||
|
||||
#define m128_zero _mm_setzero_si128()
|
||||
#define m128_one_128 mm128_mov64_128( 1 )
|
||||
#define m128_one_64 _mm_shuffle_epi32( mm128_mov64_128( 1 ), 0x44 )
|
||||
#define m128_one_32 _mm_shuffle_epi32( mm128_mov32_128( 1 ), 0x00 )
|
||||
#define m128_one_16 _mm_shuffle_epi32( \
|
||||
mm128_mov32_128( 0x00010001 ), 0x00 )
|
||||
#define m128_one_8 _mm_shuffle_epi32( \
|
||||
mm128_mov32_128( 0x01010101 ), 0x00 )
|
||||
|
||||
static inline __m128i mm128_one_128_fn()
|
||||
{
|
||||
__m128i a;
|
||||
const uint64_t one = 1;
|
||||
asm( "movq %1, %0\n\t"
|
||||
: "=x"(a)
|
||||
: "r" (one) );
|
||||
return a;
|
||||
}
|
||||
#define m128_one_128 mm128_one_128_fn()
|
||||
|
||||
static inline __m128i mm128_one_64_fn()
|
||||
{
|
||||
__m128i a;
|
||||
const uint64_t one = 1;
|
||||
asm( "movq %1, %0\n\t"
|
||||
: "=x" (a)
|
||||
: "r" (one) );
|
||||
return _mm_shuffle_epi32( a, 0x44 );
|
||||
}
|
||||
#define m128_one_64 mm128_one_64_fn()
|
||||
|
||||
static inline __m128i mm128_one_32_fn()
|
||||
{
|
||||
__m128i a;
|
||||
const uint32_t one = 1;
|
||||
asm( "movd %1, %0\n\t"
|
||||
: "=x" (a)
|
||||
: "r" (one) );
|
||||
return _mm_shuffle_epi32( a, 0x00 );
|
||||
}
|
||||
#define m128_one_32 mm128_one_32_fn()
|
||||
|
||||
static inline __m128i mm128_one_16_fn()
|
||||
{
|
||||
__m128i a;
|
||||
const uint32_t one = 0x00010001;
|
||||
asm( "movd %1, %0\n\t"
|
||||
: "=x" (a)
|
||||
: "r" (one) );
|
||||
return _mm_shuffle_epi32( a, 0x00 );
|
||||
}
|
||||
#define m128_one_16 mm128_one_16_fn()
|
||||
|
||||
static inline __m128i mm128_one_8_fn()
|
||||
{
|
||||
__m128i a;
|
||||
const uint32_t one = 0x01010101;
|
||||
asm( "movd %1, %0\n\t"
|
||||
: "=x" (a)
|
||||
: "r" (one) );
|
||||
return _mm_shuffle_epi32( a, 0x00 );
|
||||
}
|
||||
#define m128_one_8 mm128_one_8_fn()
|
||||
// ASM avoids the need to initialize return variable to avoid compiler warning.
|
||||
// Macro abstracts function parentheses to look like an identifier.
|
||||
|
||||
static inline __m128i mm128_neg1_fn()
|
||||
{
|
||||
__m128i a;
|
||||
asm( "pcmpeqd %0, %0\n\t"
|
||||
: "=x" (a) );
|
||||
#if defined(__AVX__)
|
||||
asm( "vpcmpeqq %0, %0, %0\n\t" : "=x"(a) );
|
||||
#else
|
||||
asm( "pcmpeqq %0, %0\n\t" : "=x"(a) );
|
||||
#endif
|
||||
return a;
|
||||
}
|
||||
#define m128_neg1 mm128_neg1_fn()
|
||||
|
||||
// move uint64_t to low bits of __m128i, zeros the rest
|
||||
static inline __m128i mm128_mov64_128( uint64_t n )
|
||||
{
|
||||
__m128i a;
|
||||
asm( "movq %1, %0\n\t"
|
||||
: "=x" (a)
|
||||
: "r" (n) );
|
||||
return a;
|
||||
}
|
||||
|
||||
static inline __m128i mm128_mov32_128( uint32_t n )
|
||||
{
|
||||
__m128i a;
|
||||
asm( "movd %1, %0\n\t"
|
||||
: "=x" (a)
|
||||
: "r" (n) );
|
||||
return a;
|
||||
}
|
||||
// const functions work best when arguments are immediate constants or
|
||||
// are known to be in registers. If data needs to loaded from memory or cache
|
||||
// use set.
|
||||
|
||||
static inline uint64_t mm128_mov128_64( __m128i a )
|
||||
{
|
||||
uint64_t n;
|
||||
asm( "movq %1, %0\n\t"
|
||||
: "=x" (n)
|
||||
: "r" (a) );
|
||||
return n;
|
||||
}
|
||||
// Equivalent of set1, broadcast 64 bit integer to all elements.
|
||||
#define m128_const1_64( i ) _mm_shuffle_epi32( mm128_mov64_128( i ), 0x44 )
|
||||
#define m128_const1_32( i ) _mm_shuffle_epi32( mm128_mov32_128( i ), 0x00 )
|
||||
|
||||
static inline uint32_t mm128_mov128_32( __m128i a )
|
||||
{
|
||||
uint32_t n;
|
||||
asm( "movd %1, %0\n\t"
|
||||
: "=x" (n)
|
||||
: "r" (a) );
|
||||
return n;
|
||||
}
|
||||
#if defined(__SSE4_1__)
|
||||
|
||||
static inline __m128i m128_const1_64( const uint64_t n )
|
||||
{
|
||||
__m128i a;
|
||||
asm( "movq %1, %0\n\t"
|
||||
: "=x" (a)
|
||||
: "r" (n) );
|
||||
return _mm_shuffle_epi32( a, 0x44 );
|
||||
}
|
||||
// Assign 64 bit integers to respective elements: {hi, lo}
|
||||
#define m128_const_64( hi, lo ) \
|
||||
_mm_insert_epi64( mm128_mov64_128( lo ), hi, 1 )
|
||||
|
||||
static inline __m128i m128_const1_32( const uint32_t n )
|
||||
{
|
||||
__m128i a;
|
||||
asm( "movd %1, %0\n\t"
|
||||
: "=x" (a)
|
||||
: "r" (n) );
|
||||
return _mm_shuffle_epi32( a, 0x00 );
|
||||
}
|
||||
|
||||
#if defined(__SSE41__)
|
||||
|
||||
// alternative to _mm_set_epi64x, doesn't use mem,
|
||||
|
||||
static inline __m128i m128_const_64( const uint64_t hi, const uint64_t lo )
|
||||
{
|
||||
__m128i a;
|
||||
asm( "movq %2, %0\n\t"
|
||||
"pinsrq $1, %1, %0\n\t"
|
||||
: "=x" (a)
|
||||
: "r" (hi), "r" (lo) );
|
||||
return a;
|
||||
}
|
||||
|
||||
#else
|
||||
#else // No insert in SSE2
|
||||
|
||||
#define m128_const_64 _mm_set_epi64x
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
//
|
||||
// Basic operations without equivalent SIMD intrinsic
|
||||
|
||||
@@ -207,18 +156,9 @@ static inline __m128i m128_const_64( const uint64_t hi, const uint64_t lo )
|
||||
#define mm128_xor4( a, b, c, d ) \
|
||||
_mm_xor_si128( _mm_xor_si128( a, b ), _mm_xor_si128( c, d ) )
|
||||
|
||||
// This isn't cheap, not suitable for bulk usage.
|
||||
#define mm128_extr_4x32( a0, a1, a2, a3, src ) \
|
||||
do { \
|
||||
a0 = _mm_extract_epi32( src, 0 ); \
|
||||
a1 = _mm_extract_epi32( src, 1 ); \
|
||||
a1 = _mm_extract_epi32( src, 2 ); \
|
||||
a3 = _mm_extract_epi32( src, 3 ); \
|
||||
} while(0)
|
||||
|
||||
// Horizontal vector testing
|
||||
|
||||
#if defined(__SSE41__)
|
||||
#if defined(__SSE4_1__)
|
||||
|
||||
#define mm128_allbits0( a ) _mm_testz_si128( a, a )
|
||||
#define mm128_allbits1( a ) _mm_testc_si128( a, m128_neg1 )
|
||||
@@ -235,7 +175,7 @@ do { \
|
||||
#define mm128_allbits0( a ) ( !mm128_anybits1(a) )
|
||||
#define mm128_allbits1( a ) ( !mm128_anybits0(a) )
|
||||
|
||||
#endif // SSE41 else SSE2
|
||||
#endif // SSE4.1 else SSE2
|
||||
|
||||
//
|
||||
// Vector pointer cast
|
||||
@@ -256,20 +196,6 @@ do { \
|
||||
// returns pointer p+o
|
||||
#define casto_m128i(p,o) (((__m128i*)(p))+(o))
|
||||
|
||||
// SSE2 doesn't implement extract
|
||||
#if defined(__SSE4_1)
|
||||
|
||||
#define mm128_extr_64(a,n) _mm_extract_epi64( a, n )
|
||||
#define mm128_extr_32(a,n) _mm_extract_epi32( a, n )
|
||||
|
||||
#else
|
||||
|
||||
// Doesn't work with register variables.
|
||||
#define mm128_extr_64(a,n) (((uint64_t*)&a)[n])
|
||||
#define mm128_extr_32(a,n) (((uint32_t*)&a)[n])
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
// Memory functions
|
||||
// Mostly for convenience, avoids calculating bytes.
|
||||
@@ -294,13 +220,14 @@ static inline void memcpy_128( __m128i *dst, const __m128i *src, const int n )
|
||||
//
|
||||
// Bit rotations
|
||||
|
||||
// AVX512 has implemented bit rotation for 128 bit vectors with
|
||||
// AVX512VL has implemented bit rotation for 128 bit vectors with
|
||||
// 64 and 32 bit elements.
|
||||
|
||||
// compiler doesn't like when a variable is used for the last arg of
|
||||
// _mm_rol_epi32, must be "8 bit immediate". Therefore use rol_var where
|
||||
// _mm_rol_epi32, must be "8 bit immediate". Oddly _mm_slli has the same
|
||||
// specification but works with a variable. Therefore use rol_var where
|
||||
// necessary.
|
||||
// sm3-hash-4way.c fails to compile.
|
||||
// sm3-hash-4way.c has one instance where mm128_rol_var_32 is required.
|
||||
|
||||
#define mm128_ror_var_64( v, c ) \
|
||||
_mm_or_si128( _mm_srli_epi64( v, c ), _mm_slli_epi64( v, 64-(c) ) )
|
||||
@@ -392,18 +319,19 @@ static inline void memcpy_128( __m128i *dst, const __m128i *src, const int n )
|
||||
//
|
||||
// Rotate elements within lanes.
|
||||
|
||||
// Equivalent to mm128_ror_64( v, 32 )
|
||||
#define mm128_swap32_64( v ) _mm_shuffle_epi32( v, 0xb1 )
|
||||
|
||||
// Equivalent to mm128_ror_64( v, 16 )
|
||||
#define mm128_ror16_64( v ) _mm_shuffle_epi8( v, \
|
||||
m128_const_64( 0x09080f0e0d0c0b0a, 0x0100070605040302 )
|
||||
#define mm128_rol16_64( v ) _mm_shuffle_epi8( v, \
|
||||
m128_const_64( 0x0dc0b0a09080f0e, 0x0504030201000706 )
|
||||
#define mm128_ror16_64( v ) \
|
||||
_mm_shuffle_epi8( v, m128_const_64( 0x09080f0e0d0c0b0a, \
|
||||
0x0100070605040302 )
|
||||
|
||||
// Equivalent to mm128_ror_32( v, 16 )
|
||||
#define mm128_swap16_32( v ) _mm_shuffle_epi8( v, \
|
||||
m128_const_64( 0x0d0c0f0e09080b0a, 0x0504070601000302 )
|
||||
#define mm128_rol16_64( v ) \
|
||||
_mm_shuffle_epi8( v, m128_const_64( 0x0d0c0b0a09080f0e, \
|
||||
0x0504030201000706 )
|
||||
|
||||
#define mm128_swap16_32( v ) \
|
||||
_mm_shuffle_epi8( v, m128_const_64( 0x0d0c0f0e09080b0a, \
|
||||
0x0504070601000302 )
|
||||
|
||||
//
|
||||
// Endian byte swap.
|
||||
@@ -418,8 +346,9 @@ static inline void memcpy_128( __m128i *dst, const __m128i *src, const int n )
|
||||
_mm_shuffle_epi8( v, m128_const_64( 0x0c0d0e0f08090a0b, \
|
||||
0x0405060700010203 ) )
|
||||
|
||||
#define mm128_bswap_16( v ) _mm_shuffle_epi8( \
|
||||
m128_const_64( 0x0e0f0c0d0a0b0809, 0x0607040502030001 )
|
||||
#define mm128_bswap_16( v ) \
|
||||
_mm_shuffle_epi8( v, m128_const_64( 0x0e0f0c0d0a0b0809, \
|
||||
0x0607040502030001 )
|
||||
|
||||
// 8 byte qword * 8 qwords * 2 lanes = 128 bytes
|
||||
#define mm128_block_bswap_64( d, s ) do \
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
#if !defined(SIMD_256_H__)
|
||||
#define SIMD_256_H__ 1
|
||||
|
||||
#if defined(__AVX__)
|
||||
#if defined(__AVX2__)
|
||||
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
@@ -14,176 +14,68 @@
|
||||
// is limited because 256 bit vectors are less likely to be used when 512
|
||||
// is available.
|
||||
|
||||
// set instructions load memory resident constants, this avoids mem.
|
||||
// cost 4 pinsert + 1 vinsert, estimate 8 clocks latency.
|
||||
// Move integer to low element of vector, other elements are set to zero.
|
||||
|
||||
#if defined(__AVX2__)
|
||||
#define mm256_mov64_256( n ) _mm256_castsi128_si256( mm128_mov64_128( n ) )
|
||||
#define mm256_mov32_256( n ) _mm256_castsi128_si256( mm128_mov32_128( n ) )
|
||||
|
||||
#define m256_const_128( hi, lo ) \
|
||||
#define mm256_mov256_64( a ) mm128_mov128_64( _mm256_castsi256_si128( a ) )
|
||||
#define mm256_mov256_32( a ) mm128_mov128_32( _mm256_castsi256_si128( a ) )
|
||||
|
||||
// concatenate two 128 bit vectors into one 256 bit vector: { hi, lo }
|
||||
#define mm256_concat_128( hi, lo ) \
|
||||
_mm256_inserti128_si256( _mm256_castsi128_si256( lo ), hi, 1 )
|
||||
|
||||
#define m256_const_64( i3, i2, i1, i0 ) \
|
||||
m256_const_128( m128_const_64( i3, i2 ), m128_const_64( i1, i0 ) )
|
||||
#define m256_const1_128( v ) \
|
||||
_mm256_broadcastsi128_si256( v )
|
||||
|
||||
/*
|
||||
#define m256_const_64( i3, i2, i1, i0 ) \
|
||||
_mm256_inserti128_si256( _mm256_castsi128_si256( m128_const_64( i1, i0 ) ), \
|
||||
m128_const_64( i3, i2 ), 1 )
|
||||
*/
|
||||
|
||||
#else // AVX
|
||||
|
||||
#define m256_const_64( i3, i2, i1, i0 ) _mm256_set_epi64x( i3, i2, i1, i0 )
|
||||
|
||||
#endif
|
||||
|
||||
static inline __m256i m256_const1_64( uint64_t i )
|
||||
// Equavalent of set, move 64 bit integer constants to respective 64 bit
|
||||
// elements.
|
||||
static inline __m256i m256_const_64( const uint64_t i3, const uint64_t i2,
|
||||
const uint64_t i1, const uint64_t i0 )
|
||||
{
|
||||
__m128i a;
|
||||
asm( "movq %1, %0\n\t"
|
||||
: "=x" (a)
|
||||
: "r" (i) );
|
||||
return _mm256_broadcastq_epi64( a );
|
||||
__m128i hi, lo;
|
||||
lo = mm128_mov64_128( i0 );
|
||||
hi = mm128_mov64_128( i2 );
|
||||
lo = _mm_insert_epi64( lo, i1, 1 );
|
||||
hi = _mm_insert_epi64( hi, i3, 1 );
|
||||
return mm256_concat_128( hi, lo );
|
||||
}
|
||||
|
||||
static inline __m256i m256_const1_32( uint32_t i )
|
||||
{
|
||||
__m128i a;
|
||||
asm( "movd %1, %0\n\t"
|
||||
: "=x" (a)
|
||||
: "r" (i) );
|
||||
return _mm256_broadcastd_epi32( a );
|
||||
}
|
||||
// Broadcast 128 bits in pairs of 64 bit integer constants {i1. i0} to all
|
||||
// 128 bit lanes.
|
||||
#define m256_const2_64( i1, i0 ) \
|
||||
_mm256_permute4x64_epi64( _mm256_castsi128_si256( \
|
||||
m128_const_64( i1, i0 ) ), 0x44 )
|
||||
|
||||
static inline __m256i m256_const1_16( uint16_t i )
|
||||
{
|
||||
__m128i a;
|
||||
asm( "movw %1, %0\n\t"
|
||||
: "=x" (a)
|
||||
: "r" (i) );
|
||||
return _mm256_broadcastw_epi16( a );
|
||||
}
|
||||
// Equivalent of set1, broadcast integer constant to all elements.
|
||||
#define m256_const1_64( i ) _mm256_broadcastq_epi64( mm128_mov64_128( i ) )
|
||||
#define m256_const1_32( i ) _mm256_broadcastd_epi32( mm128_mov32_128( i ) )
|
||||
#define m256_const1_16( i ) _mm256_broadcastw_epi16( mm128_mov32_128( i ) )
|
||||
#define m256_const1_8 ( i ) _mm256_broadcastb_epi8 ( mm128_mov32_128( i ) )
|
||||
|
||||
static inline __m256i m256_const1_8( uint8_t i )
|
||||
{
|
||||
__m128i a;
|
||||
asm( "movb %1, %0\n\t"
|
||||
: "=x" (a)
|
||||
: "r" (i) );
|
||||
return _mm256_broadcastb_epi8( a );
|
||||
}
|
||||
|
||||
//
|
||||
// All SIMD constant macros are actually functions containing executable
|
||||
// code and therefore can't be used as compile time initializers.
|
||||
|
||||
#define m256_zero _mm256_setzero_si256()
|
||||
|
||||
#if defined(__AVX2__)
|
||||
|
||||
// Don't call the frunction directly, use the macro to make appear like
|
||||
// a constant identifier instead of a function.
|
||||
// __m256i foo = m256_one_64;
|
||||
|
||||
static inline __m256i mm256_one_256_fn()
|
||||
{
|
||||
__m256i a;
|
||||
const uint64_t one = 1;
|
||||
asm( "movq %1, %0\n\t"
|
||||
: "=x" (a)
|
||||
: "r" (one) );
|
||||
return a;
|
||||
}
|
||||
#define m256_one_256 mm256_one_256_fn()
|
||||
|
||||
static inline __m256i mm256_one_128_fn()
|
||||
{
|
||||
__m128i a;
|
||||
const uint64_t one = 1;
|
||||
asm( "movq %1, %0\n\t"
|
||||
: "=x" (a)
|
||||
: "r" (one) );
|
||||
return _mm256_broadcastsi128_si256( a );
|
||||
}
|
||||
#define m256_one_128 mm256_one_128_fn()
|
||||
|
||||
static inline __m256i mm256_one_64_fn()
|
||||
{
|
||||
__m128i a;
|
||||
const uint64_t one = 1;
|
||||
asm( "movq %1, %0\n\t"
|
||||
: "=x" (a)
|
||||
: "r" (one) );
|
||||
return _mm256_broadcastq_epi64( a );
|
||||
}
|
||||
#define m256_one_64 mm256_one_64_fn()
|
||||
|
||||
static inline __m256i mm256_one_32_fn()
|
||||
{
|
||||
__m128i a;
|
||||
const uint64_t one = 0x0000000100000001;
|
||||
asm( "movq %1, %0\n\t"
|
||||
: "=x" (a)
|
||||
: "r" (one) );
|
||||
return _mm256_broadcastq_epi64( a );
|
||||
}
|
||||
#define m256_one_32 mm256_one_32_fn()
|
||||
|
||||
static inline __m256i mm256_one_16_fn()
|
||||
{
|
||||
__m128i a;
|
||||
const uint64_t one = 0x0001000100010001;
|
||||
asm( "movq %1, %0\n\t"
|
||||
: "=x" (a)
|
||||
: "r" (one) );
|
||||
return _mm256_broadcastq_epi64( a );
|
||||
}
|
||||
#define m256_one_16 mm256_one_16_fn()
|
||||
|
||||
static inline __m256i mm256_one_8_fn()
|
||||
{
|
||||
__m128i a;
|
||||
const uint64_t one = 0x0101010101010101;
|
||||
asm( "movq %1, %0\n\t"
|
||||
: "=x" (a)
|
||||
: "r" (one) );
|
||||
return _mm256_broadcastq_epi64( a );
|
||||
}
|
||||
#define m256_one_8 mm256_one_8_fn()
|
||||
#define m256_zero _mm256_setzero_si256()
|
||||
#define m256_one_256 mm256_mov64_256( 1 )
|
||||
#define m256_one_128 \
|
||||
_mm256_permute4x64_epi64( _mm256_castsi128_si256( \
|
||||
mm128_mov64_128( 1 ) ), 0x44 )
|
||||
#define m256_one_64 _mm256_broadcastq_epi64( mm128_mov64_128( 1 ) )
|
||||
#define m256_one_32 _mm256_broadcastd_epi32( mm128_mov64_128( 1 ) )
|
||||
#define m256_one_16 _mm256_broadcastw_epi16( mm128_mov64_128( 1 ) )
|
||||
#define m256_one_8 _mm256_broadcastb_epi8 ( mm128_mov64_128( 1 ) )
|
||||
|
||||
static inline __m256i mm256_neg1_fn()
|
||||
{
|
||||
__m256i a;
|
||||
asm( "vpcmpeqq %0, %0, %0\n\t"
|
||||
: "=x"(a) );
|
||||
asm( "vpcmpeqq %0, %0, %0\n\t" : "=x"(a) );
|
||||
return a;
|
||||
}
|
||||
#define m256_neg1 mm256_neg1_fn()
|
||||
|
||||
#else // AVX
|
||||
|
||||
#define m256_one_256 m256_const_64( m128_zero, m128_one ) \
|
||||
_mm256_inserti128_si256( _mm256_castsi128_si256( m128_one_128 ), \
|
||||
m128_zero, 1 )
|
||||
|
||||
#define m256_one_128 \
|
||||
_mm256_inserti128_si256( _mm256_castsi128_si256( m128_one_128 ), \
|
||||
m128_one_128, 1 )
|
||||
|
||||
#define m256_one_64 _mm256_set1_epi64x( 1ULL )
|
||||
#define m256_one_32 _mm256_set1_epi64x( 0x0000000100000001ULL )
|
||||
#define m256_one_16 _mm256_set1_epi64x( 0x0001000100010001ULL )
|
||||
#define m256_one_8 _mm256_set1_epi64x( 0x0101010101010101ULL )
|
||||
|
||||
// AVX doesn't have inserti128 but insertf128 will do.
|
||||
static inline __m256i mm256_neg1_fn()
|
||||
{
|
||||
__m128i a = m128_neg1;
|
||||
return _mm256_insertf128_si256( _mm256_castsi128_si256( a ), a, 1 );
|
||||
}
|
||||
#define m256_neg1 mm256_neg1_fn()
|
||||
|
||||
#endif // AVX2 else AVX
|
||||
#define m256_neg1 mm256_neg1_fn()
|
||||
|
||||
|
||||
//
|
||||
@@ -202,58 +94,32 @@ static inline __m256i mm256_neg1_fn()
|
||||
#define mm128_extr_hi128_256( a ) _mm256_extracti128_si256( a, 1 )
|
||||
|
||||
// Extract integers from 256 bit vector, ineficient, avoid if possible..
|
||||
#define mm256_extr_4x64( a0, a1, a2, a3, src ) \
|
||||
#define mm256_extr_4x64( a3, a2, a1, a0, src ) \
|
||||
do { \
|
||||
__m128i hi = _mm256_extracti128_si256( src, 1 ); \
|
||||
a0 = mm256_mov256_64( src ); \
|
||||
a0 = mm128_mov128_64( _mm256_castsi256_si128( src) ); \
|
||||
a1 = _mm_extract_epi64( _mm256_castsi256_si128( src ), 1 ); \
|
||||
a2 = mm128_mov128_64( hi ); \
|
||||
a3 = _mm_extract_epi64( hi, 1 ); \
|
||||
} while(0)
|
||||
|
||||
#define mm256_extr_8x32( a0, a1, a2, a3, a4, a5, a6, a7, src ) \
|
||||
#define mm256_extr_8x32( a7, a6, a5, a4, a3, a2, a1, a0, src ) \
|
||||
do { \
|
||||
uint64_t t = _mm_extract_epi64( _mm256_castsi256_si128( src ), 1 ); \
|
||||
__m128i hi = _mm256_extracti128_si256( src, 1 ); \
|
||||
a0 = mm256_mov256_32( src ); \
|
||||
a1 = _mm_extract_epi32( _mm256_castsi256_si128( src ), 1 ); \
|
||||
a2 = _mm_extract_epi32( _mm256_castsi256_si128( src ), 2 ); \
|
||||
a3 = _mm_extract_epi32( _mm256_castsi256_si128( src ), 3 ); \
|
||||
a2 = (uint32_t)( t ); \
|
||||
a3 = (uint32_t)( t<<32 ); \
|
||||
t = _mm_extract_epi64( hi, 1 ); \
|
||||
a4 = mm128_mov128_32( hi ); \
|
||||
a5 = _mm_extract_epi32( hi, 1 ); \
|
||||
a6 = _mm_extract_epi32( hi, 2 ); \
|
||||
a7 = _mm_extract_epi32( hi, 3 ); \
|
||||
a6 = (uint32_t)( t ); \
|
||||
a7 = (uint32_t)( t<<32 ); \
|
||||
} while(0)
|
||||
|
||||
// concatenate two 128 bit vectors into one 256 bit vector: { hi, lo }
|
||||
#define mm256_concat_128( hi, lo ) \
|
||||
_mm256_inserti128_si256( _mm256_castsi128_si256( lo ), hi, 1 )
|
||||
|
||||
// Move integer to lower bits of vector, upper bits set to zero.
|
||||
static inline __m256i mm256_mov64_256( uint64_t n )
|
||||
{
|
||||
__m128i a;
|
||||
asm( "movq %1, %0\n\t"
|
||||
: "=x" (a)
|
||||
: "r" (n) );
|
||||
return _mm256_castsi128_si256( a );
|
||||
}
|
||||
|
||||
static inline __m256i mm256_mov32_256( uint32_t n )
|
||||
{
|
||||
__m128i a;
|
||||
asm( "movd %1, %0\n\t"
|
||||
: "=x" (a)
|
||||
: "r" (n) );
|
||||
return _mm256_castsi128_si256( a );
|
||||
}
|
||||
|
||||
// Return lo bits of vector as integer.
|
||||
#define mm256_mov256_64( a ) mm128_mov128_64( _mm256_castsi256_si128( a ) )
|
||||
|
||||
#define mm256_mov256_32( a ) mm128_mov128_32( _mm256_castsi256_si128( a ) )
|
||||
|
||||
// Horizontal vector testing
|
||||
#if defined(__AVX2__)
|
||||
|
||||
#define mm256_allbits0( a ) _mm256_testz_si256( a, a )
|
||||
#define mm256_allbits1( a ) _mm256_testc_si256( a, m256_neg1 )
|
||||
@@ -261,21 +127,6 @@ static inline __m256i mm256_mov32_256( uint32_t n )
|
||||
#define mm256_anybits0 mm256_allbitsne
|
||||
#define mm256_anybits1 mm256_allbitsne
|
||||
|
||||
#else // AVX
|
||||
|
||||
// Bit-wise test of entire vector, useful to test results of cmp.
|
||||
#define mm256_anybits0( a ) \
|
||||
( (uint128_t)mm128_extr_hi128_256( a ) \
|
||||
| (uint128_t)mm128_extr_lo128_256( a ) )
|
||||
|
||||
#define mm256_anybits1( a ) \
|
||||
( ( (uint128_t)mm128_extr_hi128_256( a ) + 1 ) \
|
||||
| ( (uint128_t)mm128_extr_lo128_256( a ) + 1 ) )
|
||||
|
||||
#define mm256_allbits0_256( a ) ( !mm256_anybits1(a) )
|
||||
#define mm256_allbits1_256( a ) ( !mm256_anybits0(a) )
|
||||
|
||||
#endif // AVX2 else AVX
|
||||
|
||||
// Parallel AES, for when x is expected to be in a 256 bit register.
|
||||
// Use same 128 bit key.
|
||||
@@ -324,12 +175,6 @@ static inline void memset_256( __m256i *dst, const __m256i a, const int n )
|
||||
static inline void memcpy_256( __m256i *dst, const __m256i *src, const int n )
|
||||
{ for ( int i = 0; i < n; i ++ ) dst[i] = src[i]; }
|
||||
|
||||
///////////////////////////////
|
||||
//
|
||||
// AVX2 needed from now on.
|
||||
//
|
||||
|
||||
#if defined(__AVX2__)
|
||||
|
||||
//
|
||||
// Basic operations without SIMD equivalent
|
||||
@@ -464,6 +309,21 @@ static inline void memcpy_256( __m256i *dst, const __m256i *src, const int n )
|
||||
//
|
||||
// AVX2 has no full vector permute for elements less than 32 bits.
|
||||
// AVX512 has finer granularity full vector permutes.
|
||||
// AVX512 has full vector alignr which might be faster, especially for 32 bit
|
||||
|
||||
/*
|
||||
#if defined(__AVX512F__) && defined(__AVX512VL__) && defined(__AVX512DQ__) && defined(__AVX512BW__)
|
||||
|
||||
#define mm256_swap_128( v ) _mm256_alignr_epi64( v, v, 2 )
|
||||
#define mm256_ror_1x64( v ) _mm256_alignr_epi64( v, v, 1 )
|
||||
#define mm256_rol_1x64( v ) _mm256_alignr_epi64( v, v, 3 )
|
||||
#define mm256_ror_1x32( v ) _mm256_alignr_epi32( v, v, 1 )
|
||||
#define mm256_rol_1x32( v ) _mm256_alignr_epi32( v, v, 7 )
|
||||
#define mm256_ror_3x32( v ) _mm256_alignr_epi32( v, v, 3 )
|
||||
#define mm256_rol_3x32( v ) _mm256_alignr_epi32( v, v, 5 )
|
||||
|
||||
#else // AVX2
|
||||
*/
|
||||
|
||||
// Swap 128 bit elements in 256 bit vector.
|
||||
#define mm256_swap_128( v ) _mm256_permute4x64_epi64( v, 0x4e )
|
||||
@@ -472,7 +332,6 @@ static inline void memcpy_256( __m256i *dst, const __m256i *src, const int n )
|
||||
#define mm256_ror_1x64( v ) _mm256_permute4x64_epi64( v, 0x39 )
|
||||
#define mm256_rol_1x64( v ) _mm256_permute4x64_epi64( v, 0x93 )
|
||||
|
||||
// A little faster with avx512
|
||||
// Rotate 256 bit vector by one 32 bit element.
|
||||
#define mm256_ror_1x32( v ) \
|
||||
_mm256_permutevar8x32_epi32( v, \
|
||||
@@ -495,6 +354,9 @@ static inline void memcpy_256( __m256i *dst, const __m256i *src, const int n )
|
||||
m256_const_64( 0x0000000400000003, 0x0000000200000001, \
|
||||
0x0000000000000007, 0x0000000600000005 )
|
||||
|
||||
//#endif // AVX512 else AVX2
|
||||
|
||||
|
||||
// AVX512 can do 16 & 8 bit elements.
|
||||
#if defined(__AVX512F__) && defined(__AVX512VL__) && defined(__AVX512DQ__) && defined(__AVX512BW__)
|
||||
|
||||
@@ -537,18 +399,16 @@ static inline void memcpy_256( __m256i *dst, const __m256i *src, const int n )
|
||||
|
||||
// Invert vector: {7,6,5,4,3,2,1,0} -> {0,1,2,3,4,5,6,7}
|
||||
#define mm256_invert_16 ( v ) \
|
||||
_mm256_permutexvar_epi16( m256_const_64( 0x0000000100020003, \
|
||||
0x0004000500060007, \
|
||||
0x00080009000a000b, \
|
||||
0x000c000d000e000f ), v )
|
||||
_mm256_permutexvar_epi16( m256_const_64( \
|
||||
0x0000000100020003, 0x0004000500060007, \
|
||||
0x00080009000a000b, 0x000c000d000e000f ), v )
|
||||
|
||||
#if defined(__AVX512VBMI__)
|
||||
|
||||
#define mm256_invert_8( v ) \
|
||||
_mm256_permutexvar_epi8( m256_const_64( 0x0001020304050607, \
|
||||
0x08090a0b0c0d0e0f, \
|
||||
0x1011121314151617, \
|
||||
0x18191a1b1c1d1e1f ), v )
|
||||
_mm256_permutexvar_epi8( m256_const_64( \
|
||||
0x0001020304050607, 0x08090a0b0c0d0e0f, \
|
||||
0x1011121314151617, 0x18191a1b1c1d1e1f ), v )
|
||||
#endif // VBMI
|
||||
#endif // AVX512
|
||||
|
||||
@@ -565,27 +425,19 @@ static inline void memcpy_256( __m256i *dst, const __m256i *src, const int n )
|
||||
|
||||
// Rotate each 128 bit lane by one 16 bit element.
|
||||
#define mm256_ror1x16_128( v ) \
|
||||
_mm256_shuffle_epi8( v, m256_const_64( 0x01000f0e0d0c0b0a, \
|
||||
0x0908070605040302, \
|
||||
0x01000f0e0d0c0b0a, \
|
||||
0x0908070605040302 ) )
|
||||
_mm256_shuffle_epi8( v, m256_const2_64( 0x01000f0e0d0c0b0a, \
|
||||
0x0908070605040302 ) )
|
||||
#define mm256_rol1x16_128( v ) \
|
||||
_mm256_shuffle_epi8( v, m256_const_64( 0x0d0c0b0a09080706, \
|
||||
0x0504030201000f0e, \
|
||||
0x0d0c0b0a09080706, \
|
||||
0x0504030201000f0e ) )
|
||||
_mm256_shuffle_epi8( v, m256_const2_64( 0x0d0c0b0a09080706, \
|
||||
0x0504030201000f0e ) )
|
||||
|
||||
// Rotate each 128 bit lane by one byte
|
||||
#define mm256_ror1x8_128( v ) \
|
||||
_mm256_shuffle_epi8( v, m256_const_64( 0x000f0e0d0c0b0a09, \
|
||||
0x0807060504030201, \
|
||||
0x000f0e0d0c0b0a09, \
|
||||
0x0807060504030201 ) )
|
||||
_mm256_shuffle_epi8( v, m256_const2_64( 0x000f0e0d0c0b0a09, \
|
||||
0x0807060504030201 ) )
|
||||
#define mm256_rol1x8_128( v ) \
|
||||
_mm256_shuffle_epi8( v, m256_const_64( 0x0c0b0a09080f0e0d, \
|
||||
0x0504030201000706, \
|
||||
0x0d0c0b0a09080f0e, \
|
||||
0x0504030201000706 ) )
|
||||
_mm256_shuffle_epi8( v, m256_const2_64( 0x0d0c0b0a09080f0e, \
|
||||
0x0504030201000706 ) )
|
||||
|
||||
// Rotate each 128 bit lane by c bytes.
|
||||
#define mm256_bror_128( v, c ) \
|
||||
@@ -599,70 +451,50 @@ static inline void memcpy_256( __m256i *dst, const __m256i *src, const int n )
|
||||
#define mm256_swap32_64( v ) _mm256_shuffle_epi32( v, 0xb1 )
|
||||
|
||||
#define mm256_ror1x16_64( v ) \
|
||||
_mm256_shuffle_epi8( v, m256_const_64( 0x09080f0e0d0c0b0a, \
|
||||
0x0100070605040302, \
|
||||
0x09080f0e0d0c0b0a, \
|
||||
0x0100070605040302 ) )
|
||||
_mm256_shuffle_epi8( v, m256_const2_64( 0x09080f0e0d0c0b0a, \
|
||||
0x0100070605040302 ) )
|
||||
#define mm256_rol1x16_64( v ) \
|
||||
_mm256_shuffle_epi8( v, m256_const_64( 0x0d0c0b0a09080f0e, \
|
||||
0x0504030201000706, \
|
||||
0x0d0c0b0a09080f0e, \
|
||||
0x0504030201000706 ))
|
||||
_mm256_shuffle_epi8( v, m256_const2_64( 0x0d0c0b0a09080f0e, \
|
||||
0x0504030201000706 ) )
|
||||
|
||||
#define mm256_ror1x8_64( v ) \
|
||||
_mm256_shuffle_epi8( v, m256_const_64( 0x080f0e0d0c0b0a09, \
|
||||
0x0007060504030201, \
|
||||
0x080f0e0d0c0b0a09, \
|
||||
0x0007060504030201 ))
|
||||
_mm256_shuffle_epi8( v, m256_const2_64( 0x080f0e0d0c0b0a09, \
|
||||
0x0007060504030201 ) )
|
||||
#define mm256_rol1x8_64( v ) \
|
||||
_mm256_shuffle_epi8( v, m256_const_64( 0x0e0d0c0b0a09080f, \
|
||||
0x0605040302010007, \
|
||||
0x0e0d0c0b0a09080f, \
|
||||
0x0605040302010007 ) )
|
||||
_mm256_shuffle_epi8( v, m256_const2_64( 0x0e0d0c0b0a09080f, \
|
||||
0x0605040302010007 ) )
|
||||
|
||||
#define mm256_ror3x8_64( v ) \
|
||||
_mm256_shuffle_epi8( v, m256_const_64( 0x0a09080f0e0d0c0b, \
|
||||
0x0201000706050403, \
|
||||
0x0a09080f0e0d0c0b, \
|
||||
0x0201000706050403 ) )
|
||||
_mm256_shuffle_epi8( v, m256_const2_64( 0x0a09080f0e0d0c0b, \
|
||||
0x0201000706050403 ) )
|
||||
#define mm256_rol3x8_64( v ) \
|
||||
_mm256_shuffle_epi8( v, m256_const_64( 0x0c0b0a09080f0e0d, \
|
||||
0x0403020100070605, \
|
||||
0x0c0b0a09080f0e0d, \
|
||||
0x0403020100070605 ) )
|
||||
_mm256_shuffle_epi8( v, m256_const2_64( 0x0c0b0a09080f0e0d, \
|
||||
0x0403020100070605 ) )
|
||||
|
||||
// Swap 16 bit elements in each 32 bit lane
|
||||
#define mm256_swap16_32( v ) \
|
||||
_mm256_shuffle_epi8( v, m256_const_64( 0x0b0a09080f0e0d0c, \
|
||||
0x0302010007060504, \
|
||||
0x0b0a09080f0e0d0c, \
|
||||
0x0302010007060504 )
|
||||
_mm256_shuffle_epi8( v, m256_const2_64( 0x0b0a09080f0e0d0c, \
|
||||
0x0302010007060504 ) )
|
||||
|
||||
//
|
||||
// Swap bytes in vector elements, endian bswap.
|
||||
#define mm256_bswap_64( v ) \
|
||||
_mm256_shuffle_epi8( v, m256_const_64( 0x08090a0b0c0d0e0f, \
|
||||
0x0001020304050607, \
|
||||
0x08090a0b0c0d0e0f, \
|
||||
0x0001020304050607 ) )
|
||||
_mm256_shuffle_epi8( v, m256_const2_64( 0x08090a0b0c0d0e0f, \
|
||||
0x0001020304050607 ) )
|
||||
|
||||
#define mm256_bswap_32( v ) \
|
||||
_mm256_shuffle_epi8( v, m256_const_64( 0x0c0d0e0f08090a0b, \
|
||||
0x0405060700010203, \
|
||||
0x0c0d0e0f08090a0b, \
|
||||
0x0405060700010203 ) )
|
||||
_mm256_shuffle_epi8( v, m256_const2_64( 0x0c0d0e0f08090a0b, \
|
||||
0x0405060700010203 ) )
|
||||
|
||||
#define mm256_bswap_16( v ) \
|
||||
_mm256_shuffle_epi8( v, m256_const_64( 0x0e0f0c0d0a0b0809, \
|
||||
0x0607040502030001, \
|
||||
0x0e0f0c0d0a0b0809, \
|
||||
0x0607040502030001 ) )
|
||||
_mm256_shuffle_epi8( v, m256_const2_64( 0x0e0f0c0d0a0b0809, \
|
||||
0x0607040502030001 ) )
|
||||
|
||||
// Source and destination are pointers, may point to same memory.
|
||||
// 8 byte qword * 8 qwords * 4 lanes = 256 bytes
|
||||
#define mm256_block_bswap_64( d, s ) do \
|
||||
{ \
|
||||
__m256i ctl = m256_const_64( 0x08090a0b0c0d0e0f, 0x0001020304050607, \
|
||||
0x08090a0b0c0d0e0f, 0x0001020304050607 ); \
|
||||
__m256i ctl = m256_const2_64( 0x08090a0b0c0d0e0f, 0x0001020304050607 ); \
|
||||
casti_m256i( d, 0 ) = _mm256_shuffle_epi8( casti_m256i( s, 0 ), ctl ); \
|
||||
casti_m256i( d, 1 ) = _mm256_shuffle_epi8( casti_m256i( s, 1 ), ctl ); \
|
||||
casti_m256i( d, 2 ) = _mm256_shuffle_epi8( casti_m256i( s, 2 ), ctl ); \
|
||||
@@ -676,8 +508,7 @@ static inline void memcpy_256( __m256i *dst, const __m256i *src, const int n )
|
||||
// 4 byte dword * 8 dwords * 8 lanes = 256 bytes
|
||||
#define mm256_block_bswap_32( d, s ) do \
|
||||
{ \
|
||||
__m256i ctl = m256_const_64( 0x0c0d0e0f08090a0b, 0x0405060700010203, \
|
||||
0x0c0d0e0f08090a0b, 0x0405060700010203 ); \
|
||||
__m256i ctl = m256_const2_64( 0x0c0d0e0f08090a0b, 0x0405060700010203 ); \
|
||||
casti_m256i( d, 0 ) = _mm256_shuffle_epi8( casti_m256i( s, 0 ), ctl ); \
|
||||
casti_m256i( d, 1 ) = _mm256_shuffle_epi8( casti_m256i( s, 1 ), ctl ); \
|
||||
casti_m256i( d, 2 ) = _mm256_shuffle_epi8( casti_m256i( s, 2 ), ctl ); \
|
||||
@@ -695,6 +526,9 @@ static inline void memcpy_256( __m256i *dst, const __m256i *src, const int n )
|
||||
// Some of these can use permute but appears to be slower. Maybe a Ryzen
|
||||
// issue
|
||||
|
||||
// _mm256_alignr_epi 64/32 are only available with AVX512 but AVX512 also
|
||||
// makes these macros unnecessary.
|
||||
|
||||
#define mm256_swap256_512 (v1, v2) \
|
||||
v1 = _mm256_xor_si256(v1, v2); \
|
||||
v2 = _mm256_xor_si256(v1, v2); \
|
||||
@@ -702,75 +536,18 @@ static inline void memcpy_256( __m256i *dst, const __m256i *src, const int n )
|
||||
|
||||
#define mm256_ror1x128_512( v1, v2 ) \
|
||||
do { \
|
||||
__m256i t = _mm256_alignr_epi8( v1, v2, 16 ); \
|
||||
v1 = _mm256_alignr_epi8( v2, v1, 16 ); \
|
||||
__m256i t = _mm256_permute2x128( v1, v2, 0x03 ); \
|
||||
v1 = _mm256__mm256_permute2x128( v2, v1, 0x21 ); \
|
||||
v2 = t; \
|
||||
} while(0)
|
||||
|
||||
#define mm256_rol1x128_512( v1, v2 ) \
|
||||
do { \
|
||||
__m256i t = _mm256_alignr_epi8( v1, v2, 16 ); \
|
||||
v2 = _mm256_alignr_epi8( v2, v1, 16 ); \
|
||||
v1 = t; \
|
||||
} while(0)
|
||||
|
||||
#define mm256_ror1x64_512( v1, v2 ) \
|
||||
do { \
|
||||
__m256i t = _mm256_alignr_epi8( v1, v2, 8 ); \
|
||||
v1 = _mm256_alignr_epi8( v2, v1, 8 ); \
|
||||
v2 = t; \
|
||||
} while(0)
|
||||
|
||||
#define mm256_rol1x64_512( v1, v2 ) \
|
||||
do { \
|
||||
__m256i t = _mm256_alignr_epi8( v1, v2, 24 ); \
|
||||
v2 = _mm256_alignr_epi8( v2, v1, 24 ); \
|
||||
v1 = t; \
|
||||
} while(0)
|
||||
|
||||
#define mm256_ror1x32_512( v1, v2 ) \
|
||||
do { \
|
||||
__m256i t = _mm256_alignr_epi8( v1, v2, 4 ); \
|
||||
v1 = _mm256_alignr_epi8( v2, v1, 4 ); \
|
||||
v2 = t; \
|
||||
} while(0)
|
||||
|
||||
#define mm256_rol1x32_512( v1, v2 ) \
|
||||
do { \
|
||||
__m256i t = _mm256_alignr_epi8( v1, v2, 28 ); \
|
||||
v2 = _mm256_alignr_epi8( v2, v1, 28 ); \
|
||||
v1 = t; \
|
||||
} while(0)
|
||||
|
||||
#define mm256_ror1x16_512( v1, v2 ) \
|
||||
do { \
|
||||
__m256i t = _mm256_alignr_epi8( v1, v2, 2 ); \
|
||||
v1 = _mm256_alignr_epi8( v2, v1, 2 ); \
|
||||
v2 = t; \
|
||||
} while(0)
|
||||
|
||||
#define mm256_rol1x16_512( v1, v2 ) \
|
||||
do { \
|
||||
__m256i t = _mm256_alignr_epi8( v1, v2, 30 ); \
|
||||
v2 = _mm256_alignr_epi8( v2, v1, 30 ); \
|
||||
v1 = t; \
|
||||
} while(0)
|
||||
|
||||
#define mm256_ror1x8_512( v1, v2 ) \
|
||||
do { \
|
||||
__m256i t = _mm256_alignr_epi8( v1, v2, 1 ); \
|
||||
v1 = _mm256_alignr_epi8( v2, v1, 1 ); \
|
||||
v2 = t; \
|
||||
} while(0)
|
||||
|
||||
#define mm256_rol1x8_512( v1, v2 ) \
|
||||
do { \
|
||||
__m256i t = _mm256_alignr_epi8( v1, v2, 31 ); \
|
||||
v2 = _mm256_alignr_epi8( v2, v1, 31 ); \
|
||||
__m256i t = _mm256_permute2x128( v1, v2, 0x03 ); \
|
||||
v2 = _mm256__mm256_permute2x128( v2, v1, 0x21 ); \
|
||||
v1 = t; \
|
||||
} while(0)
|
||||
|
||||
#endif // __AVX2__
|
||||
#endif // __AVX__
|
||||
#endif // SIMD_256_H__
|
||||
|
||||
|
||||
@@ -37,74 +37,84 @@
|
||||
//
|
||||
// Experimental, not fully tested.
|
||||
|
||||
//
|
||||
// Pseudo constants.
|
||||
//
|
||||
// Vector constants are not really constants and can't be used as compile time
|
||||
// initializers. They contain executable instructions to generate values at
|
||||
// run time. They are very slow. If the same constant will be used repeatedly
|
||||
// in a function it's better to define it once in a local register variable
|
||||
// and use the variable for references.
|
||||
// Tthe simpler the constant, the more efficienct it's generation. Zero is
|
||||
// the fastest, then all elements set the same, different 64 bit elements,
|
||||
// and different smaller elements is the slowest. Caching multiple uses us
|
||||
// always faster.
|
||||
// Move integer to/from element 0 of vector.
|
||||
|
||||
#define m512_const_256( hi, lo ) \
|
||||
#define mm512_mov64_512( n ) _mm512_castsi128_si512( mm128_mov64_128( n ) )
|
||||
#define mm512_mov32_512( n ) _mm512_castsi128_si512( mm128_mov32_128( n ) )
|
||||
|
||||
#define mm512_mov256_64( a ) mm128_mov128_64( _mm256_castsi512_si128( a ) )
|
||||
#define mm512_mov256_32( a ) mm128_mov128_32( _mm256_castsi512_si128( a ) )
|
||||
|
||||
|
||||
// Insert and extract integers is a multistage operation.
|
||||
// Insert integer into __m128i, then insert __m128i to __m256i, finally
|
||||
// insert __256i into __m512i. Reverse the order for extract.
|
||||
// Do not use __m512_insert_epi64 or _mm256_insert_epi64 to perform multiple
|
||||
// inserts.
|
||||
// Avoid small integers for multiple inserts.
|
||||
// Shortcuts:
|
||||
// Use castsi to reference the low bits of a vector or sub-vector. (free)
|
||||
// Use mov to insert integer into low bits of vector or sub-vector. (cheap)
|
||||
// Use _mm_insert only to reference the high bits of __m128i. (expensive)
|
||||
// Sequence instructions to minimize data dependencies.
|
||||
// Use const or const1 only when integer is either immediate or known to be in
|
||||
// a GP register. Use set/set1 when data needs to be loaded from memory or
|
||||
// cache.
|
||||
|
||||
// Concatenate two 256 bit vectors into one 512 bit vector {hi, lo}
|
||||
#define mm512_concat_256( hi, lo ) \
|
||||
_mm512_inserti64x4( _mm512_castsi256_si512( lo ), hi, 1 )
|
||||
|
||||
#define m512_const_128( i3, i2, i1, i0 ) \
|
||||
_mm512_inserti64x4( _mm512_castsi256_si512( m256_const_128( i1, i0 ) ), \
|
||||
m256_const_128( i3,i2 ), 1 )
|
||||
|
||||
#define m512_const_64( i7, i6, i5, i4, i3, i2, i1, i0 ) \
|
||||
m512_const_256( m256_const_64( i7,i6,i5,i4 ), \
|
||||
m256_const_64( i3,i2,i1,i0 ) )
|
||||
|
||||
static inline __m512i m512_const1_256( __m256i v )
|
||||
// Equivalent of set, assign 64 bit integers to respective 64 bit elements.
|
||||
static inline __m512i m512_const_64( const uint64_t i7, const uint64_t i6,
|
||||
const uint64_t i5, const uint64_t i4,
|
||||
const uint64_t i3, const uint64_t i2,
|
||||
const uint64_t i1, const uint64_t i0 )
|
||||
{
|
||||
return _mm512_broadcast_i64x4( v );
|
||||
__m256i hi, lo;
|
||||
__m128i hi1, lo1;
|
||||
lo = mm256_mov64_256( i0 );
|
||||
lo1 = mm128_mov64_128( i2 );
|
||||
hi = mm256_mov64_256( i4 );
|
||||
hi1 = mm128_mov64_128( i6 );
|
||||
lo = _mm256_castsi128_si256(
|
||||
_mm_insert_epi64( _mm256_castsi256_si128( lo ), i1, 1 ) );
|
||||
lo1 = _mm_insert_epi64( lo1, i3, 1 );
|
||||
hi = _mm256_castsi128_si256(
|
||||
_mm_insert_epi64( _mm256_castsi256_si128( hi ), i5, 1 ) );
|
||||
hi1 = _mm_insert_epi64( hi1, i7, 1 );
|
||||
lo = _mm256_inserti128_si256( lo, lo1, 1 );
|
||||
hi = _mm256_inserti128_si256( hi, hi1, 1 );
|
||||
return mm512_concat_256( hi, lo );
|
||||
}
|
||||
|
||||
static inline __m512i m512_const1_128( __m128i v )
|
||||
// Equivalent of set4, broadcast 256 bits in groups of four 64 bit constants
|
||||
// to all 256 bit lanes: {i3,i2,i1,i0,i3,i2,i1,i0,i3,i2,i1,i0,i3,i2,i1,i0}.
|
||||
static inline __m512i mm512_const4_64( const uint64_t i3, const uint64_t i2,
|
||||
const uint64_t i1, const uint64_t i0 )
|
||||
{
|
||||
return _mm512_broadcast_i64x2( v );
|
||||
__m256i lo = mm256_mov64_256( i0 );
|
||||
__m128i hi = mm128_mov64_128( i2 );
|
||||
lo = _mm256_castsi128_si256(
|
||||
_mm_insert_epi64( _mm256_castsi256_si128(
|
||||
lo ), i1, 1 ) );
|
||||
hi = _mm_insert_epi64( hi, i3, 1 );
|
||||
return _mm512_permutex_epi64( _mm512_castsi256_si512(
|
||||
_mm256_inserti128_si256( lo, hi, 1 ) ), 0xe4 );
|
||||
}
|
||||
|
||||
static inline __m512i m512_const1_64( uint64_t i )
|
||||
{
|
||||
__m128i a;
|
||||
asm( "movq %1, %0\n\t"
|
||||
: "=x"(a)
|
||||
: "r"(i) );
|
||||
return _mm512_broadcastq_epi64( a );
|
||||
}
|
||||
// Broadcast 128 bits in pairs of 64 bit constants {i1. i0} to all
|
||||
// 128 bit lanes.
|
||||
#define mm512_const2_64( i1, i0 ) \
|
||||
_mm512_permutex_epi64( _mm512_castsi128_si512( \
|
||||
m128_const_64( i1, i0 ) ), 0x44 )
|
||||
|
||||
static inline __m512i m512_const1_32( uint32_t i )
|
||||
{
|
||||
__m128i a;
|
||||
asm( "movd %1, %0\n\t"
|
||||
: "=x"(a)
|
||||
: "r"(i) );
|
||||
return _mm512_broadcastd_epi32( a );
|
||||
}
|
||||
// Equivalent of set1, broadcast 64 bit constant to all 64 bit elements.
|
||||
#define m512_const1_64( i ) _mm512_broadcastq_epi64( mm128_mov64_128( i ) )
|
||||
#define m512_const1_32( i ) _mm512_broadcastd_epi32( mm128_mov32_128( i ) )
|
||||
#define m512_const1_16( i ) _mm512_broadcastw_epi16( mm128_mov32_128( i ) )
|
||||
#define m512_const1_8 ( i ) _mm512_broadcastb_epi8 ( mm128_mov32_128( i ) )
|
||||
|
||||
static inline __m512i m512_const1_16( uint16_t i )
|
||||
{
|
||||
__m128i a;
|
||||
asm( "movw %1, %0\n\t"
|
||||
: "=x"(a)
|
||||
: "r"(i) );
|
||||
return _mm512_broadcastw_epi16( a );
|
||||
}
|
||||
static inline __m512i m512_const1_8( uint8_t i )
|
||||
{
|
||||
__m128i a;
|
||||
asm( "movb %1, %0\n\t"
|
||||
: "=x"(a)
|
||||
: "r"(i) );
|
||||
return _mm512_broadcastb_epi8( a );
|
||||
}
|
||||
|
||||
//
|
||||
// Pseudo constants.
|
||||
@@ -114,105 +124,26 @@ static inline __m512i m512_const1_8( uint8_t i )
|
||||
// initialized to zero.
|
||||
|
||||
#define m512_zero _mm512_setzero_si512()
|
||||
#define m512_one_512 mm512_mov64_512( 1 )
|
||||
#define m512_one_256 _mm512_broadcast_i64x4 ( mm256_mov64_256( 1 ) )
|
||||
#define m512_one_128 _mm512_broadcast_i64x2 ( mm128_mov64_128( 1 ) )
|
||||
#define m512_one_64 _mm512_broadcastq_epi64( mm128_mov64_128( 1 ) )
|
||||
#define m512_one_32 _mm512_broadcastd_epi32( mm128_mov64_128( 1 ) )
|
||||
#define m512_one_16 _mm512_broadcastw_epi16( mm128_mov64_128( 1 ) )
|
||||
#define m512_one_8 _mm512_broadcastb_epi8 ( mm128_mov64_128( 1 ) )
|
||||
|
||||
/*
|
||||
#define m512_one_512 _mm512_set_epi64( 0ULL, 0ULL, 0ULL, 0ULL, \
|
||||
0ULL, 0ULL, 0ULL, 1ULL )
|
||||
#define m512_one_256 _mm512_set4_epi64( 0ULL, 0ULL, 0ULL, 1ULL )
|
||||
#define m512_one_128 _mm512_set4_epi64( 0ULL, 1ULL, 0ULL, 1ULL )
|
||||
#define m512_one_64 _mm512_set1_epi64( 1ULL )
|
||||
#define m512_one_32 _mm512_set1_epi32( 1UL )
|
||||
#define m512_one_16 _mm512_set1_epi16( 1U )
|
||||
#define m512_one_8 _mm512_set1_epi8( 1U )
|
||||
#define m512_neg1 _mm512_set1_epi64( 0xFFFFFFFFFFFFFFFFULL )
|
||||
*/
|
||||
|
||||
static inline __m512i mm512_one_512_fn()
|
||||
{
|
||||
__m512i a;
|
||||
const uint64_t one = 1;
|
||||
asm( "movq %1, %0\n\t"
|
||||
: "=x" (a)
|
||||
: "r" (one) );
|
||||
return a;
|
||||
}
|
||||
#define m512_one_512 mm512_one_512_fn()
|
||||
|
||||
static inline __m512i mm512_one_256_fn()
|
||||
{
|
||||
__m256i a;
|
||||
const uint64_t one = 1;
|
||||
asm( "movq %1, %0\n\t"
|
||||
: "=x"(a)
|
||||
: "r" (one) );
|
||||
return _mm512_broadcast_i64x4( a );
|
||||
}
|
||||
#define m512_one_256 mm512_one_256_fn()
|
||||
|
||||
static inline __m512i mm512_one_128_fn()
|
||||
{
|
||||
__m128i a;
|
||||
const uint64_t one = 1;
|
||||
asm( "movq %1, %0\n\t"
|
||||
: "=x"(a)
|
||||
: "r" (one) );
|
||||
return _mm512_broadcast_i64x2( a );
|
||||
}
|
||||
#define m512_one_128 mm512_one_128_fn()
|
||||
|
||||
static inline __m512i mm512_one_64_fn()
|
||||
{
|
||||
__m128i a;
|
||||
const uint64_t one = 1;
|
||||
asm( "movq %1, %0\n\t"
|
||||
: "=x"(a)
|
||||
: "r" (one) );
|
||||
return _mm512_broadcastq_epi64( a );
|
||||
}
|
||||
#define m512_one_64 mm512_one_64_fn()
|
||||
|
||||
static inline __m512i mm512_one_32_fn()
|
||||
{
|
||||
__m128i a;
|
||||
const uint64_t one = 0x0000000100000001;
|
||||
asm( "movd %1, %0\n\t"
|
||||
: "=x"(a)
|
||||
: "r" (one) );
|
||||
return _mm512_broadcastq_epi64( a );
|
||||
}
|
||||
#define m512_one_32 mm512_one_32_fn()
|
||||
|
||||
static inline __m512i mm512_one_16_fn()
|
||||
{
|
||||
__m128i a;
|
||||
const uint64_t one = 0x0001000100010001;
|
||||
asm( "movd %1, %0\n\t"
|
||||
: "=x"(a)
|
||||
: "r" (one) );
|
||||
return _mm512_broadcastq_epi64( a );
|
||||
}
|
||||
#define m512_one_16 mm512_one_16_fn()
|
||||
|
||||
static inline __m512i mm512_one_8_fn()
|
||||
{
|
||||
__m128i a;
|
||||
const uint64_t one = 0x0101010101010101;
|
||||
asm( "movd %1, %0\n\t"
|
||||
: "=x"(a)
|
||||
: "r" (one) );
|
||||
return _mm512_broadcastq_epi64( a );
|
||||
}
|
||||
#define m512_one_8 mm512_one_8_fn()
|
||||
#define m512_neg1 mm512_const1_64( 0xffffffffffffffff )
|
||||
|
||||
/*
|
||||
// EVEX vcmpeqq returns a bit mask instead of a vector
|
||||
static inline __m512i mm512_neg1_fn()
|
||||
{
|
||||
__m512i a;
|
||||
asm( "vpcmpeqq %0, %0, %0\n\t"
|
||||
:"=x"(a) );
|
||||
asm( "vpcmpeqq %0, %0, %0\n\t" : "=x"(a) );
|
||||
return a;
|
||||
}
|
||||
#define m512_neg1 mm512_neg1_fn()
|
||||
|
||||
*/
|
||||
|
||||
//
|
||||
// Basic operations without SIMD equivalent
|
||||
@@ -222,12 +153,6 @@ static inline __m512i mm512_neg1_fn()
|
||||
#define mm512_negate_32( x ) _mm512_sub_epi32( m512_zero, x )
|
||||
#define mm512_negate_16( x ) _mm512_sub_epi16( m512_zero, x )
|
||||
|
||||
// More efficient to use cast to extract low lanes, it's free.
|
||||
#define mm256_extr_lo256_512( a ) _mm512_castsi512_si256( a )
|
||||
#define mm256_extr_hi256_512( a ) _mm512_extracti64x4_epi64( a, 1 )
|
||||
|
||||
#define mm128_extr_lo128_512( a ) _mm512_castsi512_si256( a )
|
||||
|
||||
|
||||
//
|
||||
// Pointer casting
|
||||
@@ -267,16 +192,9 @@ static inline __m512i mm512_neg1_fn()
|
||||
_mm512_xor_si512( _mm512_xor_si256( a, b ), _mm512_xor_si256( c, d ) )
|
||||
|
||||
|
||||
// Vector size conversion
|
||||
|
||||
#define mm256_extr_lo256_512( a ) _mm512_castsi512_si256( a )
|
||||
#define mm256_extr_hi256_512( a ) _mm512_extracti64x4_epi64( a, 1 )
|
||||
|
||||
#define mm512_concat_256( hi, lo ) \
|
||||
_mm512_inserti164x4( _mm512_castsi256_si512( lo ), hi, 1 )
|
||||
|
||||
// Horizontal vector testing
|
||||
|
||||
// Returns bit mask
|
||||
#define mm512_allbits0( a ) _mm512_cmpeq_epi64_mask( a, m512_zero )
|
||||
#define mm512_allbits1( a ) _mm512_cmpeq_epi64_mask( a, m512_neg1 )
|
||||
#define mm512_anybits0( a ) _mm512_cmpneq_epi64_mask( a, m512_neg1 )
|
||||
@@ -331,25 +249,46 @@ static inline __m512i mm512_neg1_fn()
|
||||
// Swap bytes in vector elements, vectorized endian conversion.
|
||||
|
||||
#define mm512_bswap_64( v ) \
|
||||
_mm512_shuffle_epi8( v, m512_const_64( \
|
||||
0x38393A3B3C3D3E3F, 0x3031323334353637, \
|
||||
0x28292A2B2C2D2E2F, 0x2021222324252627, \
|
||||
0x18191A1B1C1D1E1F, 0x1011121314151617, \
|
||||
0x08090A0B0C0D0E0F, 0x0001020304050607 ) )
|
||||
_mm512_shuffle_epi8( v, m512_const2_64( \
|
||||
0x08090a0b0c0d0e0f, 0x0001020304050607 ) )
|
||||
|
||||
#define mm512_bswap_32( v ) \
|
||||
_mm512_shuffle_epi8( v, m512_const_64( \
|
||||
0x3C3D3E3F38393A3B, 0x3435363730313233, \
|
||||
0x3C3D3E3F38393A3B, 0x3435363730313233, \
|
||||
0x3C3D3E3F38393A3B, 0x3435363730313233, \
|
||||
0x3C3D3E3F38393A3B, 0x3435363730313233 ) )
|
||||
_mm512_shuffle_epi8( v, m512_const2_64( \
|
||||
0x0c0d0e0f08090a0b, 0x0405060700010203 ) )
|
||||
|
||||
#define mm512_bswap_16( v ) \
|
||||
_mm512_shuffle_epi8( v, m512_const_64( \
|
||||
0x3E3F3C3D3A3B3839, 0x3637343532333031, \
|
||||
0x2E2F2C2D2A2B2829, 0x2627242522232021, \
|
||||
0x1E1F1C1D1A1B1819, 0x1617141512131011, \
|
||||
0x0E0F0C0D0A0B0809, 0x0607040502030001 ) )
|
||||
_mm512_shuffle_epi8( v, m512_const2_64( \
|
||||
0x0e0f0c0d0a0b0809, 0x0607040502030001 ) )
|
||||
|
||||
// Source and destination are pointers, may point to same memory.
|
||||
// 8 lanes of 64 bytes each
|
||||
#define mm512_block_bswap_64( d, s ) do \
|
||||
{ \
|
||||
__m512i ctl = m512_const2_64( 0x08090a0b0c0d0e0f, 0x0001020304050607 ); \
|
||||
casti_m512i( d, 0 ) = _mm512_shuffle_epi8( casti_m512i( s, 0 ), ctl ); \
|
||||
casti_m512i( d, 1 ) = _mm512_shuffle_epi8( casti_m512i( s, 1 ), ctl ); \
|
||||
casti_m512i( d, 2 ) = _mm512_shuffle_epi8( casti_m512i( s, 2 ), ctl ); \
|
||||
casti_m512i( d, 3 ) = _mm512_shuffle_epi8( casti_m512i( s, 3 ), ctl ); \
|
||||
casti_m512i( d, 4 ) = _mm512_shuffle_epi8( casti_m512i( s, 4 ), ctl ); \
|
||||
casti_m512i( d, 5 ) = _mm512_shuffle_epi8( casti_m512i( s, 5 ), ctl ); \
|
||||
casti_m512i( d, 6 ) = _mm512_shuffle_epi8( casti_m512i( s, 6 ), ctl ); \
|
||||
casti_m512i( d, 7 ) = _mm512_shuffle_epi8( casti_m512i( s, 7 ), ctl ); \
|
||||
} while(0)
|
||||
|
||||
// 16 lanes of 32 bytes each
|
||||
#define mm512_block_bswap_32( d, s ) do \
|
||||
{ \
|
||||
__m512i ctl = m512_const2_64( 0x0c0d0e0f08090a0b, 0x0405060700010203 ); \
|
||||
casti_m512i( d, 0 ) = _mm512_shuffle_epi8( casti_m512i( s, 0 ), ctl ); \
|
||||
casti_m512i( d, 1 ) = _mm512_shuffle_epi8( casti_m512i( s, 1 ), ctl ); \
|
||||
casti_m512i( d, 2 ) = _mm512_shuffle_epi8( casti_m512i( s, 2 ), ctl ); \
|
||||
casti_m512i( d, 3 ) = _mm512_shuffle_epi8( casti_m512i( s, 3 ), ctl ); \
|
||||
casti_m512i( d, 4 ) = _mm512_shuffle_epi8( casti_m512i( s, 4 ), ctl ); \
|
||||
casti_m512i( d, 5 ) = _mm512_shuffle_epi8( casti_m512i( s, 5 ), ctl ); \
|
||||
casti_m512i( d, 6 ) = _mm512_shuffle_epi8( casti_m512i( s, 6 ), ctl ); \
|
||||
casti_m512i( d, 7 ) = _mm512_shuffle_epi8( casti_m512i( s, 7 ), ctl ); \
|
||||
} while(0)
|
||||
|
||||
|
||||
//
|
||||
// Rotate elements in 512 bit vector.
|
||||
@@ -367,8 +306,10 @@ static inline __m512i mm512_neg1_fn()
|
||||
|
||||
// Generic for odd rotations
|
||||
#define mm512_ror_x64( v, n ) _mm512_alignr_epi64( v, v, n )
|
||||
#define mm512_rol_x64( v, n ) _mm512_alignr_epi64( v, v, 8-n )
|
||||
|
||||
#define mm512_ror_x32( v, n ) _mm512_alignr_epi32( v, v, n )
|
||||
#define mm512_rol_x32( v, n ) _mm512_alignr_epi32( v, v, 16-n )
|
||||
|
||||
#define mm512_ror_1x16( v ) \
|
||||
_mm512_permutexvar_epi16( m512_const_64( \
|
||||
@@ -400,7 +341,11 @@ static inline __m512i mm512_neg1_fn()
|
||||
|
||||
|
||||
// Invert vector: {3,2,1,0} -> {0,1,2,3}
|
||||
#define mm512_invert_128( v ) _mm512_shuffle_i64x2( v, v, 0x1b )
|
||||
#define mm512_invert_256( v ) \
|
||||
_mm512_permutexvar_epi64( v, m512_const_64( 3,2,1,0,7,6,5,4 ) )
|
||||
|
||||
#define mm512_invert_128( v ) \
|
||||
_mm512_permutexvar_epi64( v, m512_const_64( 1,0,3,2,5,4,7,6 ) )
|
||||
|
||||
#define mm512_invert_64( v ) \
|
||||
_mm512_permutexvar_epi64( v, m512_const_64( 0,1,2,3,4,5,6,7 ) )
|
||||
@@ -438,84 +383,60 @@ static inline __m512i mm512_neg1_fn()
|
||||
|
||||
// Rotate 256 bit lanes by one 32 bit element
|
||||
#define mm512_ror1x32_256( v ) \
|
||||
_mm512_permutexvar_epi32( m512_const_64( \
|
||||
0x000000080000000f, 0x0000000e0000000d, \
|
||||
0x0000000c0000000b, 0x0000000a00000009, \
|
||||
_mm512_permutexvar_epi32( m512_const4_64( \
|
||||
0x0000000000000007, 0x0000000600000005, \
|
||||
0x0000000400000003, 0x0000000200000001, v ) )
|
||||
|
||||
0x0000000400000003, 0x0000000200000001 ), v )
|
||||
|
||||
#define mm512_rol1x32_256( v ) \
|
||||
_mm512_permutexvar_epi32( m512_const_64( \
|
||||
0x0000000e0000000d, 0x0000000c0000000b, \
|
||||
0x0000000a00000009, 0x000000080000000f, \
|
||||
_mm512_permutexvar_epi32( m512_const4_64( \
|
||||
0x0000000600000005, 0x0000000400000003, \
|
||||
0x0000000200000001, 0x0000000000000007 ), v )
|
||||
|
||||
#define mm512_ror1x16_256( v ) \
|
||||
_mm512_permutexvar_epi16( m512_const_64( \
|
||||
0x0010001F001E001D, 0x001C001B001A0019, \
|
||||
0x0018001700160015, 0x0014001300120011, \
|
||||
0x0000000F000E000D, 0x000C000B000A0009, \
|
||||
_mm512_permutexvar_epi16( m512_const4_64( \
|
||||
0x0000000f000e000d, 0x000c000b000a0009, \
|
||||
0x0008000700060005, 0x0004000300020001 ), v )
|
||||
|
||||
#define mm512_rol1x16_256( v ) \
|
||||
_mm512_permutexvar_epi16( m512_const_64( \
|
||||
0x001E001D001C001B, 0x001A001900180017, \
|
||||
0x0016001500140013, 0x001200110000000F, \
|
||||
0x000E000D000C000B, 0x000A000900080007, \
|
||||
0x0006000500040003, 0x000200010000001F ), v )
|
||||
_mm512_permutexvar_epi16( m512_const4_64( \
|
||||
0x000e000d000c000b, 0x000a000900080007, \
|
||||
0x0006000500040003, 0x000200010000000f ), v )
|
||||
|
||||
#define mm512_ror1x8_256( v ) \
|
||||
_mm512_shuffle_epi8( v, m512_const_64( \
|
||||
0x203F3E3D3C3B3A39, 0x3837363534333231, \
|
||||
0x302F2E2D2C2B2A29, 0x2827262524232221, \
|
||||
0x001F1E1D1C1B1A19, 0x1817161514131211, \
|
||||
0x100F0E0D0C0B0A09, 0x0807060504030201 ) )
|
||||
_mm512_shuffle_epi8( v, m512_const4_64( \
|
||||
0x001f1e1d1c1b1a19, 0x1817161514131211, \
|
||||
0x100f0e0d0c0b0a09, 0x0807060504030201 ), v )
|
||||
|
||||
#define mm512_rol1x8_256( v ) \
|
||||
_mm512_shuffle_epi8( v, m512_const_64( \
|
||||
0x3E3D3C3B3A393837, 0x363534333231302F, \
|
||||
0x2E2D2C2B2A292827, 0x262524232221203F, \
|
||||
0x1E1D1C1B1A191817, 0x161514131211100F, \
|
||||
0x0E0D0C0B0A090807, 0x060504030201001F ))
|
||||
_mm512_shuffle_epi8( v, m512_const4_64( \
|
||||
0x1e1d1c1b1a191817, 0x161514131211100f, \
|
||||
0x0e0d0c0b0a090807, 0x060504030201001f ), v )
|
||||
|
||||
//
|
||||
// Rotate elements within 128 bit lanes of 512 bit vector.
|
||||
|
||||
// Swap hi & lo 64 bits in each 128 bit lane
|
||||
#define mm512_swap64_128( v ) _mm512_permutex_epi64( v, 0xb1 )
|
||||
#define mm512_swap64_128( v ) _mm512_shuffle_epi32( v, 0x4e )
|
||||
|
||||
// Rotate 128 bit lanes by one 32 bit element
|
||||
#define mm512_ror1x32_128( v ) _mm512_shuffle_epi32( v, 0x39 )
|
||||
#define mm512_rol1x32_128( v ) _mm512_shuffle_epi32( v, 0x93 )
|
||||
|
||||
#define mm512_ror1x16_128( v ) \
|
||||
_mm512_permutexvar_epi16( m512_const_64( \
|
||||
0x0018001F001E001D, 0x001C001B001A0019, \
|
||||
0x0010001700160015, 0x0014001300120011, \
|
||||
0x0008000F000E000D, 0x000C000B000A0009, \
|
||||
0x0000000700060005, 0x0004000300020001 ), v )
|
||||
_mm512_permutexvar_epi16( m512_const2_64( \
|
||||
0x0000000700060005, 0x0004000300020001 ), v )
|
||||
|
||||
#define mm512_rol1x16_128( v ) \
|
||||
_mm512_permutexvar_epi16( m512_const_64( \
|
||||
0x001E001D001C001B, 0x001A00190018001F, \
|
||||
0x0016001500140013, 0x0012001100100017, \
|
||||
0x000E000D000C000B, 0x000A00090008000F, \
|
||||
0x0006000500040003, 0x0002000100000007, v ) )
|
||||
_mm512_permutexvar_epi16( m512_const2_64( \
|
||||
0x0006000500040003, 0x0002000100000007 ), v )
|
||||
|
||||
#define mm512_ror1x8_128( v ) \
|
||||
_mm512_shuffle_epi8( v, m512_const_64( \
|
||||
0x303F3E3D3C3B3A39, 0x3837363534333231, \
|
||||
0x202F2E2D2C2B2A29, 0x2827262524232221, \
|
||||
0x101F1E1D1C1B1A19, 0x1817161514131211, \
|
||||
0x000F0E0D0C0B0A09, 0x0807060504030201 ) )
|
||||
_mm512_shuffle_epi8( v, m512_const2_64( \
|
||||
0x000f0e0d0c0b0a09, 0x0807060504030201 ) )
|
||||
|
||||
#define mm512_rol1x8_128( v ) \
|
||||
_mm512_shuffle_epi8( v, m512_const_64( \
|
||||
0x3E3D3C3B3A393837, 0x363534333231303F, \
|
||||
0x2E2D2C2B2A292827, 0x262524232221202F, \
|
||||
0x1E1D1C1B1A191817, 0x161514131211101F, \
|
||||
0x0E0D0C0B0A090807, 0x060504030201000F ) )
|
||||
_mm512_shuffle_epi8( v, m512_const2_64( \
|
||||
0x0e0d0c0b0a090807, 0x060504030201000f ) )
|
||||
|
||||
// Rotate 128 bit lanes by c bytes.
|
||||
#define mm512_bror_128( v, c ) \
|
||||
@@ -652,33 +573,5 @@ do { \
|
||||
v1 = t; \
|
||||
} while(0)
|
||||
|
||||
#define mm512_ror1x16_1024( v1, v2 ) \
|
||||
do { \
|
||||
__m512i t = _mm512_alignr_epi8( v1, v2, 2 ); \
|
||||
v1 = _mm512_alignr_epi8( v2, v1, 2 ); \
|
||||
v2 = t; \
|
||||
} while(0)
|
||||
|
||||
#define mm512_rol1x16_1024( v1, v2 ) \
|
||||
do { \
|
||||
__m512i t = _mm512_alignr_epi8( v1, v2, 62 ); \
|
||||
v2 = _mm512_alignr_epi8( v2, v1, 62 ); \
|
||||
v1 = t; \
|
||||
} while(0)
|
||||
|
||||
#define mm512_ror1x8_1024( v1, v2 ) \
|
||||
do { \
|
||||
__m512i t = _mm512_alignr_epi8( v1, v2, 1 ); \
|
||||
v1 = _mm512_alignr_epi8( v2, v1, 1 ); \
|
||||
v2 = t; \
|
||||
} while(0)
|
||||
|
||||
#define mm512_rol1x8_1024( v1, v2 ) \
|
||||
do { \
|
||||
__m512i t = _mm512_alignr_epi8( v1, v2, 63 ); \
|
||||
v2 = _mm512_alignr_epi8( v2, v1, 63 ); \
|
||||
v1 = t; \
|
||||
} while(0)
|
||||
|
||||
#endif // AVX512
|
||||
#endif // SIMD_512_H__
|
||||
|
||||
Reference in New Issue
Block a user