popov/cpuminer-opt-gpu
1
0
Fork 0
mirror of https://github.com/JayDDee/cpuminer-opt.git synced 2025-09-17 23:44:27 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
9abc19a30a56ed9c0f5080edd5380120474072a8
cpuminer-opt-gpu/simd-utils/intrlv-avx512.h
Jay D Dee 0d769ee0fe v3.9.5.1
2019-07-02 15:10:38 -04:00

680 lines
32 KiB
C
Raw Blame History

#if !defined(INTRLV_AVX512_H__)
#define INTRLV_AVX512_H__ 1
#if defined(__AVX512VL__) && defined(__AVX512DQ__) && defined(__AVX512BW__)
// SSE2 functions used in AVX512 interleaving
// AVX512 block is 64 * 64 bytes
// quarter avx512 block, 16 bytes * 16 lanes
static inline void mm128_dintrlv_16x32x128( void *d00, void *d01,
void *d02, void *d03, void *d04, void *d05, void *d06, void *d07,
void *d08, void *d09, void *d10, void *d11, void *d12, void *d13,
void *d14, void *d15, const int n, const void *s )
{
cast_m128i( d00 ) = mm128_get_32( s, 0, 16, 32, 48 );
cast_m128i( d01 ) = mm128_get_32( s, 1, 17, 33, 49 );
cast_m128i( d02 ) = mm128_get_32( s, 2, 18, 34, 50 );
cast_m128i( d03 ) = mm128_get_32( s, 3, 19, 35, 51 );
cast_m128i( d04 ) = mm128_get_32( s, 4, 20, 36, 52 );
cast_m128i( d05 ) = mm128_get_32( s, 5, 21, 37, 53 );
cast_m128i( d06 ) = mm128_get_32( s, 6, 22, 38, 54 );
cast_m128i( d07 ) = mm128_get_32( s, 7, 23, 39, 55 );
cast_m128i( d08 ) = mm128_get_32( s, 8, 24, 40, 56 );
cast_m128i( d09 ) = mm128_get_32( s, 9, 25, 41, 57 );
cast_m128i( d10 ) = mm128_get_32( s, 10, 26, 42, 58 );
cast_m128i( d11 ) = mm128_get_32( s, 11, 27, 43, 59 );
cast_m128i( d12 ) = mm128_get_32( s, 12, 28, 44, 60 );
cast_m128i( d13 ) = mm128_get_32( s, 13, 29, 45, 61 );
cast_m128i( d14 ) = mm128_get_32( s, 14, 30, 46, 62 );
cast_m128i( d15 ) = mm128_get_32( s, 15, 31, 47, 63 );
}
// quarter avx512 block, 32 bytes * 8 lanes
// 8 lanes of 128 bits using 64 bit interleaving
// Used for last 16 bytes of 80 byte input, only used for testing.
static inline void mm128_dintrlv_8x64x128( void *d0, void *d1, void *d2,
void *d3, void *d4, void *d5, void *d6, void *d7,
const int n, const void *s )
{
casti_m128i( d0,n ) = mm128_get_64( s, 0, 8 );
casti_m128i( d1,n ) = mm128_get_64( s, 1, 9 );
casti_m128i( d2,n ) = mm128_get_64( s, 2, 10 );
casti_m128i( d3,n ) = mm128_get_64( s, 3, 11 );
casti_m128i( d4,n ) = mm128_get_64( s, 4, 12 );
casti_m128i( d5,n ) = mm128_get_64( s, 5, 13 );
casti_m128i( d6,n ) = mm128_get_64( s, 6, 14 );
casti_m128i( d7,n ) = mm128_get_64( s, 7, 15 );
}
static inline void mm128_dintrlv_4x128x128( void *d0, void *d1, void *d2,
void *d3, const int n, const void *s )
{
casti_m128i( d0,n ) = mm128_get_64( s, 0, 1 );
casti_m128i( d1,n ) = mm128_get_64( s, 2, 3 );
casti_m128i( d2,n ) = mm128_get_64( s, 4, 5 );
casti_m128i( d3,n ) = mm128_get_64( s, 5, 7 );
}
// AVX2 functions Used in AVX512 interleaving
static inline void mm256_dintrlv_16x32x256( void *d00, void *d01,
void *d02, void *d03, void *d04, void *d05,
void *d06, void *d07, void *d08, void *d09,
void *d10, void *d11, void *d12, void *d13,
void *d14, void *d15, const int n, const void *s )
{
casti_m256i( d00,n ) = mm256_get_32( s, 0, 16, 32, 48, 64, 80, 96,112 );
casti_m256i( d01,n ) = mm256_get_32( s, 1, 17, 33, 49, 65, 81, 97,113 );
casti_m256i( d02,n ) = mm256_get_32( s, 2, 18, 34, 50, 66, 82, 98,114 );
casti_m256i( d03,n ) = mm256_get_32( s, 3, 19, 35, 51, 67, 83, 99,115 );
casti_m256i( d04,n ) = mm256_get_32( s, 4, 20, 36, 52, 68, 84,100,116 );
casti_m256i( d05,n ) = mm256_get_32( s, 5, 21, 37, 53, 69, 85,101,117 );
casti_m256i( d06,n ) = mm256_get_32( s, 6, 22, 38, 54, 70, 86,102,118 );
casti_m256i( d07,n ) = mm256_get_32( s, 7, 23, 39, 55, 71, 87,103,119 );
casti_m256i( d08,n ) = mm256_get_32( s, 8, 24, 40, 56, 72, 88,104,120 );
casti_m256i( d09,n ) = mm256_get_32( s, 9, 25, 41, 57, 73, 89,105,121 );
casti_m256i( d10,n ) = mm256_get_32( s, 10, 26, 42, 58, 74, 90,106,122 );
casti_m256i( d11,n ) = mm256_get_32( s, 11, 27, 43, 59, 75, 91,107,123 );
casti_m256i( d12,n ) = mm256_get_32( s, 12, 28, 44, 60, 76, 92,108,124 );
casti_m256i( d13,n ) = mm256_get_32( s, 13, 29, 45, 61, 77, 93,109,125 );
casti_m256i( d14,n ) = mm256_get_32( s, 14, 30, 46, 62, 78, 94,110,126 );
casti_m256i( d15,n ) = mm256_get_32( s, 15, 31, 47, 63, 79, 95,111,127 );
}
// 8 lanes of 256 bits using 64 bit interleaving (standard final hash size)
static inline void mm256_dintrlv_8x64x256( void *d0, void *d1, void *d2,
void *d3, void *d4, void *d5, void *d6, void *d7,
const int n, const void *s )
{
casti_m256i( d0,n ) = mm256_get_64( s, 0, 8, 16, 24 );
casti_m256i( d1,n ) = mm256_get_64( s, 1, 9, 17, 25 );
casti_m256i( d2,n ) = mm256_get_64( s, 2, 10, 18, 26 );
casti_m256i( d3,n ) = mm256_get_64( s, 3, 11, 19, 27 );
casti_m256i( d4,n ) = mm256_get_64( s, 4, 12, 20, 28 );
casti_m256i( d5,n ) = mm256_get_64( s, 5, 13, 21, 29 );
casti_m256i( d6,n ) = mm256_get_64( s, 6, 14, 22, 30 );
casti_m256i( d7,n ) = mm256_get_64( s, 7, 15, 23, 31 );
}
static inline void mm256_dintrlv_4x128x256( void *d0, void *d1, void *d2,
void *d3, const int n, const void *s )
{
casti_m256i( d0,n ) = mm256_get_64( s, 0, 1, 8, 9 );
casti_m256i( d1,n ) = mm256_get_64( s, 2, 3, 10, 11 );
casti_m256i( d2,n ) = mm256_get_64( s, 4, 5, 12, 13 );
casti_m256i( d3,n ) = mm256_get_64( s, 6, 7, 14, 15 );
}
// AVX 512 helper functions.
//
// Macro functions returning vector.
// Abstracted typecasting, avoid temp pointers.
// Source arguments may be any 64 or 32 byte aligned pointer as appropriate.
#define mm512_put_64( s0, s1, s2, s3, s4, s5, s6, s7 ) \
_mm512_set_epi64( *((const uint64_t*)(s7)), *((const uint64_t*)(s6)), \
*((const uint64_t*)(s5)), *((const uint64_t*)(s4)), \
*((const uint64_t*)(s3)), *((const uint64_t*)(s2)), \
*((const uint64_t*)(s1)), *((const uint64_t*)(s0)) )
#define mm512_put_32( s00, s01, s02, s03, s04, s05, s06, s07, \
s08, s09, s10, s11, s12, s13, s14, s15 ) \
_mm512_set_epi32( *((const uint32_t*)(s15)), *((const uint32_t*)(s14)), \
*((const uint32_t*)(s13)), *((const uint32_t*)(s12)), \
*((const uint32_t*)(s11)), *((const uint32_t*)(s10)), \
*((const uint32_t*)(s09)), *((const uint32_t*)(s08)), \
*((const uint32_t*)(s07)), *((const uint32_t*)(s06)), \
*((const uint32_t*)(s05)), *((const uint32_t*)(s04)), \
*((const uint32_t*)(s03)), *((const uint32_t*)(s02)), \
*((const uint32_t*)(s01)), *((const uint32_t*)(s00)) )
#define mm512_get_64( s, i0, i1, i2, i3, i4, i5, i6, i7 ) \
_mm512_set_epi64( ((const uint64_t*)(s))[i7], ((const uint64_t*)(s))[i6], \
((const uint64_t*)(s))[i5], ((const uint64_t*)(s))[i4], \
((const uint64_t*)(s))[i3], ((const uint64_t*)(s))[i2], \
((const uint64_t*)(s))[i1], ((const uint64_t*)(s))[i0] )
#define mm512_get_32( s, i00, i01, i02, i03, i04, i05, i06, i07, \
i08, i09, i10, i11, i12, i13, i14, i15 ) \
_mm512_set_epi32( ((const uint32_t*)(s))[i15], ((const uint32_t*)(s))[i14], \
((const uint32_t*)(s))[i13], ((const uint32_t*)(s))[i12], \
((const uint32_t*)(s))[i11], ((const uint32_t*)(s))[i10], \
((const uint32_t*)(s))[i09], ((const uint32_t*)(s))[i08], \
((const uint32_t*)(s))[i07], ((const uint32_t*)(s))[i06], \
((const uint32_t*)(s))[i05], ((const uint32_t*)(s))[i04], \
((const uint32_t*)(s))[i03], ((const uint32_t*)(s))[i02], \
((const uint32_t*)(s))[i01], ((const uint32_t*)(s))[i00] )
// AVX512 has no blend, can be done with permute2xvar but at what cost?
// Can also be done with shifting and mask-or'ing for 3 instructins with
// 1 dependency. Finally it can be done with 1 _mm512_set but with 8 64 bit
// array index calculations and 8 pointer reads.
// Blend 2 vectors alternating hi & lo: { hi[n], lo[n-1], ... hi[1]. lo[0] }
#define mm512_interleave_blend_128( hi, lo ) \
_mm256_permute2xvar_epi64( hi, lo, _mm512_set_epi64( \
0x7, 0x6, 0x5, 0x4, 0xb, 0xa, 0x9, 0x8 )
#define mm512_interleave_blend_64( hi, lo ) \
_mm256_permute2xvar_epi64( hi, lo, _mm512_set_epi64( \
0x7, 0x6, 0xd, 0xc, 0x3, 0x2, 0x9, 0x8 )
#define mm512_interleave_blend_32( hi, lo ) \
_mm256_permute2xvar_epi32( hi, lo, _mm512_set_epi32( \
0x0f, 0x1e, 0x0d, 0x1c, 0x0b, 0x1a, 0x09, 0x18, \
0x07, 0x16, 0x05, 0x14, 0x03, 0x12, 0x01, 0x10 )
//
static inline void mm512_intrlv_16x32x512( void *d, const void *s00,
const void *s01, const void *s02, const void *s03, const void *s04,
const void *s05, const void *s06, const void *s07, const void *s08,
const void *s09, const void *s10, const void *s11, const void *s12,
const void *s13, const void *s14, const void *s15 )
{
casti_m512i( d, 0 ) = mm512_put_32(
s00, s01, s02, s03, s04, s05, s06, s07,
s08, s09, s10, s11, s12, s13, s14, s15 );
casti_m512i( d, 1 ) = mm512_put_32(
s00+ 4, s01+ 4, s02+ 4, s03+ 4, s04+ 4, s05+ 4, s06+ 4, s07+ 4,
s08+ 4, s09+ 4, s10+ 4, s11+ 4, s12+ 4, s13+ 4, s14+ 4, s15+ 4 );
casti_m512i( d, 2 ) = mm512_put_32(
s00+ 8, s01+ 8, s02+ 8, s03+ 8, s04+ 8, s05+ 8, s06+ 8, s07+ 8,
s08+ 8, s09+ 8, s10+ 8, s11+ 8, s12+ 8, s13+ 8, s14+ 8, s15+ 8 );
casti_m512i( d, 3 ) = mm512_put_32(
s00+12, s01+12, s02+12, s03+12, s04+12, s05+12, s06+12, s07+12,
s08+12, s09+12, s10+12, s11+12, s12+12, s13+12, s14+12, s15+12 );
casti_m512i( d, 4 ) = mm512_put_32(
s00+16, s01+16, s02+16, s03+16, s04+16, s05+16, s06+16, s07+16,
s08+16, s09+16, s10+16, s11+16, s12+16, s13+16, s14+16, s15+16 );
casti_m512i( d, 5 ) = mm512_put_32(
s00+20, s01+20, s02+20, s03+20, s04+20, s05+20, s06+20, s07+20,
s08+20, s09+20, s10+20, s11+20, s12+20, s13+20, s14+20, s15+20 );
casti_m512i( d, 6 ) = mm512_put_32(
s00+24, s01+24, s02+24, s03+24, s04+24, s05+24, s06+24, s07+24,
s08+24, s09+24, s10+24, s11+24, s12+24, s13+24, s14+24, s15+24 );
casti_m512i( d, 7 ) = mm512_put_32(
s00+28, s01+28, s02+28, s03+28, s04+28, s05+28, s06+28, s07+28,
s08+28, s09+28, s10+28, s11+28, s12+28, s13+28, s14+28, s15+28 );
casti_m512i( d, 8 ) = mm512_put_32(
s00+32, s01+28, s02+28, s03+28, s04+32, s05+28, s06+28, s07+28,
s08+32, s09+28, s10+28, s11+28, s12+32, s13+28, s14+28, s15+28 );
casti_m512i( d, 9 ) = mm512_put_32(
s00+36, s01+28, s02+28, s03+28, s04+36, s05+28, s06+28, s07+28,
s08+36, s09+28, s10+28, s11+28, s12+36, s13+28, s14+28, s15+28 );
casti_m512i( d,10 ) = mm512_put_32(
s00+40, s01+28, s02+28, s03+28, s04+40, s05+28, s06+28, s07+28,
s08+40, s09+28, s10+28, s11+28, s12+40, s13+28, s14+28, s15+28 );
casti_m512i( d,11 ) = mm512_put_32(
s00+44, s01+28, s02+28, s03+28, s04+44, s05+28, s06+28, s07+28,
s08+44, s09+28, s10+28, s11+28, s12+44, s13+28, s14+28, s15+28 );
casti_m512i( d,12 ) = mm512_put_32(
s00+48, s01+28, s02+28, s03+28, s04+48, s05+28, s06+28, s07+28,
s08+48, s09+28, s10+28, s11+28, s12+48, s13+28, s14+28, s15+28 );
casti_m512i( d,13 ) = mm512_put_32(
s00+52, s01+28, s02+28, s03+28, s04+52, s05+28, s06+28, s07+28,
s08+52, s09+28, s10+28, s11+28, s12+52, s13+28, s14+28, s15+28 );
casti_m512i( d,14 ) = mm512_put_32(
s00+56, s01+28, s02+28, s03+28, s04+56, s05+28, s06+28, s07+28,
s08+56, s09+28, s10+28, s11+28, s12+56, s13+28, s14+28, s15+28 );
casti_m512i( d,15 ) = mm512_put_32(
s00+60, s01+28, s02+28, s03+28, s04+60, s05+28, s06+28, s07+28,
s08+60, s09+28, s10+28, s11+28, s12+60, s13+28, s14+28, s15+28 );
}
static inline void mm512_intrlv_16x32x256( void *d, const void *s00,
const void *s01, const void *s02, const void *s03, const void *s04,
const void *s05, const void *s06, const void *s07, const void *s08,
const void *s09, const void *s10, const void *s11, const void *s12,
const void *s13, const void *s14, const void *s15 )
{
casti_m512i( d, 0 ) = mm512_put_32(
s00, s01, s02, s03, s04, s05, s06, s07,
s08, s09, s10, s11, s12, s13, s14, s15 );
casti_m512i( d, 1 ) = mm512_put_32(
s00+ 4, s01+ 4, s02+ 4, s03+ 4, s04+ 4, s05+ 4, s06+ 4, s07+ 4,
s08+ 4, s09+ 4, s10+ 4, s11+ 4, s12+ 4, s13+ 4, s14+ 4, s15+ 4 );
casti_m512i( d, 2 ) = mm512_put_32(
s00+ 8, s01+ 8, s02+ 8, s03+ 8, s04+ 8, s05+ 8, s06+ 8, s07+ 8,
s08+ 8, s09+ 8, s10+ 8, s11+ 8, s12+ 8, s13+ 8, s14+ 8, s15+ 8 );
casti_m512i( d, 3 ) = mm512_put_32(
s00+12, s01+12, s02+12, s03+12, s04+12, s05+12, s06+12, s07+12,
s08+12, s09+12, s10+12, s11+12, s12+12, s13+12, s14+12, s15+12 );
casti_m512i( d, 4 ) = mm512_put_32(
s00+16, s01+16, s02+16, s03+16, s04+16, s05+16, s06+16, s07+16,
s08+16, s09+16, s10+16, s11+16, s12+16, s13+16, s14+16, s15+16 );
casti_m512i( d, 5 ) = mm512_put_32(
s00+20, s01+20, s02+20, s03+20, s04+20, s05+20, s06+20, s07+20,
s08+20, s09+20, s10+20, s11+20, s12+20, s13+20, s14+20, s15+20 );
casti_m512i( d, 6 ) = mm512_put_32(
s00+24, s01+24, s02+24, s03+24, s04+24, s05+24, s06+24, s07+24,
s08+24, s09+24, s10+24, s11+24, s12+24, s13+24, s14+24, s15+24 );
casti_m512i( d, 7 ) = mm512_put_32(
s00+28, s01+28, s02+28, s03+28, s04+28, s05+28, s06+28, s07+28,
s08+28, s09+28, s10+28, s11+28, s12+28, s13+28, s14+28, s15+28 );
}
// Last 16 bytes of input
static inline void mm512_intrlv_16x32x128( void *d, const void *s00,
const void *s01, const void *s02, const void *s03, const void *s04,
const void *s05, const void *s06, const void *s07, const void *s08,
const void *s09, const void *s10, const void *s11, const void *s12,
const void *s13, const void *s14, const void *s15 )
{
casti_m512i( d, 0 ) = mm512_put_32(
s00, s01, s02, s03, s04, s05, s06, s07,
s08, s09, s10, s11, s12, s13, s14, s15 );
casti_m512i( d, 1 ) = mm512_put_32(
s00+ 4, s01+ 4, s02+ 4, s03+ 4, s04+ 4, s05+ 4, s06+ 4, s07+ 4,
s08+ 4, s09+ 4, s10+ 4, s11+ 4, s12+ 4, s13+ 4, s14+ 4, s15+ 4 );
casti_m512i( d, 2 ) = mm512_put_32(
s00+ 8, s01+ 8, s02+ 8, s03+ 8, s04+ 8, s05+ 8, s06+ 8, s07+ 8,
s08+ 8, s09+ 8, s10+ 8, s11+ 8, s12+ 8, s13+ 8, s14+ 8, s15+ 8 );
casti_m512i( d, 3 ) = mm512_put_32(
s00+12, s01+12, s02+12, s03+12, s04+12, s05+12, s06+12, s07+12,
s08+12, s09+12, s10+12, s11+12, s12+12, s13+12, s14+12, s15+12 );
}
// can be called directly for 64 byte hash.
static inline void mm512_dintrlv_16x32x512( void *d00, void *d01,
void *d02, void *d03, void *d04, void *d05, void *d06,
void *d07, void *d08, void *d09, void *d10, void *d11,
void *d12, void *d13, void *d14, void *d15, const int n,
const void *s )
{
casti_m512i(d00,n) = mm512_get_32( s, 0, 16, 32, 48, 64, 80, 96,112,
128,144,160,176,192,208,224,240 );
casti_m512i(d01,n) = mm512_get_32( s, 1, 17, 33, 49, 65, 81, 97,113,
129,145,161,177,193,209,225,241 );
casti_m512i(d02,n) = mm512_get_32( s, 2, 18, 34, 50, 66, 82, 98,114,
130,146,162,178,194,210,226,242 );
casti_m512i(d03,n) = mm512_get_32( s, 3, 19, 35, 51, 67, 83, 99,115,
131,147,163,179,195,211,227,243 );
casti_m512i(d04,n) = mm512_get_32( s, 4, 20, 36, 52, 68, 84,100,116,
132,148,164,180,196,212,228,244 );
casti_m512i(d05,n) = mm512_get_32( s, 5, 21, 37, 53, 69, 85,101,117,
133,149,165,181,197,213,229,245 );
casti_m512i(d06,n) = mm512_get_32( s, 6, 22, 38, 54, 70, 86,102,118,
134,150,166,182,198,214,230,246 );
casti_m512i(d07,n) = mm512_get_32( s, 7, 23, 39, 55, 71, 87,103,119,
135,151,167,183,199,215,231,247 );
casti_m512i(d08,n) = mm512_get_32( s, 8, 24, 40, 56, 72, 88,104,120,
136,152,168,184,200,216,232,248 );
casti_m512i(d09,n) = mm512_get_32( s, 9, 25, 41, 57, 73, 89,105,121,
137,153,169,185,201,217,233,249 );
casti_m512i(d10,n) = mm512_get_32( s, 10, 26, 42, 58, 74, 90,106,122,
138,154,170,186,202,218,234,250 );
casti_m512i(d11,n) = mm512_get_32( s, 11, 27, 43, 59, 75, 91,107,123,
139,155,171,187,203,219,235,251 );
casti_m512i(d12,n) = mm512_get_32( s, 12, 28, 44, 60, 76, 92,108,124,
140,156,172,188,204,220,236,252 );
casti_m512i(d13,n) = mm512_get_32( s, 13, 29, 45, 61, 77, 93,109,125,
141,157,173,189,205,221,237,253 );
casti_m512i(d14,n) = mm512_get_32( s, 14, 30, 46, 62, 78, 94,110,126,
142,158,174,190,206,222,238,254 );
casti_m512i(d15,n) = mm512_get_32( s, 15, 31, 47, 63, 79, 95,111,127,
143,159,175,191,207,223,239,255 );
}
static inline void mm512_intrlv_8x64x512( void *d, const void *s0,
const void *s1, const void *s2, const void *s3,
const void *s4, const void *s5, const void *s6,
const void *s7 )
{
casti_m512i( d,0 ) = mm512_put_64( s0, s1, s2, s3,
s4, s5, s6, s7 );
casti_m512i( d,1 ) = mm512_put_64( s0+ 8, s1+ 8, s2+ 8, s3+ 8,
s4+ 8, s5+ 8, s6+ 8, s7+ 8 );
casti_m512i( d,2 ) = mm512_put_64( s0+16, s1+16, s2+16, s3+16,
s4+16, s5+16, s6+16, s7+16 );
casti_m512i( d,3 ) = mm512_put_64( s0+24, s1+24, s2+24, s3+24,
s4+24, s5+24, s6+24, s7+24 );
casti_m512i( d,4 ) = mm512_put_64( s0+32, s1+32, s2+32, s3+32,
s4+32, s5+32, s6+32, s7+32 );
casti_m512i( d,5 ) = mm512_put_64( s0+40, s1+40, s2+40, s3+40,
s4+40, s5+40, s6+40, s7+40 );
casti_m512i( d,6 ) = mm512_put_64( s0+48, s1+48, s2+48, s3+48,
s4+48, s5+48, s6+48, s7+48 );
casti_m512i( d,7 ) = mm512_put_64( s0+56, s1+56, s2+56, s3+56,
s4+56, s5+56, s6+56, s7+56 );
}
static inline void mm512_intrlv_8x64x256( void *d, const void *s0,
const void *s1, const void *s2, const void *s3,
const void *s4, const void *s5, const void *s6,
const void *s7 )
{
casti_m512i( d,0 ) = mm512_put_64( s0, s1, s2, s3,
s4, s5, s6, s7 );
casti_m512i( d,1 ) = mm512_put_64( s0+ 8, s1+ 8, s2+ 8, s3+ 8,
s4+ 8, s5+ 8, s6+ 8, s7+ 8 );
casti_m512i( d,2 ) = mm512_put_64( s0+16, s1+16, s2+16, s3+16,
s4+16, s5+16, s6+16, s7+16 );
casti_m512i( d,3 ) = mm512_put_64( s0+24, s1+24, s2+24, s3+24,
s4+24, s5+24, s6+24, s7+24 );
}
// 8 lanes of 512 bits using 64 bit interleaving (typical intermediate hash)
static inline void mm512_dintrlv_8x64x512( void *d0, void *d1, void *d2,
void *d3, void *d4, void *d5, void *d6, void *d7,
const int n, const void *s )
{
casti_m512i( d0,n ) = mm512_get_64( s, 0, 8, 16, 24, 32, 40, 48, 56 );
casti_m512i( d1,n ) = mm512_get_64( s, 1, 9, 17, 25, 33, 41, 49, 57 );
casti_m512i( d2,n ) = mm512_get_64( s, 2, 10, 18, 26, 34, 42, 50, 58 );
casti_m512i( d3,n ) = mm512_get_64( s, 3, 11, 19, 27, 35, 43, 51, 59 );
casti_m512i( d4,n ) = mm512_get_64( s, 4, 12, 20, 28, 36, 44, 52, 60 );
casti_m512i( d5,n ) = mm512_get_64( s, 5, 13, 21, 29, 37, 45, 53, 61 );
casti_m512i( d6,n ) = mm512_get_64( s, 6, 14, 22, 30, 38, 46, 54, 62 );
casti_m512i( d7,n ) = mm512_get_64( s, 7, 15, 23, 31, 39, 47, 55, 63 );
}
static inline void mm512_dintrlv_4x128x512( void *d0, void *d1, void *d2,
void *d3, const int n, const void *s )
{
casti_m512i( d0,n ) = mm512_get_64( s, 0, 1, 8, 9, 16, 17, 24, 25 );
casti_m512i( d1,n ) = mm512_get_64( s, 2, 3, 10, 11, 18, 19, 16, 27 );
casti_m512i( d2,n ) = mm512_get_64( s, 4, 5, 12, 13, 20, 21, 28, 29 );
casti_m512i( d3,n ) = mm512_get_64( s, 6, 7, 14, 15, 22, 23, 30, 31 );
}
// AVX-512 user facing functions.
static inline void mm512_intrlv_16x32( void *d, const void *s00,
const void *s01, const void *s02, const void *s03, const void *s04,
const void *s05, const void *s06, const void *s07, const void *s08,
const void *s09, const void *s10, const void *s11, const void *s12,
const void *s13, const void *s14, const void *s15, int bit_len )
{
if ( bit_len <= 256 )
{
mm512_intrlv_16x32x256( d, s00, s01, s02, s03, s04, s05, s06, s07,
s08, s09, s10, s11, s12, s13, s14, s15 );
return;
}
mm512_intrlv_16x32x512( d, s00, s01, s02, s03, s04, s05, s06, s07,
s08, s09, s10, s11, s12, s13, s14, s15 );
if ( bit_len <= 512 ) return;
if ( bit_len <= 640 )
{
mm512_intrlv_16x32x128( d+1024, s00+64, s01+64, s02+64, s03+64,
s04+64, s05+64, s06+64, s07+64, s08+64, s09+64,
s10+64, s11+64, s12+64, s13+64, s14+64, s15+64 );
return;
}
mm512_intrlv_16x32x512( d+1024, s00+64, s01+64, s02+64, s03+64,
s04+64, s05+64, s06+64, s07+64, s08+64, s09+64,
s10+64, s11+64, s12+64, s13+64, s14+64, s15+64 );
// bit_len == 1024
}
// sub-functions can be called directly for 32 & 64 byte hash.
static inline void mm512_dintrlv_16x32( void *d00, void *d01, void *d02,
void *d03, void *d04, void *d05, void *d06, void *d07, void *d08,
void *d09, void *d10, void *d11, void *d12, void *d13, void *d14,
void *d15, const void *src, const int bit_len )
{
if ( bit_len <= 256 )
{
mm256_dintrlv_16x32x256( d00, d01, d02, d03, d04, d05, d06, d07,
d08, d09, d10, d11, d12, d13, d14, d15,
0,src );
return;
}
mm512_dintrlv_16x32x512( d00, d01, d02, d03, d04, d05, d06, d07,
d08, d09, d10, d11, d12, d13, d14, d15,
0, src );
if ( bit_len <= 512 ) return;
if ( bit_len <= 640 )
{
// short block, final 16 bytes of input data.
mm128_dintrlv_16x32x128( d00, d01, d02, d03, d04, d05, d06, d07,
d08, d09, d10, d11, d12, d13, d14, d15,
1, src+1024 );
return;
}
// bit_len == 1024
mm512_dintrlv_16x32x512( d00, d01, d02, d03, d04, d05, d06, d07,
d08, d09, d10, d11, d12, d13, d14, d15,
1, src+1024 );
}
static inline void mm512_extr_lane_16x32( void *dst, const void *src,
const int lane, const int bit_len )
{
if ( bit_len <= 256 )
{
cast_m256i( dst ) = mm256_get_32( src, lane, lane+16, lane+32, lane+48,
lane+64, lane+80, lane+96, lane+112 );
return;
}
cast_m512i( dst ) = mm512_get_32( src, lane, lane+ 16, lane+ 32, lane+ 48,
lane+ 64, lane+ 80, lane+ 96, lane+112, lane+128, lane+144,
lane+160, lane+176, lane+192, lane+208, lane+224, lane+248 );
}
//
static inline void mm512_intrlv_8x64( void *d, const void *s0,
const void *s1, const void *s2, const void *s3,
const void *s4, const void *s5, const void *s6,
const void *s7, int bit_len )
{
if ( bit_len <= 256 )
{
mm512_intrlv_8x64x256( d, s0, s1, s2, s3, s4, s5, s6, s7 );
return;
}
mm512_intrlv_8x64x512( d, s0, s1, s2, s3, s4, s5, s6, s7 );
if ( bit_len <= 512 ) return;
if ( bit_len <= 640 )
{
casti_m512i( d, 8 ) = mm512_put_64( s7+64, s6+64, s5+64, s4+64,
s3+64, s2+64, s1+64, s0+64 );
casti_m512i( d, 9 ) = mm512_put_64( s7+72, s6+72, s5+72, s4+72,
s3+72, s2+72, s1+72, s0+72 );
return;
}
// bitlen == 1024
mm512_intrlv_8x64x512( d+512, s0+64, s1+64, s2+64, s3+64,
s4+64, s5+64, s6+64, s7+64 );
}
static inline void mm512_dintrlv_8x64( void *d0, void *d1, void *d2,
void *d3, void *d4, void *d5, void *d6, void *d7,
const void *s, const int bit_len )
{
if ( bit_len <= 256 )
{
mm256_dintrlv_8x64x256( d0, d1, d2, d3, d4, d5, d6, d7, 0, s );
return;
}
mm512_dintrlv_8x64x512( d0, d1, d2, d3, d4, d5, d6, d7, 0, s );
if ( bit_len <= 512 ) return;
if ( bit_len <= 640 )
{
// short block, final 16 bytes of input data.
mm128_dintrlv_8x64x128( d0, d1, d2, d3, d4, d5, d6, d7, 1, s+512 );
return;
}
// bit_len == 1024
mm512_dintrlv_8x64x512( d0, d1, d2, d3, d4, d5, d6, d7, 1, s+512 );
}
// Extract one lane from 64 bit interleaved data
static inline void mm512_extr_lane_8x64( void *d, const void *s,
const int lane, const int bit_len )
{
if ( bit_len <= 256 )
{
cast_m256i( d ) = mm256_get_64( s, lane, lane+8, lane+16, lane+24 );
return;
}
// else bit_len == 512
cast_m512i( d ) = mm512_get_64( s, lane , lane+ 8, lane+16, lane+24,
lane+32, lane+40, lane+48, lane+56 );
}
//
static inline void mm512_intrlv_4x128( void *d, const void *s0,
const void *s1, const void *s2, const void *s3, const int bit_len )
{
casti_m512i( d, 0 ) = mm512_put_64( s0, s0+8, s1, s1+8,
s2, s2+8, s3, s3+8 );
casti_m512i( d, 1 ) = mm512_put_64( s0+16, s0+24, s1+16, s1+24,
s2+16, s2+24, s3+16, s3+24 );
if ( bit_len <= 256 ) return;
casti_m512i( d, 2 ) = mm512_put_64( s0+32, s0+40, s1+32, s1+40,
s2+32, s2+40, s3+32, s3+40 );
casti_m512i( d, 3 ) = mm512_put_64( s0+48, s0+56, s1+48, s1+56,
s2+48, s2+56, s3+48, s3+56 );
if ( bit_len <= 512 ) return;
casti_m512i( d, 4 ) = mm512_put_64( s0+64, s0+72, s1+64, s1+72,
s2+64, s2+72, s3+64, s3+72 );
if ( bit_len <= 640 ) return;
casti_m512i( d, 5 ) = mm512_put_64( s0+ 80, s0+ 88, s1+ 80, s1+ 88,
s2+ 80, s2+ 88, s3+ 80, s3+ 88 );
casti_m512i( d, 6 ) = mm512_put_64( s0+ 96, s0+104, s1+ 96, s1+104,
s2+ 96, s2+104, s3+ 96, s3+104 );
casti_m512i( d, 7 ) = mm512_put_64( s0+112, s0+120, s1+112, s1+120,
s2+112, s2+120, s3+112, s3+120 );
// bit_len == 1024
}
static inline void mm512_dintrlv_4x128( void *d0, void *d1, void *d2,
void *d3, const void *s, const int bit_len )
{
if ( bit_len <= 256 )
{
mm256_dintrlv_4x128x256( d0, d1, d2, d3, 0, s );
return;
}
mm512_dintrlv_4x128x512( d0, d1, d2, d3, 0, s );
if ( bit_len <= 512 ) return;
if ( bit_len <= 640 )
{
mm128_dintrlv_4x128x128( d0, d1, d2, d3, 1, s+256 );
return;
}
// bit_len == 1024
mm512_dintrlv_4x128x512( d0, d1, d2, d3, 1, s+256 );
}
// input one 8x64 buffer and return 2*4*128
static inline void mm512_rintrlv_8x64_4x128( void *dst0, void *dst1,
const void *src, int bit_len )
{
__m512i* d0 = (__m512i*)dst0;
__m512i* d1 = (__m512i*)dst1;
uint64_t *s = (uint64_t*)src;
d0[0] = _mm512_set_epi64( s[ 11], s[ 3], s[ 10], s[ 2],
s[ 9], s[ 1], s[ 8], s[ 0] );
d0[1] = _mm512_set_epi64( s[ 27], s[ 19], s[ 26], s[ 18],
s[ 25], s[ 17], s[ 24], s[ 16] );
d0[2] = _mm512_set_epi64( s[ 15], s[ 7], s[ 14], s[ 6],
s[ 13], s[ 5], s[ 12], s[ 4] );
d0[3] = _mm512_set_epi64( s[ 31], s[ 23], s[ 30], s[ 22],
s[ 29], s[ 21], s[ 28], s[ 20] );
d1[0] = _mm512_set_epi64( s[ 43], s[ 35], s[ 42], s[ 34],
s[ 41], s[ 33], s[ 40], s[ 32] );
d1[1] = _mm512_set_epi64( s[ 59], s[ 51], s[ 58], s[ 50],
s[ 57], s[ 49], s[ 56], s[ 48] );
d1[2] = _mm512_set_epi64( s[ 47], s[ 39], s[ 46], s[ 38],
s[ 45], s[ 37], s[ 44], s[ 36] );
d1[3] = _mm512_set_epi64( s[ 63], s[ 55], s[ 62], s[ 54],
s[ 61], s[ 53], s[ 60], s[ 52] );
if ( bit_len <= 512 ) return;
d0[4] = _mm512_set_epi64( s[ 75], s[ 67], s[ 74], s[ 66],
s[ 73], s[ 65], s[ 72], s[ 64] );
d0[5] = _mm512_set_epi64( s[ 91], s[ 83], s[ 90], s[ 82],
s[ 89], s[ 81], s[ 88], s[ 80] );
d0[6] = _mm512_set_epi64( s[ 79], s[ 71], s[ 78], s[ 70],
s[ 77], s[ 69], s[ 76], s[ 68] );
d0[7] = _mm512_set_epi64( s[ 95], s[ 87], s[ 94], s[ 86],
s[ 93], s[ 85], s[ 92], s[ 84] );
d1[4] = _mm512_set_epi64( s[107], s[ 99], s[106], s[ 98],
s[105], s[ 97], s[104], s[ 96] );
d1[5] = _mm512_set_epi64( s[123], s[115], s[122], s[114],
s[121], s[113], s[120], s[112] );
d1[6] = _mm512_set_epi64( s[111], s[103], s[110], s[102],
s[109], s[101], s[108], s[100] );
d1[7] = _mm512_set_epi64( s[127], s[119], s[126], s[118],
s[125], s[117], s[124], s[116] );
}
// input 2 4x128 return 8x64
static inline void mm512_rintrlv_4x128_8x64( void *dst, const void *src0,
const void *src1, int bit_len )
{
__m512i* d = (__m512i*)dst;
uint64_t *s0 = (uint64_t*)src0;
uint64_t *s1 = (uint64_t*)src1;
d[0] = _mm512_set_epi64( s1[ 6], s1[ 4], s1[ 2], s1[ 0],
s0[ 6], s0[ 4], s0[ 2], s0[ 0] );
d[1] = _mm512_set_epi64( s1[ 7], s1[ 5], s1[ 3], s1[ 1],
s0[ 7], s0[ 5], s0[ 3], s0[ 1] );
d[2] = _mm512_set_epi64( s1[14], s1[12], s1[10], s1[ 8],
s0[14], s0[12], s0[10], s0[ 8] );
d[3] = _mm512_set_epi64( s1[15], s1[13], s1[11], s1[ 9],
s0[15], s0[13], s0[11], s0[ 9] );
d[4] = _mm512_set_epi64( s1[22], s1[20], s1[18], s1[16],
s0[22], s0[20], s0[18], s0[16] );
d[5] = _mm512_set_epi64( s1[23], s1[21], s1[19], s1[17],
s0[24], s0[21], s0[19], s0[17] );
d[6] = _mm512_set_epi64( s1[22], s1[28], s1[26], s1[24],
s0[22], s0[28], s0[26], s0[24] );
d[7] = _mm512_set_epi64( s1[31], s1[29], s1[27], s1[25],
s0[31], s0[29], s0[27], s0[25] );
if ( bit_len <= 512 ) return;
d[0] = _mm512_set_epi64( s1[38], s1[36], s1[34], s1[32],
s0[38], s0[36], s0[34], s0[32] );
d[1] = _mm512_set_epi64( s1[39], s1[37], s1[35], s1[33],
s0[39], s0[37], s0[35], s0[33] );
d[2] = _mm512_set_epi64( s1[46], s1[44], s1[42], s1[40],
s0[46], s0[44], s0[42], s0[40] );
d[3] = _mm512_set_epi64( s1[47], s1[45], s1[43], s1[41],
s0[47], s0[45], s0[43], s0[41] );
d[4] = _mm512_set_epi64( s1[54], s1[52], s1[50], s1[48],
s0[54], s0[52], s0[50], s0[48] );
d[5] = _mm512_set_epi64( s1[55], s1[53], s1[51], s1[49],
s0[55], s0[53], s0[51], s0[49] );
d[6] = _mm512_set_epi64( s1[62], s1[60], s1[58], s1[56],
s0[62], s0[60], s0[58], s0[56] );
d[7] = _mm512_set_epi64( s1[63], s1[61], s1[59], s1[57],
s0[63], s0[61], s0[59], s0[57] );
}
static inline void mm512_extr_lane_4x128( void *d, const void *s,
const int lane, const int bit_len )
{
int l = lane<<1;
if ( bit_len <= 256 )
{
cast_m256i( d ) = mm256_get_64( s, l, l+1, l+8, l+9 );
return;
}
// else bit_len == 512
cast_m512i( d ) = mm512_get_64( s, l , l+ 1, l+ 8, l+ 9,
l+16, l+17, l+24, l+25 );
}
#endif // AVX512
#endif // INTRLV_AVX512_H__
Reference in New Issue View Git Blame Copy Permalink