popov/cpuminer-opt-gpu
1
0
Fork 0
mirror of https://github.com/JayDDee/cpuminer-opt.git synced 2025-09-17 23:44:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

v23.5

Browse Source
This commit is contained in:
Jay D Dee
2023-10-25 20:36:20 -04:00
parent 31c4dedf59
commit 160608cce5
180 changed files with 10318 additions and 13097 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
algo/sha/sha256-hash-4way.c
View File

@@ -173,7 +173,7 @@ void sha256_4way_prehash_3rounds( v128_t *state_mid, v128_t *X,
X[ 3] = v128_add32( v128_add32( SSG2_1( X[ 1] ), W[12] ), SSG2_0( W[ 4] ) );
X[ 4] = v128_add32( v128_add32( W[13], SSG2_0( W[ 5] ) ), W[ 4] );
X[ 5] = v128_add32( v128_add32( W[14], SSG2_0( W[ 6] ) ), W[ 5] );
X [6] = v128_add32( v128_add32( W[15], SSG2_0( W[ 7] ) ), W[ 6] );
X[ 6] = v128_add32( v128_add32( W[15], SSG2_0( W[ 7] ) ), W[ 6] );
X[ 7] = v128_add32( v128_add32( X[ 0], SSG2_0( W[ 8] ) ), W[ 7] );
X[ 8] = v128_add32( v128_add32( X[ 1], SSG2_0( W[ 9] ) ), W[ 8] );
X[ 9] = v128_add32( SSG2_0( W[10] ), W[ 9] );

21
algo/sha/sha256-hash.c
View File

@@ -1,5 +1,7 @@
#include "sha256-hash.h"
#if ( defined(__x86_64__) && defined(__SHA__) ) || defined(__ARM_NEON) && defined(__ARM_FEATURE_SHA2)
static const uint32_t SHA256_IV[8] =
{
0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A,
@@ -861,9 +863,7 @@ void sha256_ni2x_final_rounds( uint32_t *out_X, uint32_t *out_Y,
#endif // SHA
#if defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_SHA2)
#pragma message "NEON SHA2 for sha256"
#if defined(__ARM_NEON) && defined(__ARM_FEATURE_SHA2)
static const uint32_t K256[64] =
{
@@ -1010,7 +1010,7 @@ static const uint32_t K256[64] =
vst1q_u32( state_out+4, STATE1 ); \
}
void sha256_neon_transform_be( uint32_t *state_out, const void *input,
void sha256_neon_sha_transform_be( uint32_t *state_out, const void *input,
const uint32_t *state_in )
{
#define load_msg( m, i ) v128_bswap32( casti_v128( m, i ) );
@@ -1018,7 +1018,7 @@ void sha256_neon_transform_be( uint32_t *state_out, const void *input,
#undef load_msg
}
void sha256_neon_transform_le( uint32_t *state_out, const void *input,
void sha256_neon_sha_transform_le( uint32_t *state_out, const void *input,
const uint32_t *state_in )
{
#define load_msg( m, i ) casti_v128( m, i );
@@ -1026,7 +1026,7 @@ void sha256_neon_transform_le( uint32_t *state_out, const void *input,
#undef load_msg
}
#define sha256_neon2x_rounds( state_out_X, state_out_Y, input_X, \
#define sha256_neon_x2sha_rounds( state_out_X, state_out_Y, input_X, \
input_Y, state_in_X, state_in_Y ) \
{ \
uint32x4_t STATE0_X, STATE1_X, ABEF_SAVE_X, CDGH_SAVE_X; \
@@ -1255,21 +1255,21 @@ void sha256_neon_transform_le( uint32_t *state_out, const void *input,
vst1q_u32( state_out_Y+4, STATE1_Y ); \
}
void sha256_neon2x_transform_le( uint32_t *out_X, uint32_t*out_Y,
void sha256_neon_x2sha_transform_le( uint32_t *out_X, uint32_t*out_Y,
const void *msg_X, const void *msg_Y,
const uint32_t *in_X, const uint32_t *in_Y )
{
#define load_msg( m, i ) casti_v128( m, i )
sha256_neon2x_rounds( out_X, out_Y, msg_X, msg_Y, in_X, in_Y );
sha256_neon_x2sha_rounds( out_X, out_Y, msg_X, msg_Y, in_X, in_Y );
#undef load_msg
}
void sha256_neon2x_transform_be( uint32_t *out_X, uint32_t*out_Y,
void sha256_neon_x2sha_transform_be( uint32_t *out_X, uint32_t*out_Y,
const void *msg_X, const void *msg_Y,
const uint32_t *in_X, const uint32_t *in_Y )
{
#define load_msg( m, i ) v128_bswap32( casti_v128( m, i ) )
sha256_neon2x_rounds( out_X, out_Y, msg_X, msg_Y, in_X, in_Y );
sha256_neon_x2sha_rounds( out_X, out_Y, msg_X, msg_Y, in_X, in_Y );
#undef load_msg
}
@@ -1367,3 +1367,4 @@ void sha256_full( void *hash, const void *data, size_t len )
sha256_final( &ctx, hash );
}
#endif

187
algo/sha/sha256-hash.h
View File

@@ -27,74 +27,89 @@ void sha256_transform_be( uint32_t *state_out, const uint32_t *data,
#if defined(__x86_64__) && defined(__SHA__)
void sha256_opt_transform_le( uint32_t *state_out, const void *input,
const uint32_t *state_in );
void sha256_x86_sha_transform_le( uint32_t *state_out, const void *input,
const uint32_t *state_in );
void sha256_opt_transform_be( uint32_t *state_out, const void *input,
const uint32_t *state_in );
void sha256_x86_sha_transform_be( uint32_t *state_out, const void *input,
const uint32_t *state_in );
// 2 way serial with interleaved instructions
void sha256_ni2x_transform_le( uint32_t *out_X, uint32_t*out_Y,
void sha256_x86_x2sha_transform_le( uint32_t *out_X, uint32_t*out_Y,
const void *msg_X, const void *msg_Y,
const uint32_t *in_X, const uint32_t *in_Y );
void sha256_ni2x_transform_be( uint32_t *out_X, uint32_t*out_Y,
void sha256_x86_x2sha_transform_be( uint32_t *out_X, uint32_t*out_Y,
const void *msg_X, const void *msg_Y,
const uint32_t *in_X, const uint32_t *in_Y );
void sha256_ni_prehash_3rounds( uint32_t *ostate, const void *msg,
void sha256_x86_sha_prehash_3rounds( uint32_t *ostate, const void *msg,
uint32_t *sstate, const uint32_t *istate );
void sha256_ni2x_final_rounds( uint32_t *state_out_X, uint32_t *state_out_Y,
void sha256_x86_x2sha_final_rounds( uint32_t *state_out_X, uint32_t *state_out_Y,
const void *msg_X, const void *msg_Y,
const uint32_t *state_mid_X, const uint32_t *state_mid_Y,
const uint32_t *state_save_X, const uint32_t *state_save_Y );
#define sha256_transform_le sha256_opt_transform_le
#define sha256_transform_be sha256_opt_transform_be
#define sha256_2x_transform_le sha256_ni2x_transform_le
#define sha256_2x_transform_be sha256_ni2x_transform_be
#define sha256_prehash_3rounds sha256_ni_prehash_3rounds
#define sha256_2x_final_rounds sha256_ni2x_final_rounds
// Temporary during name transition
#define sha256_opt_transform_le sha256_x86_sha_transform_le
#define sha256_opt_transform_be sha256_x86_sha_transform_be
#define sha256_ni2x_transform_le sha256_x86_x2sha_transform_le
#define sha256_ni2x_transform_be sha256_x86_x2sha_transform_be
#define sha256_ni_prehash_3rounds sha256_x86_sha_prehash_3rounds
#define sha256_ni2x_final_rounds sha256_x86_x2sha_final_rounds
#elif defined(__aarch64__) && defined(__ARM_NEON)
// generic API
#define sha256_transform_le sha256_x86_sha_transform_le
#define sha256_transform_be sha256_x86_sha_transform_be
#define sha256_2x_transform_le sha256_x86_x2sha_transform_le
#define sha256_2x_transform_be sha256_x86_x2sha_transform_be
#define sha256_prehash_3rounds sha256_x86_sha_prehash_3rounds
#define sha256_2x_final_rounds sha256_x86_x2sha_final_rounds
void sha256_neon_transform_be( uint32_t *state_out, const void *input,
const uint32_t *state_in );
void sha256_neon_transform_le( uint32_t *state_out, const void *input,
const uint32_t *state_in );
#elif defined(__ARM_NEON) && defined(__ARM_FEATURE_SHA2)
void sha256_neon2x_transform_le( uint32_t *out_X, uint32_t*out_Y,
const void *msg_X, const void *msg_Y,
const uint32_t *in_X, const uint32_t *in_Y );
void sha256_neon_sha_transform_be( uint32_t *state_out, const void *input,
const uint32_t *state_in );
void sha256_neon_sha_transform_le( uint32_t *state_out, const void *input,
const uint32_t *state_in );
void sha256_neon2x_transform_be( uint32_t *out_X, uint32_t*out_Y,
const void *msg_X, const void *msg_Y,
const uint32_t *in_X, const uint32_t *in_Y );
void sha256_neon_x2sha_transform_le( uint32_t *out_X, uint32_t*out_Y,
const void *msg_X, const void *msg_Y,
const uint32_t *in_X, const uint32_t *in_Y );
void sha256_neon_prehash_3rounds( uint32_t *ostate, const void *msg,
uint32_t *sstate, const uint32_t *istate );
void sha256_neon_x2sha_transform_be( uint32_t *out_X, uint32_t*out_Y,
const void *msg_X, const void *msg_Y,
const uint32_t *in_X, const uint32_t *in_Y );
void sha256_neon2x_final_rounds( uint32_t *state_out_X, uint32_t *state_out_Y,
const void *msg_X, const void *msg_Y,
void sha256_neon_sha_prehash_3rounds( uint32_t *ostate, const void *msg,
uint32_t *sstate, const uint32_t *istate );
void sha256_neon_x2sha_final_rounds( uint32_t *state_out_X,
uint32_t *state_out_Y, const void *msg_X, const void *msg_Y,
const uint32_t *state_mid_X, const uint32_t *state_mid_Y,
const uint32_t *state_save_X, const uint32_t *state_save_Y );
#define sha256_transform_le sha256_neon_transform_le
#define sha256_transform_be sha256_neon_transform_be
#define sha256_2x_transform_le sha256_neon2x_transform_le
#define sha256_2x_transform_be sha256_neon2x_transform_be
#define sha256_prehash_3rounds sha256_neon_prehash_3rounds
#define sha256_2x_final_rounds sha256_neon2x_final_rounds
// Temporary during name transition
#define sha256_transform_le sha256_neon_sha_transform_le
#define sha256_transform_be sha256_neon_sha_transform_be
#define sha256_2x_transform_le sha256_neon_x2sha_transform_le
#define sha256_2x_transform_be sha256_neon_x2sha_transform_be
#define sha256_prehash_3rounds sha256_neon_sha_prehash_3rounds
#define sha256_2x_final_rounds sha256_neon_x2sha_final_rounds
#else
// without HW acceleration...
#include "sph_sha2.h"
#define sha256_full sph_sha256_full
#define sha256_ctx_init sph_sha256_init
#define sha256_update sph_sha256
#define sha256_final sph_sha256_close
#define sha256_transform_le sph_sha256_transform_le
#define sha256_transform_be sph_sha256_transform_be
#define sha256_prehash_3rounds sph_sha256_prehash_3rounds
#endif
#if defined(__AVX512F__) && defined(__AVX512VL__) && defined(__AVX512DQ__) && defined(__AVX512BW__)
@@ -106,24 +121,35 @@ typedef struct
__m512i buf[64>>2];
__m512i val[8];
uint32_t count_high, count_low;
} sha256_16way_context __attribute__ ((aligned (128)));
} sha256_16x32_context __attribute__ ((aligned (128)));
void sha256_16way_init( sha256_16way_context *sc );
void sha256_16way_update( sha256_16way_context *sc, const void *data, size_t len );
void sha256_16way_close( sha256_16way_context *sc, void *dst );
void sha256_16way_full( void *dst, const void *data, size_t len );
void sha256_16way_transform_le( __m512i *state_out, const __m512i *data,
void sha256_16x32_init( sha256_16x32_context *sc );
void sha256_16x32_update( sha256_16x32_context *sc, const void *data, size_t len );
void sha256_16x32_close( sha256_16x32_context *sc, void *dst );
void sha256_16x32_full( void *dst, const void *data, size_t len );
void sha256_16x32_transform_le( __m512i *state_out, const __m512i *data,
const __m512i *state_in );
void sha256_16way_transform_be( __m512i *state_out, const __m512i *data,
void sha256_16x32_transform_be( __m512i *state_out, const __m512i *data,
const __m512i *state_in );
void sha256_16way_prehash_3rounds( __m512i *state_mid, __m512i *X,
void sha256_16x32_prehash_3rounds( __m512i *state_mid, __m512i *X,
const __m512i *W, const __m512i *state_in );
void sha256_16way_final_rounds( __m512i *state_out, const __m512i *data,
void sha256_16x32_final_rounds( __m512i *state_out, const __m512i *data,
const __m512i *state_in, const __m512i *state_mid, const __m512i *X );
int sha256_16way_transform_le_short( __m512i *state_out, const __m512i *data,
int sha256_16x32_transform_le_short( __m512i *state_out, const __m512i *data,
const __m512i *state_in, const uint32_t *target );
#define sha256_16way_context sha256_16x32_context
#define sha256_16way_init sha256_16x32_init
#define sha256_16way_update sha256_16x32_update
#define sha256_16way_close sha256_16x32_close
#define sha256_16way_full sha256_16x32_full
#define sha256_16way_transform_le sha256_16x32_transform_le
#define sha256_16way_transform_be sha256_16x32_transform_be
#define sha256_16way_prehash_3rounds sha256_16x32_prehash_3rounds
#define sha256_16way_final_rounds sha256_16x32_final_rounds
#define sha256_16way_transform_le_short sha256_16x32_transform_le_short
#endif // AVX512
#if defined (__AVX2__)
@@ -135,24 +161,36 @@ typedef struct
__m256i buf[64>>2];
__m256i val[8];
uint32_t count_high, count_low;
} sha256_8way_context __attribute__ ((aligned (64)));
} sha256_8x32_context __attribute__ ((aligned (64)));
void sha256_8way_init( sha256_8way_context *sc );
void sha256_8way_update( sha256_8way_context *sc, const void *data, size_t len );
void sha256_8way_close( sha256_8way_context *sc, void *dst );
void sha256_8way_full( void *dst, const void *data, size_t len );
void sha256_8way_transform_le( __m256i *state_out, const __m256i *data,
void sha256_8x32_init( sha256_8x32_context *sc );
void sha256_8x32_update( sha256_8x32_context *sc, const void *data, size_t len );
void sha256_8x32_close( sha256_8x32_context *sc, void *dst );
void sha256_8x32_full( void *dst, const void *data, size_t len );
void sha256_8x32_transform_le( __m256i *state_out, const __m256i *data,
const __m256i *state_in );
void sha256_8way_transform_be( __m256i *state_out, const __m256i *data,
void sha256_8x32_transform_be( __m256i *state_out, const __m256i *data,
const __m256i *state_in );
void sha256_8way_prehash_3rounds( __m256i *state_mid, __m256i *X,
void sha256_8x32_prehash_3rounds( __m256i *state_mid, __m256i *X,
const __m256i *W, const __m256i *state_in );
void sha256_8way_final_rounds( __m256i *state_out, const __m256i *data,
void sha256_8x32_final_rounds( __m256i *state_out, const __m256i *data,
const __m256i *state_in, const __m256i *state_mid, const __m256i *X );
int sha256_8way_transform_le_short( __m256i *state_out, const __m256i *data,
int sha256_8x32_transform_le_short( __m256i *state_out, const __m256i *data,
const __m256i *state_in, const uint32_t *target );
// Temporary API during naming transition
#define sha256_8way_context sha256_8x32_context
#define sha256_8way_init sha256_8x32_init
#define sha256_8way_update sha256_8x32_update
#define sha256_8way_close sha256_8x32_close
#define sha256_8way_full sha256_8x32_full
#define sha256_8way_transform_le sha256_8x32_transform_le
#define sha256_8way_transform_be sha256_8x32_transform_be
#define sha256_8way_prehash_3rounds sha256_8x32_prehash_3rounds
#define sha256_8way_final_rounds sha256_8x32_final_rounds
#define sha256_8way_transform_le_short sha256_8x32_transform_le_short
#endif // AVX2
// SHA-256 4 way
@@ -162,23 +200,34 @@ typedef struct
v128_t buf[64>>2];
v128_t val[8];
uint32_t count_high, count_low;
} sha256_4way_context __attribute__ ((aligned (32)));
} sha256_4x32_context __attribute__ ((aligned (32)));
void sha256_4way_init( sha256_4way_context *sc );
void sha256_4way_update( sha256_4way_context *sc, const void *data,
void sha256_4x32_init( sha256_4x32_context *sc );
void sha256_4x32_update( sha256_4x32_context *sc, const void *data,
size_t len );
void sha256_4way_close( sha256_4way_context *sc, void *dst );
void sha256_4way_full( void *dst, const void *data, size_t len );
void sha256_4way_transform_le( v128_t *state_out, const v128_t *data,
const v128_t *state_in );
void sha256_4way_transform_be( v128_t *state_out, const v128_t *data,
const v128_t *state_in );
void sha256_4way_prehash_3rounds( v128_t *state_mid, v128_t *X,
const v128_t *W, const v128_t *state_in );
void sha256_4way_final_rounds( v128_t *state_out, const v128_t *data,
void sha256_4x32_close( sha256_4x32_context *sc, void *dst );
void sha256_4x32_full( void *dst, const void *data, size_t len );
void sha256_4x32_transform_le( v128_t *state_out, const v128_t *data,
const v128_t *state_in );
void sha256_4x32_transform_be( v128_t *state_out, const v128_t *data,
const v128_t *state_in );
void sha256_4x32_prehash_3rounds( v128_t *state_mid, v128_t *X,
const v128_t *W, const v128_t *state_in );
void sha256_4x32_final_rounds( v128_t *state_out, const v128_t *data,
const v128_t *state_in, const v128_t *state_mid, const v128_t *X );
int sha256_4way_transform_le_short( v128_t *state_out, const v128_t *data,
const v128_t *state_in, const uint32_t *target );
int sha256_4x32_transform_le_short( v128_t *state_out, const v128_t *data,
const v128_t *state_in, const uint32_t *target );
// Temporary API during naming transition
#define sha256_4way_context sha256_4x32_context
#define sha256_4way_init sha256_4x32_init
#define sha256_4way_update sha256_4x32_update
#define sha256_4way_close sha256_4x32_close
#define sha256_4way_full sha256_4x32_full
#define sha256_4way_transform_le sha256_4x32_transform_le
#define sha256_4way_transform_be sha256_4x32_transform_be
#define sha256_4way_prehash_3rounds sha256_4x32_prehash_3rounds
#define sha256_4way_final_rounds sha256_4x32_final_rounds
#define sha256_4way_transform_le_short sha256_4x32_transform_le_short
#endif

17
algo/sha/sha256d-4way.c
View File

@@ -33,7 +33,7 @@ int scanhash_sha256d_sha( struct work *work, uint32_t max_nonce,
const int thr_id = mythr->id;
const bool bench = opt_benchmark;
const v128_t shuf_bswap32 =
v128_set_64( 0x0c0d0e0f08090a0bULL, 0x0405060700010203ULL );
v128_set64( 0x0c0d0e0f08090a0bULL, 0x0405060700010203ULL );
// hash first 64 byte block of data
sha256_transform_le( mstatea, pdata, sha256_iv );
@@ -112,8 +112,7 @@ int scanhash_sha256d_neon_sha2( struct work *work, uint32_t max_nonce,
uint32_t block2b[16] __attribute__ ((aligned (64)));
uint32_t hasha[8] __attribute__ ((aligned (32)));
uint32_t hashb[8] __attribute__ ((aligned (32)));
uint32_t mstatea[8] __attribute__ ((aligned (32)));
uint32_t sstate[8] __attribute__ ((aligned (32)));
uint32_t mstate[8] __attribute__ ((aligned (32)));
uint32_t *pdata = work->data;
uint32_t *ptarget = work->target;
const uint32_t first_nonce = pdata[19];
@@ -121,11 +120,9 @@ int scanhash_sha256d_neon_sha2( struct work *work, uint32_t max_nonce,
uint32_t n = first_nonce;
const int thr_id = mythr->id;
const bool bench = opt_benchmark;
const v128_t shuf_bswap32 =
v128_set_64( 0x0c0d0e0f08090a0bULL, 0x0405060700010203ULL );
// hash first 64 byte block of data
sha256_transform_le( mstatea, pdata, sha256_iv );
sha256_transform_le( mstate, pdata, sha256_iv );
// fill & pad second bock without nonce
memcpy( block1a, pdata + 16, 12 );
@@ -149,10 +146,10 @@ int scanhash_sha256d_neon_sha2( struct work *work, uint32_t max_nonce,
// Insert nonce for second block
block1a[3] = n;
block1b[3] = n+1;
sha256_neon2x_transform_le( block2a, block2b, block1a, block1b,
mstatea, mstatea );
sha256_neon_x2sha_transform_le( block2a, block2b, block1a, block1b,
mstate, mstate );
sha256_neon2x_transform_le( hasha, hashb, block2a, block2b,
sha256_neon_x2sha_transform_le( hasha, hashb, block2a, block2b,
sha256_iv, sha256_iv );
if ( unlikely( bswap_32( hasha[7] ) <= ptarget[7] ) )
@@ -388,7 +385,7 @@ int scanhash_sha256d_4way( struct work *work, const uint32_t max_nonce,
for ( int i = 0; i < 19; i++ )
vdata[i] = v128_32( pdata[i] );
*noncev = v128_set_32( n+ 3, n+ 2, n+1, n );
*noncev = v128_set32( n+ 3, n+ 2, n+1, n );
vdata[16+4] = last_byte;
v128_memset_zero( vdata+16 + 5, 10 );

257
algo/sha/sha256dt.c
View File

@@ -4,28 +4,30 @@
#include <string.h>
#include <stdio.h>
#include "sha256-hash.h"
#include "sph_sha2.h"
#if defined(__AVX512F__) && defined(__AVX512VL__) && defined(__AVX512DQ__) && defined(__AVX512BW__)
#define SHA256DT_16WAY 1
#elif defined(__SHA__)
#define SHA256DT_SHA 1
#define SHA256DT_16X64 1
#elif defined(__x86_64__) && defined(__SHA__)
#define SHA256DT_X86_SHA256 1
#elif defined(__ARM_NEON) && defined(__ARM_FEATURE_SHA2)
#define SHA256DT_NEON_SHA2 1
#define SHA256DT_NEON_SHA256 1
#elif defined(__AVX2__)
#define SHA256DT_8WAY 1
#else
#define SHA256DT_4WAY 1
#define SHA256DT_8X64 1
#elif defined (__SSE2__) || defined(__ARM_NEON)
#define SHA256DT_4X64 1
#endif
// else ref, should never happen
static const uint32_t sha256dt_iv[8] __attribute__ ((aligned (32))) =
{
0xdfa9bf2c, 0xb72074d4, 0x6bb01122, 0xd338e869,
0xaa3ff126, 0x475bbf30, 0x8fd52e5b, 0x9f75c9ad
};
{
0xdfa9bf2c, 0xb72074d4, 0x6bb01122, 0xd338e869,
0xaa3ff126, 0x475bbf30, 0x8fd52e5b, 0x9f75c9ad
};
#if defined(SHA256DT_SHA)
#if defined(SHA256DT_X86_SHA256)
int scanhash_sha256dt_sha( struct work *work, uint32_t max_nonce,
int scanhash_sha256dt_x86_x2sha( struct work *work, uint32_t max_nonce,
uint64_t *hashes_done, struct thr_info *mythr )
{
uint32_t block1a[16] __attribute__ ((aligned (64)));
@@ -44,8 +46,6 @@ int scanhash_sha256dt_sha( struct work *work, uint32_t max_nonce,
uint32_t n = first_nonce;
const int thr_id = mythr->id;
const bool bench = opt_benchmark;
const v128_t shuf_bswap32 =
v128_set64( 0x0c0d0e0f08090a0bULL, 0x0405060700010203ULL );
// hash first 64 byte block of data
sha256_transform_le( mstatea, pdata, sha256dt_iv );
@@ -106,13 +106,9 @@ int scanhash_sha256dt_sha( struct work *work, uint32_t max_nonce,
return 0;
}
#endif
#elif defined(SHA256DT_NEON_SHA256)
#if defined(SHA256DT_NEON_SHA2)
#pragma message "SHA256DT MEON SHA"
int scanhash_sha256dt_neon_sha2( struct work *work, uint32_t max_nonce,
int scanhash_sha256dt_neon_x2sha( struct work *work, uint32_t max_nonce,
uint64_t *hashes_done, struct thr_info *mythr )
{
uint32_t block1a[16] __attribute__ ((aligned (64)));
@@ -121,8 +117,7 @@ int scanhash_sha256dt_neon_sha2( struct work *work, uint32_t max_nonce,
uint32_t block2b[16] __attribute__ ((aligned (64)));
uint32_t hasha[8] __attribute__ ((aligned (32)));
uint32_t hashb[8] __attribute__ ((aligned (32)));
uint32_t mstatea[8] __attribute__ ((aligned (32)));
uint32_t sstate[8] __attribute__ ((aligned (32)));
uint32_t mstate[8] __attribute__ ((aligned (32)));
uint32_t *pdata = work->data;
uint32_t *ptarget = work->target;
const uint32_t first_nonce = pdata[19];
@@ -130,11 +125,9 @@ int scanhash_sha256dt_neon_sha2( struct work *work, uint32_t max_nonce,
uint32_t n = first_nonce;
const int thr_id = mythr->id;
const bool bench = opt_benchmark;
const v128_t shuf_bswap32 =
v128_set64( 0x0c0d0e0f08090a0bULL, 0x0405060700010203ULL );
// hash first 64 byte block of data
sha256_neon_transform_le( mstatea, pdata, sha256dt_iv );
sha256_neon_sha_transform_le( mstate, pdata, sha256dt_iv );
// fill & pad second bock without nonce
memcpy( block1a, pdata + 16, 12 );
@@ -156,10 +149,10 @@ int scanhash_sha256dt_neon_sha2( struct work *work, uint32_t max_nonce,
// Insert nonce for second block
block1a[3] = n;
block1b[3] = n+1;
sha256_neon2x_transform_le( block2a, block2b, block1a, block1b,
mstatea, mstatea );
sha256_neon_x2sha_transform_le( block2a, block2b, block1a, block1b,
mstate, mstate );
sha256_neon2x_transform_le( hasha, hashb, block2a, block2b,
sha256_neon_x2sha_transform_le( hasha, hashb, block2a, block2b,
sha256dt_iv, sha256dt_iv );
if ( unlikely( bswap_32( hasha[7] ) <= ptarget[7] ) )
@@ -190,11 +183,9 @@ int scanhash_sha256dt_neon_sha2( struct work *work, uint32_t max_nonce,
return 0;
}
#endif
#elif defined(SHA256DT_16X64)
#if defined(SHA256DT_16WAY)
int scanhash_sha256dt_16way( struct work *work, const uint32_t max_nonce,
int scanhash_sha256dt_16x64( struct work *work, const uint32_t max_nonce,
uint64_t *hashes_done, struct thr_info *mythr )
{
__m512i block[16] __attribute__ ((aligned (128)));
@@ -241,7 +232,7 @@ int scanhash_sha256dt_16way( struct work *work, const uint32_t max_nonce,
buf[15] = v512_32( 0x480 ); // sha256dt funky bit count
// partially pre-expand & prehash second message block, avoiding the nonces
sha256_16way_prehash_3rounds( mstate2, mexp_pre, buf, mstate1 );
sha256_16x32_prehash_3rounds( mstate2, mexp_pre, buf, mstate1 );
// vectorize IV for second hash
istate[0] = v512_32( sha256dt_iv[0] );
@@ -260,8 +251,8 @@ int scanhash_sha256dt_16way( struct work *work, const uint32_t max_nonce,
do
{
sha256_16way_final_rounds( block, buf, mstate1, mstate2, mexp_pre );
if ( unlikely( sha256_16way_transform_le_short(
sha256_16x32_final_rounds( block, buf, mstate1, mstate2, mexp_pre );
if ( unlikely( sha256_16x32_transform_le_short(
hash32, block, istate, ptarget ) ) )
{
for ( int lane = 0; lane < 16; lane++ )
@@ -284,11 +275,9 @@ int scanhash_sha256dt_16way( struct work *work, const uint32_t max_nonce,
return 0;
}
#endif
#elif defined(SHA256DT_8X64)
#if defined(SHA256DT_8WAY)
int scanhash_sha256dt_8way( struct work *work, const uint32_t max_nonce,
int scanhash_sha256dt_8x64( struct work *work, const uint32_t max_nonce,
uint64_t *hashes_done, struct thr_info *mythr )
{
__m256i vdata[32] __attribute__ ((aligned (64)));
@@ -335,15 +324,15 @@ int scanhash_sha256dt_8way( struct work *work, const uint32_t max_nonce,
istate[6] = v256_32( sha256dt_iv[6] );
istate[7] = v256_32( sha256dt_iv[7] );
sha256_8way_transform_le( mstate1, vdata, istate );
sha256_8x32_transform_le( mstate1, vdata, istate );
// Do 3 rounds on the first 12 bytes of the next block
sha256_8way_prehash_3rounds( mstate2, mexp_pre, vdata + 16, mstate1 );
sha256_8x32_prehash_3rounds( mstate2, mexp_pre, vdata + 16, mstate1 );
do
{
sha256_8way_final_rounds( block, vdata+16, mstate1, mstate2, mexp_pre );
if ( unlikely( sha256_8way_transform_le_short( hash32, block,
sha256_8x32_final_rounds( block, vdata+16, mstate1, mstate2, mexp_pre );
if ( unlikely( sha256_8x32_transform_le_short( hash32, block,
istate, ptarget ) ) )
{
for ( int lane = 0; lane < 8; lane++ )
@@ -366,19 +355,17 @@ int scanhash_sha256dt_8way( struct work *work, const uint32_t max_nonce,
return 0;
}
#endif
#elif defined(SHA256DT_4X64)
#if defined(SHA256DT_4WAY)
int scanhash_sha256dt_4way( struct work *work, const uint32_t max_nonce,
int scanhash_sha256dt_4x64( struct work *work, const uint32_t max_nonce,
uint64_t *hashes_done, struct thr_info *mythr )
{
v128_t vdata[32] __attribute__ ((aligned (64)));
v128_t block[16] __attribute__ ((aligned (32)));
v128_t hash32[8] __attribute__ ((aligned (32)));
v128_t initstate[8] __attribute__ ((aligned (32)));
v128_t midstate[8] __attribute__ ((aligned (32)));
uint32_t lane_hash[8] __attribute__ ((aligned (32)));
v128_t iv[8] __attribute__ ((aligned (32)));
v128_t mhash[8] __attribute__ ((aligned (32)));
uint32_t lhash[8] __attribute__ ((aligned (32)));
uint32_t *hash32_d7 = (uint32_t*)&( hash32[7] );
uint32_t *pdata = work->data;
const uint32_t *ptarget = work->target;
@@ -406,33 +393,83 @@ int scanhash_sha256dt_4way( struct work *work, const uint32_t max_nonce,
block[15] = v128_32( 0x300 );
// initialize state
initstate[0] = v128_32( sha256dt_iv[0] );
initstate[1] = v128_32( sha256dt_iv[1] );
initstate[2] = v128_32( sha256dt_iv[2] );
initstate[3] = v128_32( sha256dt_iv[3] );
initstate[4] = v128_32( sha256dt_iv[4] );
initstate[5] = v128_32( sha256dt_iv[5] );
initstate[6] = v128_32( sha256dt_iv[6] );
initstate[7] = v128_32( sha256dt_iv[7] );
iv[0] = v128_32( sha256dt_iv[0] );
iv[1] = v128_32( sha256dt_iv[1] );
iv[2] = v128_32( sha256dt_iv[2] );
iv[3] = v128_32( sha256dt_iv[3] );
iv[4] = v128_32( sha256dt_iv[4] );
iv[5] = v128_32( sha256dt_iv[5] );
iv[6] = v128_32( sha256dt_iv[6] );
iv[7] = v128_32( sha256dt_iv[7] );
// hash first 64 bytes of data
sha256_4way_transform_le( midstate, vdata, initstate );
sha256_4x32_transform_le( mhash, vdata, iv );
/*
uint32_t m1 [8] __attribute__ ((aligned (32)));
uint32_t h1 [8] __attribute__ ((aligned (32)));
uint32_t b1 [16] __attribute__ ((aligned (32)));
uint32_t e16 [16] __attribute__ ((aligned (32)));
uint32_t *m4 = (uint32_t*)&midstate;
uint32_t *h4 = (uint32_t*)hash32;
sha256_transform_le( m1, pdata, sha256dt_iv );
memcpy( e16, pdata + 16, 12 );
e16[3] = n;
e16[4] = 0x80000000;
memset( &e16[5], 0, 40 );
e16[15] = 0x480; // funky bit count
b1[8] = 0x80000000;
memset( &b1[9], 0, 24 );
b1[9] = b1[10] = b1[11] = b1[12] = b1[13] = b1[14] = 0;
b1[15] = 0x300; // bit count
*/
do
{
sha256_4way_transform_le( block, vdata+16, midstate );
sha256_4way_transform_le( hash32, block, initstate );
sha256_4x32_transform_le( block, vdata+16, mhash );
v128_block_bswap32( hash32, hash32 );
//sha256_transform_le( b1, e16, m1 );
sha256_4x32_transform_le( hash32, block, iv );
/*
sha256_transform_le( h1, b1, sha256dt_iv );
printf("final hash1: %08x %08x %08x %08x %08x %08x %08x %08x\n",
h1[0],h1[1],h1[2],h1[3],h1[4],h1[5],h1[6],h1[7]);
printf("final hash4: %08x %08x %08x %08x %08x %08x %08x %08x\n",
h4[0],h4[4],h4[8],h4[12],h4[16],h4[20],h4[24],h4[28]);
casti_v128( h1,0 ) = v128_bswap32( casti_v128( h1,0 ) );
casti_v128( h1,1 ) = v128_bswap32( casti_v128( h1,1 ) );
*/
// v128_block_bswap32( hash32, hash32 );
/*
printf("bswap hash1: %08x %08x %08x %08x %08x %08x %08x %08x\n",
h1[0],h1[1],h1[2],h1[3],h1[4],h1[5],h1[6],h1[7]);
printf("bswap hash4: %08x %08x %08x %08x %08x %08x %08x %08x\n",
h4[0],h4[4],h4[8],h4[12],h4[16],h4[20],h4[24],h4[28]);
exit(0);
*/
for ( int lane = 0; lane < 4; lane++ )
if ( unlikely( hash32_d7[ lane ] <= targ32_d7 ) )
{
extr_lane_4x32( lane_hash, hash32, lane, 256 );
if ( likely( valid_hash( lane_hash, ptarget ) && !bench ) )
if ( unlikely( bswap_32( hash32_d7[ lane ] ) <= targ32_d7 ) )
{
pdata[19] = n + lane;
submit_solution( work, lane_hash, mythr );
extr_lane_4x32( lhash, hash32, lane, 256 );
casti_v128( lhash, 0 ) = v128_bswap32( casti_v128( lhash, 0 ) );
casti_v128( lhash, 1 ) = v128_bswap32( casti_v128( lhash, 1 ) );
if ( likely( valid_hash( lhash, ptarget ) && !bench ) )
{
pdata[19] = n + lane;
submit_solution( work, lhash, mythr );
}
}
}
*noncev = v128_add32( *noncev, four );
@@ -443,26 +480,84 @@ int scanhash_sha256dt_4way( struct work *work, const uint32_t max_nonce,
return 0;
}
#else
int scanhash_sha256dt_ref( struct work *work, uint32_t max_nonce,
uint64_t *hashes_done, struct thr_info *mythr )
{
uint32_t block1a[16] __attribute__ ((aligned (32)));
uint32_t block2a[16] __attribute__ ((aligned (32)));
uint32_t hasha[8] __attribute__ ((aligned (32)));
uint32_t mstate[8] __attribute__ ((aligned (32)));
uint32_t *pdata = work->data;
uint32_t *ptarget = work->target;
const uint32_t first_nonce = pdata[19];
const uint32_t last_nonce = max_nonce - 2;
uint32_t n = first_nonce;
const int thr_id = mythr->id;
const bool bench = opt_benchmark;
// hash first 64 byte block of data
sha256_transform_le( mstate, pdata, sha256dt_iv );
// fill & pad second bock without nonce
memcpy( block1a, pdata + 16, 12 );
block1a[ 3] = 0;
block1a[ 4] = 0x80000000;
memset( block1a + 5, 0, 40 );
block1a[15] = 0x480; // funky bit count
// Pad third block
block2a[ 8] = 0x80000000;
memset( block2a + 9, 0, 24 );
block2a[15] = 0x300; // bit count
do
{
// Insert nonce for second block
block1a[3] = n;
sha256_transform_le( block2a, block1a, mstate );
sha256_transform_le( hasha, block2a, sha256dt_iv );
if ( unlikely( bswap_32( hasha[7] ) <= ptarget[7] ) )
{
casti_v128( hasha, 0 ) = v128_bswap32( casti_v128( hasha, 0 ) );
casti_v128( hasha, 1 ) = v128_bswap32( casti_v128( hasha, 1 ) );
if ( likely( valid_hash( hasha, ptarget ) && !bench ) )
{
pdata[19] = n;
submit_solution( work, hasha, mythr );
}
}
n += 1;
} while ( (n < last_nonce) && !work_restart[thr_id].restart );
pdata[19] = n;
*hashes_done = n - first_nonce;
return 0;
}
#endif
bool register_sha256dt_algo( algo_gate_t* gate )
{
gate->optimizations = SSE2_OPT | AVX2_OPT | AVX512_OPT;
#if defined(SHA256DT_16WAY)
gate->scanhash = (void*)&scanhash_sha256dt_16way;
#elif defined(SHA256DT_SHA)
gate->optimizations = SSE2_OPT | AVX2_OPT | AVX512_OPT | NEON_OPT;
#if defined(SHA256DT_16X64)
gate->scanhash = (void*)&scanhash_sha256dt_16x64;
#elif defined(SHA256DT_X86_SHA256)
gate->optimizations = SHA_OPT;
gate->scanhash = (void*)&scanhash_sha256dt_sha;
#elif defined(SHA256DT_NEON_SHA2)
gate->scanhash = (void*)&scanhash_sha256dt_x86_x2sha;
#elif defined(SHA256DT_NEON_SHA256)
gate->optimizations = SHA_OPT;
gate->scanhash = (void*)&scanhash_sha256dt_neon_sha2;
#elif defined(SHA256DT_8WAY)
gate->scanhash = (void*)&scanhash_sha256dt_8way;
#elif defined(SHA256DT_4WAY)
gate->scanhash = (void*)&scanhash_sha256dt_4way;
gate->scanhash = (void*)&scanhash_sha256dt_neon_x2sha;
#elif defined(SHA256DT_8X64)
gate->scanhash = (void*)&scanhash_sha256dt_8x64;
#elif defined(SHA256DT_4X64)
gate->scanhash = (void*)&scanhash_sha256dt_4x64;
#else
gate->scanhash = (void*)&scanhash_sha256dt_ref;
#endif
return true;
}

17
algo/sha/sha256t-4way.c
View File

@@ -35,7 +35,7 @@ int scanhash_sha256t_16way( struct work *work, const uint32_t max_nonce,
const int thr_id = mythr->id;
const __m512i sixteen = v512_32( 16 );
const bool bench = opt_benchmark;
const __m256i bswap_shuf = mm256_bcast_m128( _mm_set_epi64x(
const __m256i bswap_shuf = mm256_bcast_m128( v128_set64(
0x0c0d0e0f08090a0b, 0x0405060700010203 ) );
// prehash first block directly from pdata
@@ -208,8 +208,7 @@ int scanhash_sha256t_neon_sha2( struct work *work, uint32_t max_nonce,
uint32_t block2b[16] __attribute__ ((aligned (64)));
uint32_t hasha[8] __attribute__ ((aligned (32)));
uint32_t hashb[8] __attribute__ ((aligned (32)));
uint32_t mstatea[8] __attribute__ ((aligned (32)));
uint32_t sstate[8] __attribute__ ((aligned (32)));
uint32_t mstate[8] __attribute__ ((aligned (32)));
uint32_t *pdata = work->data;
uint32_t *ptarget = work->target;
const uint32_t first_nonce = pdata[19];
@@ -219,7 +218,7 @@ int scanhash_sha256t_neon_sha2( struct work *work, uint32_t max_nonce,
const bool bench = opt_benchmark;
// hash first 64 byte block of data
sha256_transform_le( mstatea, pdata, sha256_iv );
sha256_transform_le( mstate, pdata, sha256_iv );
// fill & pad second bock without nonce
memcpy( block1a, pdata + 16, 12 );
@@ -242,11 +241,11 @@ int scanhash_sha256t_neon_sha2( struct work *work, uint32_t max_nonce,
// Insert nonce for second block
block1a[3] = n;
block1b[3] = n+1;
sha256_neon2x_transform_le( block2a, block2b, block1a, block1b,
mstatea, mstatea );
sha256_neon2x_transform_le( block2a, block2b, block2a, block2b,
sha256_neon_x2sha_transform_le( block2a, block2b, block1a, block1b,
mstate, mstate );
sha256_neon_x2sha_transform_le( block2a, block2b, block2a, block2b,
sha256_iv, sha256_iv );
sha256_neon2x_transform_le( hasha, hashb, block2a, block2b,
sha256_neon_x2sha_transform_le( hasha, hashb, block2a, block2b,
sha256_iv, sha256_iv );
if ( unlikely( bswap_32( hasha[7] ) <= ptarget[7] ) )
@@ -397,7 +396,7 @@ int scanhash_sha256t_4way( struct work *work, const uint32_t max_nonce,
for ( int i = 0; i < 19; i++ )
vdata[i] = v128_32( pdata[i] );
*noncev = v128_set_32( n+ 3, n+ 2, n+1, n );
*noncev = v128_set32( n+ 3, n+ 2, n+1, n );
vdata[16+4] = last_byte;
v128_memset_zero( vdata+16 + 5, 10 );

2
algo/sha/sha256t-gate.c
View File

@@ -8,6 +8,8 @@ bool register_sha256t_algo( algo_gate_t* gate )
#elif defined(SHA256T_SHA)
gate->optimizations = SHA_OPT;
gate->scanhash = (void*)&scanhash_sha256t_sha;
#elif defined(SHA256T_NEON_SHA2)
gate->scanhash = (void*)&scanhash_sha256t_neon_sha2;
#elif defined(SHA256T_8WAY)
gate->scanhash = (void*)&scanhash_sha256t_8way;
#elif defined(SHA256T_4WAY)

2
algo/sha/sha256t-gate.h
View File

@@ -9,7 +9,7 @@
#elif defined(__SHA__)
#define SHA256T_SHA 1
#elif defined(__ARM_NEON) && defined(__ARM_FEATURE_SHA2)
#define SHA125DT_NEON_SHA2 1
#define SHA256T_NEON_SHA2 1
#elif defined(__AVX2__)
#define SHA256T_8WAY 1
#else

239
algo/sha/sha512-hash-4way.c
View File

@@ -30,7 +30,6 @@
* @author Thomas Pornin <thomas.pornin@cryptolog.com>
*/
#if defined(__AVX2__)
#include <stddef.h>
#include <string.h>
@@ -501,7 +500,7 @@ do { \
} while (0)
static void
sha512_8way_round( sha512_8way_context *ctx, __m512i *in, __m512i r[8] )
sha512_8x64_round( sha512_8x64_context *ctx, __m512i *in, __m512i r[8] )
{
int i;
register __m512i A, B, C, D, E, F, G, H;
@@ -574,13 +573,13 @@ sha512_8way_round( sha512_8way_context *ctx, __m512i *in, __m512i r[8] )
}
}
void sha512_8way_init( sha512_8way_context *sc )
void sha512_8x64_init( sha512_8x64_context *sc )
{
sc->initialized = false;
sc->count = 0;
}
void sha512_8way_update( sha512_8way_context *sc, const void *data, size_t len )
void sha512_8x64_update( sha512_8x64_context *sc, const void *data, size_t len )
{
__m512i *vdata = (__m512i*)data;
size_t ptr;
@@ -599,14 +598,14 @@ void sha512_8way_update( sha512_8way_context *sc, const void *data, size_t len )
len -= clen;
if ( ptr == buf_size )
{
sha512_8way_round( sc, sc->buf, sc->val );
sha512_8x64_round( sc, sc->buf, sc->val );
ptr = 0;
}
sc->count += clen;
}
}
void sha512_8way_close( sha512_8way_context *sc, void *dst )
void sha512_8x64_close( sha512_8x64_context *sc, void *dst )
{
unsigned ptr;
const int buf_size = 128;
@@ -620,7 +619,7 @@ void sha512_8way_close( sha512_8way_context *sc, void *dst )
if ( ptr > pad )
{
memset_zero_512( sc->buf + (ptr>>3), (buf_size - ptr) >> 3 );
sha512_8way_round( sc, sc->buf, sc->val );
sha512_8x64_round( sc, sc->buf, sc->val );
memset_zero_512( sc->buf, pad >> 3 );
}
else
@@ -630,14 +629,23 @@ void sha512_8way_close( sha512_8way_context *sc, void *dst )
v512_64( sc->count >> 61 ), shuff_bswap64 );
sc->buf[ ( pad+8 ) >> 3 ] = _mm512_shuffle_epi8(
v512_64( sc->count << 3 ), shuff_bswap64 );
sha512_8way_round( sc, sc->buf, sc->val );
sha512_8x64_round( sc, sc->buf, sc->val );
mm512_block_bswap_64( dst, sc->val );
}
void sha512_8x64_ctx( sha512_8x64_context *sc, void *dst, const void *data,
size_t len )
{
sha512_8x64_init( sc);
sha512_8x64_update( sc, data,len );
sha512_8x64_close( sc, dst );
}
#endif // AVX512
#if defined(__AVX2__)
// SHA-512 4 way 64 bit
#define BSG5_0( x ) mm256_xor3( mm256_ror_64( x, 28 ), \
@@ -704,7 +712,7 @@ do { \
#endif // AVX512VL AVX10_256
static void
sha512_4way_round( sha512_4way_context *ctx, __m256i *in, __m256i r[8] )
sha512_4x64_round( sha512_4x64_context *ctx, __m256i *in, __m256i r[8] )
{
int i;
register __m256i A, B, C, D, E, F, G, H;
@@ -788,13 +796,13 @@ sha512_4way_round( sha512_4way_context *ctx, __m256i *in, __m256i r[8] )
}
}
void sha512_4way_init( sha512_4way_context *sc )
void sha512_4x64_init( sha512_4x64_context *sc )
{
sc->initialized = false;
sc->count = 0;
}
void sha512_4way_update( sha512_4way_context *sc, const void *data, size_t len )
void sha512_4x64_update( sha512_4x64_context *sc, const void *data, size_t len )
{
__m256i *vdata = (__m256i*)data;
size_t ptr;
@@ -813,14 +821,14 @@ void sha512_4way_update( sha512_4way_context *sc, const void *data, size_t len )
len -= clen;
if ( ptr == buf_size )
{
sha512_4way_round( sc, sc->buf, sc->val );
sha512_4x64_round( sc, sc->buf, sc->val );
ptr = 0;
}
sc->count += clen;
}
}
void sha512_4way_close( sha512_4way_context *sc, void *dst )
void sha512_4x64_close( sha512_4x64_context *sc, void *dst )
{
unsigned ptr;
const int buf_size = 128;
@@ -834,7 +842,7 @@ void sha512_4way_close( sha512_4way_context *sc, void *dst )
if ( ptr > pad )
{
memset_zero_256( sc->buf + (ptr>>3), (buf_size - ptr) >> 3 );
sha512_4way_round( sc, sc->buf, sc->val );
sha512_4x64_round( sc, sc->buf, sc->val );
memset_zero_256( sc->buf, pad >> 3 );
}
else
@@ -844,9 +852,210 @@ void sha512_4way_close( sha512_4way_context *sc, void *dst )
v256_64( sc->count >> 61 ), shuff_bswap64 );
sc->buf[ ( pad+8 ) >> 3 ] = _mm256_shuffle_epi8(
v256_64( sc->count << 3 ), shuff_bswap64 );
sha512_4way_round( sc, sc->buf, sc->val );
sha512_4x64_round( sc, sc->buf, sc->val );
mm256_block_bswap_64( dst, sc->val );
}
void sha512_4x64_ctx( sha512_4x64_context *sc, void *dst, const void *data,
size_t len )
{
sha512_4x64_init( sc);
sha512_4x64_update( sc, data,len );
sha512_4x64_close( sc, dst );
}
#endif // __AVX2__
// SHA512 2 way 64 SSE2 or NEON
#define BSG5_0_2x64( x ) v128_xor3( v128_ror64( x, 28 ), \
v128_ror64( x, 34 ), \
v128_ror64( x, 39 ) )
#define BSG5_1_2x64( x ) v128_xor3( v128_ror64( x, 14 ), \
v128_ror64( x, 18 ), \
v128_ror64( x, 41 ) )
#define SSG5_0_2x64( x ) v128_xor3( v128_ror64( x, 1 ), \
v128_ror64( x, 8 ), \
v128_sr64( x, 7 ) )
#define SSG5_1_2x64( x ) v128_xor3( v128_ror64( x, 19 ), \
v128_ror64( x, 61 ), \
v128_sr64( x, 6 ) )
#define CH_2x64(X, Y, Z) \
v128_xor( v128_and( v128_xor( Y, Z ), X ), Z )
#define MAJ_2x64(X, Y, Z) \
v128_xor( Y, v128_and( X_xor_Y = v128_xor( X, Y ), Y_xor_Z ) )
#define SHA3_2x64_STEP( A, B, C, D, E, F, G, H, i ) \
do { \
v128u64_t T0 = v128_add64( v128_64( K512[i] ), W[i] ); \
v128u64_t T1 = BSG5_1_2x64( E ); \
v128u64_t T2 = BSG5_0_2x64( A ); \
T0 = v128_add64( T0, CH_2x64( E, F, G ) ); \
T1 = v128_add64( T1, H ); \
T2 = v128_add64( T2, MAJ_2x64( A, B, C ) ); \
T1 = v128_add64( T1, T0 ); \
Y_xor_Z = X_xor_Y; \
D = v128_add64( D, T1 ); \
H = v128_add64( T1, T2 ); \
} while (0)
static void
sha512_2x64_round( sha512_2x64_context *ctx, v128u64_t *in, v128u64_t r[8] )
{
int i;
register v128u64_t A, B, C, D, E, F, G, H;
v128u64_t X_xor_Y, Y_xor_Z;
v128u64_t W[80];
v128_block_bswap64( W , in );
v128_block_bswap64( (&W[8]), (&in[8]) );
for ( i = 16; i < 80; i++ )
W[i] = v128_add4_64( SSG5_0_2x64( W[i-15] ), SSG5_1_2x64( W[i-2] ),
W[ i- 7 ], W[ i-16 ] );
if ( ctx->initialized )
{
A = r[0];
B = r[1];
C = r[2];
D = r[3];
E = r[4];
F = r[5];
G = r[6];
H = r[7];
}
else
{
A = v128_64( 0x6A09E667F3BCC908 );
B = v128_64( 0xBB67AE8584CAA73B );
C = v128_64( 0x3C6EF372FE94F82B );
D = v128_64( 0xA54FF53A5F1D36F1 );
E = v128_64( 0x510E527FADE682D1 );
F = v128_64( 0x9B05688C2B3E6C1F );
G = v128_64( 0x1F83D9ABFB41BD6B );
H = v128_64( 0x5BE0CD19137E2179 );
}
Y_xor_Z = v128_xor( B, C );
for ( i = 0; i < 80; i += 8 )
{
SHA3_2x64_STEP( A, B, C, D, E, F, G, H, i + 0 );
SHA3_2x64_STEP( H, A, B, C, D, E, F, G, i + 1 );
SHA3_2x64_STEP( G, H, A, B, C, D, E, F, i + 2 );
SHA3_2x64_STEP( F, G, H, A, B, C, D, E, i + 3 );
SHA3_2x64_STEP( E, F, G, H, A, B, C, D, i + 4 );
SHA3_2x64_STEP( D, E, F, G, H, A, B, C, i + 5 );
SHA3_2x64_STEP( C, D, E, F, G, H, A, B, i + 6 );
SHA3_2x64_STEP( B, C, D, E, F, G, H, A, i + 7 );
}
if ( ctx->initialized )
{
r[0] = v128_add64( r[0], A );
r[1] = v128_add64( r[1], B );
r[2] = v128_add64( r[2], C );
r[3] = v128_add64( r[3], D );
r[4] = v128_add64( r[4], E );
r[5] = v128_add64( r[5], F );
r[6] = v128_add64( r[6], G );
r[7] = v128_add64( r[7], H );
}
else
{
ctx->initialized = true;
r[0] = v128_add64( A, v128_64( 0x6A09E667F3BCC908 ) );
r[1] = v128_add64( B, v128_64( 0xBB67AE8584CAA73B ) );
r[2] = v128_add64( C, v128_64( 0x3C6EF372FE94F82B ) );
r[3] = v128_add64( D, v128_64( 0xA54FF53A5F1D36F1 ) );
r[4] = v128_add64( E, v128_64( 0x510E527FADE682D1 ) );
r[5] = v128_add64( F, v128_64( 0x9B05688C2B3E6C1F ) );
r[6] = v128_add64( G, v128_64( 0x1F83D9ABFB41BD6B ) );
r[7] = v128_add64( H, v128_64( 0x5BE0CD19137E2179 ) );
}
}
void sha512_2x64_init( sha512_2x64_context *sc )
{
sc->initialized = false;
sc->count = 0;
}
void sha512_2x64_update( sha512_2x64_context *sc, const void *data, size_t len )
{
v128u64_t *vdata = (v128u64_t*)data;
size_t ptr;
const int buf_size = 128;
ptr = (unsigned)sc->count & (buf_size - 1U);
while ( len > 0 )
{
size_t clen;
clen = buf_size - ptr;
if ( clen > len )
clen = len;
v128_memcpy( sc->buf + (ptr>>3), vdata, clen>>3 );
vdata = vdata + (clen>>3);
ptr += clen;
len -= clen;
if ( ptr == buf_size )
{
sha512_2x64_round( sc, sc->buf, sc->val );
ptr = 0;
}
sc->count += clen;
}
}
void sha512_2x64_close( sha512_2x64_context *sc, void *dst )
{
unsigned ptr;
const int buf_size = 128;
const int pad = buf_size - 16;
ptr = (unsigned)sc->count & (buf_size - 1U);
sc->buf[ ptr>>3 ] = v128_64( 0x80 );
ptr += 8;
if ( ptr > pad )
{
v128_memset_zero( sc->buf + (ptr>>3), (buf_size - ptr) >> 3 );
sha512_2x64_round( sc, sc->buf, sc->val );
v128_memset_zero( sc->buf, pad >> 3 );
}
else
v128_memset_zero( sc->buf + (ptr>>3), (pad - ptr) >> 3 );
sc->buf[ pad >> 3 ] = v128_bswap64( v128_64( sc->count >> 61 ) );
sc->buf[ ( pad+8 ) >> 3 ] = v128_bswap64( v128_64( sc->count << 3 ) );
sha512_2x64_round( sc, sc->buf, sc->val );
v128_block_bswap64( castp_v128u64( dst ), sc->val );
}
void sha512_2x64( void *dst, const void *data, size_t len )
{
sha512_2x64_context sc;
sha512_2x64_init( &sc );
sha512_2x64_update( &sc, data, len );
sha512_2x64_close( &sc, dst );
}
void sha512_2x64_ctx( sha512_2x64_context *sc, void *dst, const void *data,
size_t len )
{
sha512_2x64_init( sc );
sha512_2x64_update( sc, data, len );
sha512_2x64_close( sc, dst );
}

48
algo/sha/sha512-hash.h
View File

@@ -35,13 +35,19 @@ typedef struct
__m512i val[8];
uint64_t count;
bool initialized;
} sha512_8way_context __attribute__ ((aligned (128)));
} sha512_8x64_context __attribute__ ((aligned (128)));
#define sha512_8way_context sha512_8x64_context
void sha512_8way_init( sha512_8way_context *sc);
void sha512_8way_update( sha512_8way_context *sc, const void *data,
void sha512_8x64_init( sha512_8x64_context *sc);
void sha512_8x64_update( sha512_8x64_context *sc, const void *data,
size_t len );
void sha512_8way_close( sha512_8way_context *sc, void *dst );
void sha512_8way_full( void *dst, const void *data, size_t len );
void sha512_8x64_close( sha512_8x64_context *sc, void *dst );
void sha512_8x64_ctx( sha512_8x64_context *sc, void *dst, const void *data,
size_t len );
#define sha512_8way_init sha512_8x64_init
#define sha512_8way_update sha512_8x64_update
#define sha512_8way_close sha512_8x64_close
#endif // AVX512
@@ -55,14 +61,36 @@ typedef struct
__m256i val[8];
uint64_t count;
bool initialized;
} sha512_4way_context __attribute__ ((aligned (64)));
} sha512_4x64_context __attribute__ ((aligned (64)));
#define sha512_4way_context sha512_4x64_context
void sha512_4way_init( sha512_4way_context *sc);
void sha512_4way_update( sha512_4way_context *sc, const void *data,
void sha512_4x64_init( sha512_4x64_context *sc);
void sha512_4x64_update( sha512_4x64_context *sc, const void *data,
size_t len );
void sha512_4way_close( sha512_4way_context *sc, void *dst );
void sha512_4way_full( void *dst, const void *data, size_t len );
void sha512_4x64_close( sha512_4x64_context *sc, void *dst );
void sha512_4x64_ctx( sha512_4x64_context *sc, void *dst, const void *data,
size_t len );
#define sha512_4way_init sha512_4x64_init
#define sha512_4way_update sha512_4x64_update
#define sha512_4way_close sha512_4x64_close
#endif // AVX2
typedef struct
{
v128u64_t buf[128>>3];
v128u64_t val[8];
uint64_t count;
bool initialized;
} sha512_2x64_context __attribute__ ((aligned (64)));
void sha512_2x64_init( sha512_2x64_context *sc);
void sha512_2x64_update( sha512_2x64_context *sc, const void *data,
size_t len );
void sha512_2x64_close( sha512_2x64_context *sc, void *dst );
void sha512_2x64( void *dst, const void *data, size_t len );
void sha512_2x64_ctx( sha512_2x64_context *sc, void *dst, const void *data,
size_t len );
#endif