blake3_impl.h Source File (original) (raw)

1#ifndef BLAKE3_IMPL_H

2#define BLAKE3_IMPL_H

4#include <assert.h>

5#include <stdbool.h>

6#include <stddef.h>

7#include <stdint.h>

16#define BLAKE3_PRIVATE

31#if defined(_MSC_VER)

32#define INLINE static __forceinline

33#else

34#define INLINE static inline __attribute__((always_inline))

35#endif

37#if (defined(__x86_64__) || defined(_M_X64)) && !defined(_M_ARM64EC)

38#define IS_X86

39#define IS_X86_64

40#endif

42#if defined(__i386__) || defined(_M_IX86)

43#define IS_X86

44#define IS_X86_32

45#endif

47#if defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC)

48#define IS_AARCH64

49#endif

51#if defined(IS_X86)

52#if defined(_MSC_VER)

53#include <intrin.h>

54#endif

55#include <immintrin.h>

56#endif

58#if !defined(BLAKE3_USE_NEON)

60 #if defined(IS_AARCH64)

61 #if defined(__ARM_BIG_ENDIAN)

62 #define BLAKE3_USE_NEON 0

63 #else

64 #define BLAKE3_USE_NEON 1

65 #endif

66 #else

67 #define BLAKE3_USE_NEON 0

68 #endif

69#endif

71#if defined(IS_X86)

72#define MAX_SIMD_DEGREE 16

73#elif BLAKE3_USE_NEON == 1

74#define MAX_SIMD_DEGREE 4

75#else

76#define MAX_SIMD_DEGREE 1

77#endif

81#define MAX_SIMD_DEGREE_OR_2 (MAX_SIMD_DEGREE > 2 ? MAX_SIMD_DEGREE : 2)

83static const uint32_t IV[8] = {0x6A09E667UL, 0xBB67AE85UL, 0x3C6EF372UL,

84 0xA54FF53AUL, 0x510E527FUL, 0x9B05688CUL,

85 0x1F83D9ABUL, 0x5BE0CD19UL};

88 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},

89 {2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8},

90 {3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1},

91 {10, 7, 12, 9, 14, 3, 13, 15, 4, 0, 11, 2, 5, 8, 1, 6},

92 {12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4},

93 {9, 14, 11, 5, 8, 12, 15, 1, 13, 3, 0, 10, 2, 6, 4, 7},

94 {11, 15, 5, 0, 1, 9, 8, 6, 14, 10, 2, 12, 3, 4, 7, 13},

95};

100#if defined(__GNUC__) || defined(__clang__)

101 return 63 ^ (unsigned int)__builtin_clzll(x);

102#elif defined(_MSC_VER) && defined(IS_X86_64)

103 unsigned long index;

104 _BitScanReverse64(&index, x);

105 return index;

106#elif defined(_MSC_VER) && defined(IS_X86_32)

107 if(x >> 32) {

108 unsigned long index;

109 _BitScanReverse(&index, (unsigned long)(x >> 32));

110 return 32 + index;

111 } else {

112 unsigned long index;

113 _BitScanReverse(&index, (unsigned long)x);

114 return index;

115 }

116#else

117 unsigned int c = 0;

118 if(x & 0xffffffff00000000ULL) { x >>= 32; c += 32; }

119 if(x & 0x00000000ffff0000ULL) { x >>= 16; c += 16; }

120 if(x & 0x000000000000ff00ULL) { x >>= 8; c += 8; }

121 if(x & 0x00000000000000f0ULL) { x >>= 4; c += 4; }

122 if(x & 0x000000000000000cULL) { x >>= 2; c += 2; }

123 if(x & 0x0000000000000002ULL) { c += 1; }

124 return c;

125#endif

126}

127

128

130#if defined(__GNUC__) || defined(__clang__)

131 return (unsigned int)__builtin_popcountll(x);

132#else

133 unsigned int count = 0;

134 while (x != 0) {

135 count += 1;

136 x &= x - 1;

137 }

138 return count;

139#endif

140}

141

142

143

147

149

151 return (uint32_t)(counter >> 32);

152}

153

159

162 key_words[0] = load32(&key[0 * 4]);

163 key_words[1] = load32(&key[1 * 4]);

164 key_words[2] = load32(&key[2 * 4]);

165 key_words[3] = load32(&key[3 * 4]);

166 key_words[4] = load32(&key[4 * 4]);

167 key_words[5] = load32(&key[5 * 4]);

168 key_words[6] = load32(&key[6 * 4]);

169 key_words[7] = load32(&key[7 * 4]);

170}

171

174 for (size_t i = 0; i < 16; i++) {

176 }

177}

178

186

188 store32(&bytes_out[0 * 4], cv_words[0]);

189 store32(&bytes_out[1 * 4], cv_words[1]);

190 store32(&bytes_out[2 * 4], cv_words[2]);

191 store32(&bytes_out[3 * 4], cv_words[3]);

192 store32(&bytes_out[4 * 4], cv_words[4]);

193 store32(&bytes_out[5 * 4], cv_words[5]);

194 store32(&bytes_out[6 * 4], cv_words[6]);

195 store32(&bytes_out[7 * 4], cv_words[7]);

196}

197

203

209

214 uint8_t out[64], size_t outblocks);

215

219 bool increment_counter, uint8_t flags,

221

224

228 uint8_t *out, bool use_tbb);

229

230#if defined(BLAKE3_USE_TBB)

231BLAKE3_PRIVATE void blake3_compress_subtree_wide_join_tbb(

232

234

235 const uint8_t *l_input, size_t l_input_len, uint64_t l_chunk_counter,

236 uint8_t *l_cvs, size_t *l_n,

237

238 const uint8_t *r_input, size_t r_input_len, uint64_t r_chunk_counter,

239 uint8_t *r_cvs, size_t *r_n) NOEXCEPT;

240#endif

241

242

248

254

258 uint64_t counter, bool increment_counter,

261

262#if defined(IS_X86)

263#if !defined(BLAKE3_NO_SSE2)

277 uint64_t counter, bool increment_counter,

280#endif

281#if !defined(BLAKE3_NO_SSE41)

295 uint64_t counter, bool increment_counter,

298#endif

299#if !defined(BLAKE3_NO_AVX2)

303 uint64_t counter, bool increment_counter,

306#endif

307#if !defined(BLAKE3_NO_AVX512)

313

319

323 uint64_t counter, bool increment_counter,

326

327#if !defined(_WIN32) && !defined(__CYGWIN__)

332 uint8_t* out, size_t outblocks);

333#endif

334#endif

335#endif

336

337#if BLAKE3_USE_NEON == 1

341 uint64_t counter, bool increment_counter,

344#endif

345

346

347#endif

bbsections Prepares for basic block by splitting functions into clusters of basic blocks

#define LLVM_LIBRARY_VISIBILITY

unify loop Fixup each natural loop to have a single exit block

static const uint8_t MSG_SCHEDULE[7][16]

Definition blake3_impl.h:87

INLINE unsigned int popcnt(uint64_t x)

Definition blake3_impl.h:129

LLVM_LIBRARY_VISIBILITY size_t blake3_simd_degree(void)

#define INLINE

Definition blake3_impl.h:34

#define BLAKE3_PRIVATE

Definition blake3_impl.h:16

static const uint32_t IV[8]

Definition blake3_impl.h:83

static unsigned int highest_one(uint64_t x)

Definition blake3_impl.h:99

INLINE uint32_t counter_high(uint64_t counter)

Definition blake3_impl.h:150

INLINE void load_key_words(const uint8_t key[BLAKE3_KEY_LEN], uint32_t key_words[8])

Definition blake3_impl.h:160

INLINE void load_block_words(const uint8_t block[BLAKE3_BLOCK_LEN], uint32_t block_words[16])

Definition blake3_impl.h:172

INLINE void store_cv_words(uint8_t bytes_out[32], uint32_t cv_words[8])

Definition blake3_impl.h:187

INLINE uint64_t round_down_to_power_of_2(uint64_t x)

Definition blake3_impl.h:144

INLINE uint32_t load32(const void *src)

Definition blake3_impl.h:154

INLINE void store32(void *dst, uint32_t w)

Definition blake3_impl.h:179

blake3_flags

Definition blake3_impl.h:19

@ CHUNK_START

Definition blake3_impl.h:20

@ PARENT

Definition blake3_impl.h:22

@ KEYED_HASH

Definition blake3_impl.h:24

@ DERIVE_KEY_MATERIAL

Definition blake3_impl.h:26

@ DERIVE_KEY_CONTEXT

Definition blake3_impl.h:25

@ ROOT

Definition blake3_impl.h:23

@ CHUNK_END

Definition blake3_impl.h:21

INLINE uint32_t counter_low(uint64_t counter)

Definition blake3_impl.h:148

#define blake3_compress_in_place_sse41

#define blake3_hash_many_neon

#define blake3_hash_many_avx512

#define blake3_hash_many_avx2

#define blake3_compress_xof_sse2

#define blake3_hash_many_sse41

#define blake3_compress_xof

#define blake3_compress_xof_sse41

#define blake3_compress_in_place_sse2

#define blake3_compress_in_place

#define blake3_compress_xof_avx512

#define blake3_xof_many_avx512

#define blake3_compress_xof_portable

#define blake3_hash_many_portable

#define blake3_hash_many_sse2

#define blake3_compress_in_place_portable

#define blake3_compress_subtree_wide

#define blake3_compress_in_place_avx512