Mercurial > vec
annotate src/impl/x86/avx512f.c @ 33:4655b49eaf9f
Backed out changeset 6c91cd9a2f2d
| author | Paper <paper@tflc.us> |
|---|---|
| date | Fri, 25 Apr 2025 17:40:42 -0400 |
| parents | bf6ad516f1e6 |
| children | 99e4539f922f |
| rev | line source |
|---|---|
|
23
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
1 /** |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
2 * vec - a tiny SIMD vector library in C99 |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
3 * |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
4 * Copyright (c) 2024 Paper |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
5 * |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
6 * Permission is hereby granted, free of charge, to any person obtaining a copy |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
7 * of this software and associated documentation files (the "Software"), to deal |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
8 * in the Software without restriction, including without limitation the rights |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
10 * copies of the Software, and to permit persons to whom the Software is |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
11 * furnished to do so, subject to the following conditions: |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
12 * |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
13 * The above copyright notice and this permission notice shall be included in all |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
14 * copies or substantial portions of the Software. |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
15 * |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
22 * SOFTWARE. |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
23 **/ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
24 |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
25 #include "vec/impl/x86/avx512f.h" |
| 31 | 26 #include "vec/impl/generic.h" |
|
23
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
27 |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
28 #include <immintrin.h> |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
29 |
| 31 | 30 // this is a stupid amount of work just to do these operations, is it really worth it ? |
| 31 // also same note in avx2.c applies here, these do not handle sign bits properly, which | |
| 32 // isn't that big of a deal for regular arithmetic operations, but matters quite a bit | |
| 33 // when doing things like arithmetic shifts. | |
| 34 #define VEC_AVX512F_OPERATION_8x64(op, sign) \ | |
| 35 do { \ | |
| 36 union v##sign##int8x64_impl_data *vec1d = (union v##sign##int8x64_impl_data *)&vec1; \ | |
| 37 union v##sign##int8x64_impl_data *vec2d = (union v##sign##int8x64_impl_data *)&vec2; \ | |
|
23
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
38 \ |
| 31 | 39 /* unpack and operate */ \ |
| 40 __m512i dst_1 = _mm512_##op##_epi32(_mm512_srli_epi32(_mm512_slli_epi32(vec1d->avx512f, 24), 24), _mm512_srli_epi32(_mm512_slli_epi32(vec2d->avx512f, 24), 24)); \ | |
| 41 __m512i dst_2 = _mm512_##op##_epi32(_mm512_srli_epi32(_mm512_slli_epi32(vec1d->avx512f, 16), 24), _mm512_srli_epi32(_mm512_slli_epi32(vec2d->avx512f, 16), 24)); \ | |
| 42 __m512i dst_3 = _mm512_##op##_epi32(_mm512_srli_epi32(_mm512_slli_epi32(vec1d->avx512f, 8), 24), _mm512_srli_epi32(_mm512_slli_epi32(vec2d->avx512f, 8), 24)); \ | |
| 43 __m512i dst_4 = _mm512_##op##_epi32(_mm512_srli_epi32(vec1d->avx512f, 24), _mm512_srli_epi32(vec2d->avx512f, 24)); \ | |
|
23
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
44 \ |
| 31 | 45 /* repack */ \ |
| 46 vec1d->avx512f = _mm512_or_si512( \ | |
| 47 _mm512_or_si512( \ | |
| 48 _mm512_srli_epi32(_mm512_slli_epi32(dst_1, 24), 24), \ | |
| 49 _mm512_srli_epi32(_mm512_slli_epi32(dst_2, 24), 16) \ | |
| 50 ), \ | |
| 51 _mm512_or_si512( \ | |
| 52 _mm512_srli_epi32(_mm512_slli_epi32(dst_3, 24), 8), \ | |
| 53 _mm512_slli_epi32(dst_4, 24) \ | |
| 54 ) \ | |
| 55 ); \ | |
|
23
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
56 \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
57 return vec1d->vec; \ |
| 31 | 58 } while (0) |
|
23
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
59 |
| 31 | 60 #define VEC_AVX512F_OPERATION_16x32(op, sign) \ |
| 61 do { \ | |
| 62 union v##sign##int16x32_impl_data *vec1d = (union v##sign##int16x32_impl_data *)&vec1; \ | |
| 63 union v##sign##int16x32_impl_data *vec2d = (union v##sign##int16x32_impl_data *)&vec2; \ | |
| 64 \ | |
| 65 /* unpack and operate; it would be nice if we had an _m512_andi_epi32... */ \ | |
| 66 __m512i dst_1 = _mm512_##op##_epi32(_mm512_srli_epi32(_mm512_slli_epi32(vec1d->avx512f, 16), 16), _mm512_srli_epi32(_mm512_slli_epi32(vec2d->avx512f, 16), 16)); \ | |
| 67 __m512i dst_2 = _mm512_##op##_epi32(_mm512_srli_epi32(vec1d->avx512f, 16), _mm512_srli_epi32(vec2d->avx512f, 16)); \ | |
| 68 \ | |
| 69 /* repack */ \ | |
| 70 vec1d->avx512f = _mm512_or_si512( \ | |
| 71 _mm512_srli_epi32(_mm512_slli_epi32(dst_1, 16), 16), \ | |
| 72 _mm512_slli_epi32(dst_2, 16) \ | |
| 73 ); \ | |
| 74 return vec1d->vec; \ | |
| 75 } while (0) | |
|
23
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
76 |
| 31 | 77 #define VEC_AVX512F_ADD_8x64(sign) \ |
| 78 VEC_AVX512F_OPERATION_8x64(add, sign) | |
| 79 | |
| 80 #define VEC_AVX512F_ADD_16x32(sign) \ | |
| 81 VEC_AVX512F_OPERATION_16x32(add, sign) | |
| 82 | |
| 83 #define VEC_AVX512F_ADD_32x16(sign) \ | |
| 84 do { \ | |
| 85 union v##sign##int32x16_impl_data *vec1d = (union v##sign##int32x16_impl_data *)&vec1; \ | |
| 86 union v##sign##int32x16_impl_data *vec2d = (union v##sign##int32x16_impl_data *)&vec2; \ | |
| 87 \ | |
| 88 vec1d->avx512f = _mm512_add_epi32(vec1d->avx512f, vec2d->avx512f); \ | |
| 89 return vec1d->vec; \ | |
| 90 } while (0) | |
|
23
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
91 |
| 31 | 92 #define VEC_AVX512F_ADD_64x8(sign) \ |
| 93 do { \ | |
| 94 union v##sign##int64x8_impl_data *vec1d = (union v##sign##int64x8_impl_data *)&vec1; \ | |
| 95 union v##sign##int64x8_impl_data *vec2d = (union v##sign##int64x8_impl_data *)&vec2; \ | |
| 96 \ | |
| 97 vec1d->avx512f = _mm512_add_epi64(vec1d->avx512f, vec2d->avx512f); \ | |
| 98 return vec1d->vec; \ | |
| 99 } while (0) | |
| 100 | |
| 101 #define VEC_AVX512F_SUB_8x64(sign) \ | |
| 102 VEC_AVX512F_OPERATION_8x64(sub, sign) | |
| 103 | |
| 104 #define VEC_AVX512F_SUB_16x32(sign) \ | |
| 105 VEC_AVX512F_OPERATION_16x32(sub, sign) | |
| 106 | |
| 107 #define VEC_AVX512F_SUB_32x16(sign) \ | |
| 108 do { \ | |
| 109 union v##sign##int32x16_impl_data *vec1d = (union v##sign##int32x16_impl_data *)&vec1; \ | |
| 110 union v##sign##int32x16_impl_data *vec2d = (union v##sign##int32x16_impl_data *)&vec2; \ | |
| 111 \ | |
| 112 vec1d->avx512f = _mm512_sub_epi32(vec1d->avx512f, vec2d->avx512f); \ | |
| 113 return vec1d->vec; \ | |
| 114 } while (0) | |
|
23
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
115 |
| 31 | 116 #define VEC_AVX512F_SUB_64x8(sign) \ |
| 117 do { \ | |
| 118 union v##sign##int64x8_impl_data *vec1d = (union v##sign##int64x8_impl_data *)&vec1; \ | |
| 119 union v##sign##int64x8_impl_data *vec2d = (union v##sign##int64x8_impl_data *)&vec2; \ | |
| 120 \ | |
| 121 vec1d->avx512f = _mm512_sub_epi64(vec1d->avx512f, vec2d->avx512f); \ | |
| 122 return vec1d->vec; \ | |
| 123 } while (0) | |
| 124 | |
| 125 #define VEC_AVX512F_MUL_8x64(sign) \ | |
| 126 VEC_AVX512F_OPERATION_8x64(mullo, sign) | |
|
23
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
127 |
| 31 | 128 #define VEC_AVX512F_MUL_16x32(sign) \ |
| 129 VEC_AVX512F_OPERATION_16x32(mullo, sign) | |
| 130 | |
| 131 #define VEC_AVX512F_MUL_32x16(sign) \ | |
| 132 do { \ | |
| 133 union v##sign##int32x16_impl_data *vec1d = (union v##sign##int32x16_impl_data *)&vec1; \ | |
| 134 union v##sign##int32x16_impl_data *vec2d = (union v##sign##int32x16_impl_data *)&vec2; \ | |
| 135 \ | |
| 136 vec1d->avx512f = _mm512_mullo_epi32(vec1d->avx512f, vec2d->avx512f); \ | |
| 137 return vec1d->vec; \ | |
| 138 } while (0) | |
|
23
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
139 |
| 31 | 140 #define VEC_AVX512F_MUL_64x8(sign) \ |
| 141 do { \ | |
| 142 union v##sign##int64x8_impl_data *vec1d = (union v##sign##int64x8_impl_data *)&vec1; \ | |
| 143 union v##sign##int64x8_impl_data *vec2d = (union v##sign##int64x8_impl_data *)&vec2; \ | |
| 144 \ | |
| 145 __m512i ac = _mm512_mul_epu32(vec1d->avx512f, vec2d->avx512f); \ | |
| 146 __m512i b = _mm512_srli_epi64(vec1d->avx512f, 32); \ | |
| 147 __m512i bc = _mm512_mul_epu32(b, vec2d->avx512f); \ | |
| 148 __m512i d = _mm512_srli_epi64(vec2d->avx512f, 32); \ | |
| 149 __m512i ad = _mm512_mul_epu32(vec1d->avx512f, d); \ | |
| 150 __m512i hi = _mm512_add_epi64(bc, ad); \ | |
| 151 hi = _mm512_slli_epi64(hi, 32); \ | |
| 152 \ | |
| 153 vec1d->avx512f = _mm512_add_epi64(hi, ac); \ | |
| 154 return vec1d->vec; \ | |
| 155 } while (0) | |
| 156 | |
| 157 #define VEC_AVX512F_LSHIFT_8x64(sign) \ | |
| 158 VEC_AVX512F_OPERATION_8x64(sllv, sign) | |
|
23
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
159 |
| 31 | 160 #define VEC_AVX512F_LSHIFT_16x32(sign) \ |
| 161 VEC_AVX512F_OPERATION_16x32(sllv, sign) | |
|
23
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
162 |
| 31 | 163 #define VEC_AVX512F_LSHIFT_32x16(sign) \ |
| 164 do { \ | |
| 165 union v##sign##int32x16_impl_data *vec1d = (union v##sign##int32x16_impl_data *)&vec1; \ | |
| 166 union v##sign##int32x16_impl_data *vec2d = (union v##sign##int32x16_impl_data *)&vec2; \ | |
| 167 \ | |
| 168 vec1d->avx512f = _mm512_sllv_epi32(vec1d->avx512f, vec2d->avx512f); \ | |
| 169 return vec1d->vec; \ | |
| 170 } while (0) | |
| 171 | |
| 172 #define VEC_AVX512F_LSHIFT_64x8(sign) \ | |
| 173 do { \ | |
| 174 union v##sign##int64x8_impl_data *vec1d = (union v##sign##int64x8_impl_data *)&vec1; \ | |
| 175 union v##sign##int64x8_impl_data *vec2d = (union v##sign##int64x8_impl_data *)&vec2; \ | |
| 176 \ | |
| 177 vec1d->avx512f = _mm512_sllv_epi64(vec1d->avx512f, vec2d->avx512f); \ | |
| 178 return vec1d->vec; \ | |
| 179 } while (0) | |
|
23
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
180 |
| 31 | 181 #define VEC_AVX512F_lRSHIFT_8x64(sign) \ |
| 182 VEC_AVX512F_OPERATION_8x64(srlv, sign) | |
| 183 | |
| 184 #define VEC_AVX512F_lRSHIFT_16x32(sign) \ | |
| 185 VEC_AVX512F_OPERATION_16x32(srlv, sign) | |
|
23
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
186 |
| 31 | 187 #define VEC_AVX512F_aRSHIFT_8x64(sign) \ |
| 188 do { \ | |
| 189 return v##sign##int8x64_generic_rshift(vec1, vec2); \ | |
| 190 } while (0) | |
| 191 | |
| 192 #define VEC_AVX512F_aRSHIFT_16x32(sign) \ | |
| 193 do { \ | |
| 194 return v##sign##int16x32_generic_rshift(vec1, vec2); \ | |
| 195 } while (0) | |
| 196 | |
| 197 #define VEC_AVX512F_RSHIFT_8x64(sign, aORl) VEC_AVX512F_##aORl##RSHIFT_8x64(sign) | |
| 198 #define VEC_AVX512F_RSHIFT_16x32(sign, aORl) VEC_AVX512F_##aORl##RSHIFT_16x32(sign) | |
|
23
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
199 |
| 31 | 200 #define VEC_AVX512F_RSHIFT_32x16(sign, aORl) \ |
| 201 do { \ | |
| 202 union v##sign##int32x16_impl_data *vec1d = (union v##sign##int32x16_impl_data *)&vec1; \ | |
| 203 union v##sign##int32x16_impl_data *vec2d = (union v##sign##int32x16_impl_data *)&vec2; \ | |
| 204 \ | |
| 205 vec1d->avx512f = _mm512_sr##aORl##v_epi32(vec1d->avx512f, vec2d->avx512f); \ | |
| 206 return vec1d->vec; \ | |
| 207 } while (0) | |
|
23
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
208 |
| 31 | 209 #define VEC_AVX512F_RSHIFT_64x8(sign, aORl) \ |
| 210 do { \ | |
| 211 union v##sign##int64x8_impl_data *vec1d = (union v##sign##int64x8_impl_data *)&vec1; \ | |
| 212 union v##sign##int64x8_impl_data *vec2d = (union v##sign##int64x8_impl_data *)&vec2; \ | |
| 213 \ | |
| 214 vec1d->avx512f = _mm512_sr##aORl##v_epi64(vec1d->avx512f, vec2d->avx512f); \ | |
| 215 return vec1d->vec; \ | |
| 216 } while (0) | |
| 217 | |
| 218 #define VEC_AVX512F_uRSHIFT_8x64(sign, aORl) VEC_AVX512F_RSHIFT_8x64(sign, l) | |
| 219 #define VEC_AVX512F_uRSHIFT_16x32(sign, aORl) VEC_AVX512F_RSHIFT_16x32(sign, l) | |
| 220 #define VEC_AVX512F_uRSHIFT_32x16(sign, aORl) VEC_AVX512F_RSHIFT_32x16(sign, l) | |
| 221 #define VEC_AVX512F_uRSHIFT_64x8(sign, aORl) VEC_AVX512F_RSHIFT_64x8(sign, l) | |
|
23
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
222 |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
223 #define VEC_AVX512F_DEFINE_OPERATIONS_SIGN(sign, bits, size) \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
224 union v##sign##int##bits##x##size##_impl_data { \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
225 v##sign##int##bits##x##size vec; \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
226 __m512i avx512f; \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
227 }; \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
228 \ |
|
24
e49e70f7012f
impl/x86: add static assertions for alignment and size
Paper <paper@tflc.us>
parents:
23
diff
changeset
|
229 VEC_STATIC_ASSERT(VEC_ALIGNOF(__m512i) <= VEC_ALIGNOF(v##sign##int##bits##x##size), "vec: v" #sign "int" #bits "x" #size " alignment needs to be expanded to fit intrinsic type size"); \ |
|
e49e70f7012f
impl/x86: add static assertions for alignment and size
Paper <paper@tflc.us>
parents:
23
diff
changeset
|
230 VEC_STATIC_ASSERT(sizeof(__m512i) <= sizeof(v##sign##int##bits##x##size), "vec: v" #sign "int" #bits "x" #size " needs to be expanded to fit intrinsic type size"); \ |
|
e49e70f7012f
impl/x86: add static assertions for alignment and size
Paper <paper@tflc.us>
parents:
23
diff
changeset
|
231 \ |
| 31 | 232 static v##sign##int##bits##x##size v##sign##int##bits##x##size##_avx512f_load_aligned(const vec_##sign##int##bits in[size]) \ |
|
23
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
233 { \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
234 union v##sign##int##bits##x##size##_impl_data vec; \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
235 vec.avx512f = _mm512_load_si512((const __m512i *)in); \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
236 return vec.vec; \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
237 } \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
238 \ |
| 31 | 239 static v##sign##int##bits##x##size v##sign##int##bits##x##size##_avx512f_load(const vec_##sign##int##bits in[size]) \ |
|
23
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
240 { \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
241 union v##sign##int##bits##x##size##_impl_data vec; \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
242 vec.avx512f = _mm512_loadu_si512((const __m512i *)in); \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
243 return vec.vec; \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
244 } \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
245 \ |
| 31 | 246 static void v##sign##int##bits##x##size##_avx512f_store_aligned(v##sign##int##bits##x##size vec, vec_##sign##int##bits out[size]) \ |
|
23
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
247 { \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
248 _mm512_store_si512((__m512i *)out, ((union v##sign##int##bits##x##size##_impl_data *)&vec)->avx512f); \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
249 } \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
250 \ |
| 31 | 251 static void v##sign##int##bits##x##size##_avx512f_store(v##sign##int##bits##x##size vec, vec_##sign##int##bits out[size]) \ |
|
23
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
252 { \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
253 _mm512_storeu_si512((__m512i *)out, ((union v##sign##int##bits##x##size##_impl_data *)&vec)->avx512f); \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
254 } \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
255 \ |
| 31 | 256 static v##sign##int##bits##x##size v##sign##int##bits##x##size##_avx512f_add(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2) \ |
| 257 { \ | |
| 258 VEC_AVX512F_ADD_##bits##x##size(sign); \ | |
| 259 } \ | |
|
23
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
260 \ |
| 31 | 261 static v##sign##int##bits##x##size v##sign##int##bits##x##size##_avx512f_sub(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2) \ |
| 262 { \ | |
| 263 VEC_AVX512F_SUB_##bits##x##size(sign); \ | |
| 264 } \ | |
| 265 \ | |
| 266 static v##sign##int##bits##x##size v##sign##int##bits##x##size##_avx512f_mul(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2) \ | |
| 267 { \ | |
| 268 VEC_AVX512F_MUL_##bits##x##size(sign); \ | |
| 269 } \ | |
| 270 \ | |
| 271 static v##sign##int##bits##x##size v##sign##int##bits##x##size##_avx512f_and(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2) \ | |
|
23
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
272 { \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
273 union v##sign##int##bits##x##size##_impl_data *vec1d = (union v##sign##int##bits##x##size##_impl_data *)&vec1; \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
274 union v##sign##int##bits##x##size##_impl_data *vec2d = (union v##sign##int##bits##x##size##_impl_data *)&vec2; \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
275 \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
276 vec1d->avx512f = _mm512_and_si512(vec1d->avx512f, vec2d->avx512f); \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
277 return vec1d->vec; \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
278 } \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
279 \ |
| 31 | 280 static v##sign##int##bits##x##size v##sign##int##bits##x##size##_avx512f_or(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2) \ |
|
23
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
281 { \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
282 union v##sign##int##bits##x##size##_impl_data *vec1d = (union v##sign##int##bits##x##size##_impl_data *)&vec1; \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
283 union v##sign##int##bits##x##size##_impl_data *vec2d = (union v##sign##int##bits##x##size##_impl_data *)&vec2; \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
284 \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
285 vec1d->avx512f = _mm512_or_si512(vec1d->avx512f, vec2d->avx512f); \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
286 return vec1d->vec; \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
287 } \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
288 \ |
| 31 | 289 static v##sign##int##bits##x##size v##sign##int##bits##x##size##_avx512f_xor(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2) \ |
|
23
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
290 { \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
291 union v##sign##int##bits##x##size##_impl_data *vec1d = (union v##sign##int##bits##x##size##_impl_data *)&vec1; \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
292 union v##sign##int##bits##x##size##_impl_data *vec2d = (union v##sign##int##bits##x##size##_impl_data *)&vec2; \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
293 \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
294 vec1d->avx512f = _mm512_xor_si512(vec1d->avx512f, vec2d->avx512f); \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
295 return vec1d->vec; \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
296 } \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
297 \ |
| 31 | 298 static v##sign##int##bits##x##size v##sign##int##bits##x##size##_avx512f_lshift(v##sign##int##bits##x##size vec1, vuint##bits##x##size vec2) \ |
| 299 { \ | |
| 300 VEC_AVX512F_LSHIFT_##bits##x##size(sign); \ | |
| 301 } \ | |
|
23
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
302 \ |
| 31 | 303 static v##sign##int##bits##x##size v##sign##int##bits##x##size##_avx512f_rshift(v##sign##int##bits##x##size vec1, vuint##bits##x##size vec2) \ |
| 304 { \ | |
| 305 VEC_AVX512F_##sign##RSHIFT_##bits##x##size(sign, a); \ | |
| 306 } \ | |
| 307 \ | |
| 308 static v##sign##int##bits##x##size v##sign##int##bits##x##size##_avx512f_lrshift(v##sign##int##bits##x##size vec1, vuint##bits##x##size vec2) \ | |
| 309 { \ | |
| 310 VEC_AVX512F_RSHIFT_##bits##x##size(sign, l); \ | |
| 311 } \ | |
|
23
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
312 \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
313 const v##sign##int##bits##x##size##_impl v##sign##int##bits##x##size##_impl_avx512f = { \ |
| 31 | 314 v##sign##int##bits##x##size##_generic_splat, \ |
| 315 v##sign##int##bits##x##size##_avx512f_load_aligned, \ | |
| 316 v##sign##int##bits##x##size##_avx512f_load, \ | |
| 317 v##sign##int##bits##x##size##_avx512f_store_aligned, \ | |
| 318 v##sign##int##bits##x##size##_avx512f_store, \ | |
| 319 v##sign##int##bits##x##size##_avx512f_add, \ | |
| 320 v##sign##int##bits##x##size##_avx512f_sub, \ | |
| 321 v##sign##int##bits##x##size##_avx512f_mul, \ | |
| 322 v##sign##int##bits##x##size##_generic_div, \ | |
| 323 v##sign##int##bits##x##size##_generic_avg, \ | |
| 324 v##sign##int##bits##x##size##_avx512f_and, \ | |
| 325 v##sign##int##bits##x##size##_avx512f_or, \ | |
| 326 v##sign##int##bits##x##size##_avx512f_xor, \ | |
| 327 v##sign##int##bits##x##size##_generic_not, \ | |
| 328 v##sign##int##bits##x##size##_avx512f_lshift, \ | |
| 329 v##sign##int##bits##x##size##_avx512f_rshift, \ | |
| 330 v##sign##int##bits##x##size##_avx512f_lrshift, \ | |
| 331 v##sign##int##bits##x##size##_generic_cmplt, \ | |
| 332 v##sign##int##bits##x##size##_generic_cmple, \ | |
| 333 v##sign##int##bits##x##size##_generic_cmpeq, \ | |
| 334 v##sign##int##bits##x##size##_generic_cmpge, \ | |
| 335 v##sign##int##bits##x##size##_generic_cmpgt, \ | |
|
23
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
336 }; |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
337 |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
338 #define VEC_AVX512F_DEFINE_OPERATIONS(bits, size) \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
339 VEC_AVX512F_DEFINE_OPERATIONS_SIGN(u, bits, size) \ |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
340 VEC_AVX512F_DEFINE_OPERATIONS_SIGN( , bits, size) |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
341 |
| 31 | 342 VEC_AVX512F_DEFINE_OPERATIONS(8, 64) |
| 343 VEC_AVX512F_DEFINE_OPERATIONS(16, 32) | |
|
23
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
344 VEC_AVX512F_DEFINE_OPERATIONS(32, 16) |
|
e26874655738
*: huge refactor, new major release (hahaha)
Paper <paper@tflc.us>
parents:
diff
changeset
|
345 VEC_AVX512F_DEFINE_OPERATIONS(64, 8) |