Mercurial > vec

diff src/impl/x86/avx2.c @ 23:e26874655738

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
*: huge refactor, new major release (hahaha) I keep finding things that are broken... The problem NOW was that vec would unintentionally build some functions with extended instruction sets, which is Bad and would mean that for all intents and purposes the CPU detection was completely broken. Now vec is no longer header only either. Boohoo. However this gives a lot more flexibility to vec since we no longer want or need to care about C++ crap. The NEON and Altivec implementations have not been updated which means they won't compile hence why they're commented out in the cmake build file.
author Paper <paper@tflc.us>
date Sun, 24 Nov 2024 02:52:40 -0500
parents
children e49e70f7012f
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/impl/x86/avx2.c	Sun Nov 24 02:52:40 2024 -0500
@@ -0,0 +1,219 @@
+/**
+ * vec - a tiny SIMD vector library in C99
+ * 
+ * Copyright (c) 2024 Paper
+ * 
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ * 
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+**/
+
+#include "vec/impl/x86/avx2.h"
+#include "vec/impl/generic.h"
+
+#include <immintrin.h>
+
+// this does NOT handle sign bits properly, use with caution
+#define VEC_AVX2_OPERATION_8x32_16x16(op, sign) \
+	do { \
+		union v##sign##int8x32_impl_data *vec1d = (union v##sign##int8x32_impl_data *)&vec1; \
+		union v##sign##int8x32_impl_data *vec2d = (union v##sign##int8x32_impl_data *)&vec2; \
+	\
+		/* unpack and multiply */ \
+		__m256i dst_even = _mm256_##op##_epi16(vec1d->avx2, vec2d->avx2); \
+		__m256i dst_odd = _mm256_##op##_epi16(_mm256_srli_epi16(vec1d->avx2, 8), _mm256_srli_epi16(vec2d->avx2, 8)); \
+	\
+		/* repack */ \
+		vec1d->avx2 = _mm256_or_si256( \
+			_mm256_slli_epi16(dst_odd, 8), \
+			_mm256_srli_epi16(_mm256_slli_epi16(dst_even, 8), 8) \
+		); \
+		return vec1d->vec; \
+	} while (0)
+
+#define VEC_AVX2_OPERATION_16x16(op, sign) \
+	do { \
+		union v##sign##int16x16_impl_data *vec1d = (union v##sign##int16x16_impl_data *)&vec1; \
+		union v##sign##int16x16_impl_data *vec2d = (union v##sign##int16x16_impl_data *)&vec2; \
+	\
+		/* unpack and multiply */ \
+		__m256i dst_even = _mm256_##op##_epi32(vec1d->avx2, vec2d->avx2); \
+		__m256i dst_odd = _mm256_##op##_epi32(_mm256_srli_epi32(vec1d->avx2, 16), _mm256_srli_epi32(vec2d->avx2, 16)); \
+	\
+		/* repack */ \
+		vec1d->avx2 = _mm256_or_si256( \
+			_mm256_slli_epi32(dst_odd, 16), \
+			_mm256_srli_epi32(_mm256_slli_epi16(dst_even, 16), 16) \
+		); \
+		return vec1d->vec; \
+	} while (0)
+
+// multiplication
+
+#define VEC_AVX2_MUL_8x32(sign) \
+	VEC_AVX2_OPERATION_8x32_16x16(mullo, sign)
+
+#define VEC_AVX2_MUL_16x16(sign) \
+	do { \
+		union v##sign##int16x16_impl_data *vec1d = (union v##sign##int16x16_impl_data *)&vec1; \
+		union v##sign##int16x16_impl_data *vec2d = (union v##sign##int16x16_impl_data *)&vec2; \
+	\
+		vec1d->avx2 = _mm256_mullo_epi16(vec1d->avx2, vec2d->avx2); \
+		return vec1d->vec; \
+	} while (0)
+
+#define VEC_AVX2_MUL_32x8(sign) \
+	do { \
+		union v##sign##int32x8_impl_data *vec1d = (union v##sign##int32x8_impl_data *)&vec1; \
+		union v##sign##int32x8_impl_data *vec2d = (union v##sign##int32x8_impl_data *)&vec2; \
+	\
+		vec1d->avx2 = _mm256_mullo_epi32(vec1d->avx2, vec2d->avx2); \
+		return vec1d->vec; \
+	} while (0)
+
+#define VEC_AVX2_MUL_64x4(sign) \
+	do { \
+		union v##sign##int64x4_impl_data *vec1d = (union v##sign##int64x4_impl_data *)&vec1; \
+		union v##sign##int64x4_impl_data *vec2d = (union v##sign##int64x4_impl_data *)&vec2; \
+	\
+		__m256i ac = _mm256_mul_epu32(vec1d->avx2, vec2d->avx2); \
+		__m256i b  = _mm256_srli_epi64(vec1d->avx2, 32); \
+		__m256i bc = _mm256_mul_epu32(b, vec2d->avx2); \
+		__m256i d  = _mm256_srli_epi64(vec2d->avx2, 32); \
+		__m256i ad = _mm256_mul_epu32(vec1d->avx2, d); \
+		__m256i hi = _mm256_add_epi64(bc, ad); \
+		hi = _mm256_slli_epi64(hi, 32); \
+	\
+		vec1d->avx2 = _mm256_add_epi64(hi, ac); \
+		return vec1d->vec; \
+	} while (0)
+
+// operations
+
+#define VEC_AVX2_DEFINE_OPERATIONS_SIGN(sign, bits, size) \
+	union v##sign##int##bits##x##size##_impl_data { \
+		v##sign##int##bits##x##size vec; \
+		__m256i avx2; \
+	}; \
+	\
+	static v##sign##int##bits##x##size v##sign##int##bits##x##size##_avx2_load_aligned(const vec_##sign##int##bits in[size]) \
+	{ \
+		union v##sign##int##bits##x##size##_impl_data vec; \
+		vec.avx2 = _mm256_load_si256((const __m256i *)in); \
+		return vec.vec; \
+	} \
+	\
+	static v##sign##int##bits##x##size v##sign##int##bits##x##size##_avx2_load(const vec_##sign##int##bits in[size]) \
+	{ \
+		union v##sign##int##bits##x##size##_impl_data vec; \
+		vec.avx2 = _mm256_loadu_si256((const __m256i *)in); \
+		return vec.vec; \
+	} \
+	\
+	static void v##sign##int##bits##x##size##_avx2_store_aligned(v##sign##int##bits##x##size vec, vec_##sign##int##bits out[size]) \
+	{ \
+		_mm256_store_si256((__m256i *)out, ((union v##sign##int##bits##x##size##_impl_data*)&vec)->avx2); \
+	} \
+	\
+	static void v##sign##int##bits##x##size##_avx2_store(v##sign##int##bits##x##size vec, vec_##sign##int##bits out[size]) \
+	{ \
+		_mm256_storeu_si256((__m256i *)out, ((union v##sign##int##bits##x##size##_impl_data*)&vec)->avx2); \
+	} \
+	\
+	static v##sign##int##bits##x##size v##sign##int##bits##x##size##_avx2_add(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2) \
+	{ \
+		union v##sign##int##bits##x##size##_impl_data *vec1d = (union v##sign##int##bits##x##size##_impl_data *)&vec1; \
+		union v##sign##int##bits##x##size##_impl_data *vec2d = (union v##sign##int##bits##x##size##_impl_data *)&vec2; \
+	\
+		vec1d->avx2 = _mm256_add_epi##bits(vec1d->avx2, vec2d->avx2); \
+		return vec1d->vec; \
+	} \
+	\
+	static v##sign##int##bits##x##size v##sign##int##bits##x##size##_avx2_sub(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2) \
+	{ \
+		union v##sign##int##bits##x##size##_impl_data *vec1d = (union v##sign##int##bits##x##size##_impl_data *)&vec1; \
+		union v##sign##int##bits##x##size##_impl_data *vec2d = (union v##sign##int##bits##x##size##_impl_data *)&vec2; \
+	\
+		vec1d->avx2 = _mm256_sub_epi##bits(vec1d->avx2, vec2d->avx2); \
+		return vec1d->vec; \
+	} \
+	\
+	static v##sign##int##bits##x##size v##sign##int##bits##x##size##_avx2_mul(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2) \
+	{ \
+		VEC_AVX2_MUL_##bits##x##size(sign); \
+	} \
+	\
+	static v##sign##int##bits##x##size v##sign##int##bits##x##size##_avx2_and(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2) \
+	{ \
+		union v##sign##int##bits##x##size##_impl_data *vec1d = (union v##sign##int##bits##x##size##_impl_data *)&vec1; \
+		union v##sign##int##bits##x##size##_impl_data *vec2d = (union v##sign##int##bits##x##size##_impl_data *)&vec2; \
+	\
+		vec1d->avx2 = _mm256_and_si256(vec1d->avx2, vec2d->avx2); \
+		return vec1d->vec; \
+	} \
+	\
+	static v##sign##int##bits##x##size v##sign##int##bits##x##size##_avx2_or(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2) \
+	{ \
+		union v##sign##int##bits##x##size##_impl_data *vec1d = (union v##sign##int##bits##x##size##_impl_data *)&vec1; \
+		union v##sign##int##bits##x##size##_impl_data *vec2d = (union v##sign##int##bits##x##size##_impl_data *)&vec2; \
+	\
+		vec1d->avx2 = _mm256_or_si256(vec1d->avx2, vec2d->avx2); \
+		return vec1d->vec; \
+	} \
+	\
+	static v##sign##int##bits##x##size v##sign##int##bits##x##size##_avx2_xor(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2) \
+	{ \
+		union v##sign##int##bits##x##size##_impl_data *vec1d = (union v##sign##int##bits##x##size##_impl_data *)&vec1; \
+		union v##sign##int##bits##x##size##_impl_data *vec2d = (union v##sign##int##bits##x##size##_impl_data *)&vec2; \
+	\
+		vec1d->avx2 = _mm256_xor_si256(vec1d->avx2, vec2d->avx2); \
+		return vec1d->vec; \
+	} \
+	\
+	const v##sign##int##bits##x##size##_impl v##sign##int##bits##x##size##_impl_avx2 = { \
+		v##sign##int##bits##x##size##_generic_splat, \
+		v##sign##int##bits##x##size##_avx2_load_aligned, \
+		v##sign##int##bits##x##size##_avx2_load, \
+		v##sign##int##bits##x##size##_avx2_store_aligned, \
+		v##sign##int##bits##x##size##_avx2_store, \
+		v##sign##int##bits##x##size##_avx2_add, \
+		v##sign##int##bits##x##size##_avx2_sub, \
+		v##sign##int##bits##x##size##_avx2_mul, \
+		v##sign##int##bits##x##size##_generic_div, \
+		v##sign##int##bits##x##size##_generic_avg, \
+		v##sign##int##bits##x##size##_avx2_and, \
+		v##sign##int##bits##x##size##_avx2_or, \
+		v##sign##int##bits##x##size##_avx2_xor, \
+		v##sign##int##bits##x##size##_generic_not, \
+		v##sign##int##bits##x##size##_generic_lshift, \
+		v##sign##int##bits##x##size##_generic_rshift, \
+		v##sign##int##bits##x##size##_generic_lrshift, \
+		v##sign##int##bits##x##size##_generic_cmplt, \
+		v##sign##int##bits##x##size##_generic_cmple, \
+		v##sign##int##bits##x##size##_generic_cmpeq, \
+		v##sign##int##bits##x##size##_generic_cmpge, \
+		v##sign##int##bits##x##size##_generic_cmpgt, \
+	};
+
+#define VEC_AVX2_DEFINE_OPERATIONS(bits, size) \
+	VEC_AVX2_DEFINE_OPERATIONS_SIGN( , bits, size) \
+	VEC_AVX2_DEFINE_OPERATIONS_SIGN(u, bits, size)
+
+VEC_AVX2_DEFINE_OPERATIONS(8, 32)
+VEC_AVX2_DEFINE_OPERATIONS(16, 16)
+VEC_AVX2_DEFINE_OPERATIONS(32, 8)
+VEC_AVX2_DEFINE_OPERATIONS(64, 4)