/**
 * vec - a tiny SIMD vector library in C99
 * 
 * Copyright (c) 2024 Paper
 * 
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 * 
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 * 
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
**/

#ifndef VEC_VEC_H_
#define VEC_VEC_H_

#ifdef __cplusplus
extern "C" {
#endif


#ifdef VEC_HAVE_IMPL_INTEGER_H
# include "impl/integer.h"
#else
# if __cplusplus >= (201103L)
#  include <cstdint>
#  include <cstddef>
typedef std::size_t    vec_uintsize;

typedef std::uint8_t   vec_uint8;
typedef std::uint16_t  vec_uint16;
typedef std::uint32_t  vec_uint32;
typedef std::uint64_t  vec_uint64;
typedef std::uintmax_t vec_uintmax;
typedef std::uintptr_t vec_uintptr;

typedef std::int8_t    vec_int8;
typedef std::int16_t   vec_int16;
typedef std::int32_t   vec_int32;
typedef std::int64_t   vec_int64;
typedef std::intmax_t  vec_intmax;
# elif __STDC_VERSION__ >= 199901L
#  include <stdint.h>
#  include <stddef.h>
typedef uint8_t   vec_uint8;
typedef uint16_t  vec_uint16;
typedef uint32_t  vec_uint32;
typedef uint64_t  vec_uint64;
typedef uintmax_t vec_uintmax;
typedef uintptr_t vec_uintptr;
typedef size_t    vec_uintsize;
typedef int8_t    vec_int8;
typedef int16_t   vec_int16;
typedef int32_t   vec_int32;
typedef int64_t   vec_int64;
typedef intmax_t  vec_intmax;
# else
#  error Unable to find integer types with known size.
# endif
#endif

#define VEC_SEMVER_ATLEAST(a, b, c, x, y, z) \
	(((a) >= (x)) && \
	 ((a) > x || (b) >= (y)) && \
	 ((a) > x || (b) > (y) || (c) >= (z)))

#ifdef __GNUC__
# define VEC_GNUC_ATLEAST(x, y, z) \
	VEC_SEMVER_ATLEAST(__GNUC__, __GNUC_MINOR__, __GNUC_PATCHLEVEL__, x, y, z)
#else
# define VEC_GNUC_ATLEAST(x, y, z) (0)
#endif

/* GCC/clang attributes */
#if defined(__has_attribute)
# define VEC_GNUC_HAS_ATTRIBUTE(x, major, minor, patch) __has_attribute(x)
#else
# define VEC_GNUC_HAS_ATTRIBUTE(x, major, minor, patch) VEC_GNUC_ATLEAST(major, minor, patch)
#endif

#if (__cplusplus >= 201103L) || (__STDC_VERSION__ >= 202311L)
# define VEC_STATIC_ASSERT(x, msg) static_assert(x, msg)
#elif (__STDC_VERSION__ >= 201112L)
# define VEC_STATIC_ASSERT(x, msg) _Static_assert(x, msg)
#else
# define VEC_STATIC_ASSERT(x, msg) \
	extern int (*vec_impl_Static_assert_function_(void)) \
		[!!sizeof (struct { int __error_if_negative: (x) ? 2 : -1; })]
#endif

#ifndef VEC_ASSERT
# ifndef VEC_DISABLE_ASSERTIONS
#  include <assert.h>
#  define VEC_ASSERT(x, msg) assert(msg && x)
# else
#  define VEC_ASSERT(x, msg)
# endif
#endif

/* --------------------------------------------------------------- */
/* Detect compiler SIMD support */

#define VEC_NEON_ALIGNMENT    16
#define VEC_ALTIVEC_ALIGNMENT 16
#define VEC_SSE2_ALIGNMENT    16
#define VEC_AVX2_ALIGNMENT    32
#define VEC_AVX512F_ALIGNMENT 64

// for the generic implementation
#define VINT8x2_ALIGNMENT   1
#define VUINT8x2_ALIGNMENT  1

#define VINT8x4_ALIGNMENT   VINT8x2_ALIGNMENT
#define VINT16x2_ALIGNMENT  2
#define VUINT8x4_ALIGNMENT  VUINT8x2_ALIGNMENT
#define VUINT16x2_ALIGNMENT 2

#define VINT8x8_ALIGNMENT   VINT8x4_ALIGNMENT
#define VINT16x4_ALIGNMENT  VINT16x2_ALIGNMENT
#define VINT32x2_ALIGNMENT  4
#define VUINT8x8_ALIGNMENT  VUINT8x4_ALIGNMENT
#define VUINT16x4_ALIGNMENT VUINT16x2_ALIGNMENT
#define VUINT32x2_ALIGNMENT 4

#define VINT8x16_ALIGNMENT  VINT8x8_ALIGNMENT
#define VINT16x8_ALIGNMENT  VINT16x4_ALIGNMENT
#define VINT32x4_ALIGNMENT  VINT32x2_ALIGNMENT
#define VINT64x2_ALIGNMENT  8
#define VUINT8x16_ALIGNMENT VUINT8x8_ALIGNMENT
#define VUINT16x8_ALIGNMENT VUINT16x4_ALIGNMENT
#define VUINT32x4_ALIGNMENT VUINT32x2_ALIGNMENT
#define VUINT64x2_ALIGNMENT 8

#define VINT8x32_ALIGNMENT   VINT8x16_ALIGNMENT
#define VINT16x16_ALIGNMENT  VINT16x8_ALIGNMENT
#define VINT32x8_ALIGNMENT   VINT32x4_ALIGNMENT
#define VINT64x4_ALIGNMENT   VINT64x2_ALIGNMENT
#define VUINT8x32_ALIGNMENT  VUINT8x16_ALIGNMENT
#define VUINT16x16_ALIGNMENT VUINT16x8_ALIGNMENT
#define VUINT32x8_ALIGNMENT  VUINT32x4_ALIGNMENT
#define VUINT64x4_ALIGNMENT  VUINT64x2_ALIGNMENT

#define VINT8x64_ALIGNMENT VINT8x32_ALIGNMENT
#define VINT16x32_ALIGNMENT VINT16x16_ALIGNMENT
#define VINT32x16_ALIGNMENT VINT32x8_ALIGNMENT
#define VINT64x8_ALIGNMENT VINT64x4_ALIGNMENT
#define VUINT8x64_ALIGNMENT VUINT8x32_ALIGNMENT
#define VUINT16x32_ALIGNMENT VUINT16x16_ALIGNMENT
#define VUINT32x16_ALIGNMENT VUINT32x8_ALIGNMENT
#define VUINT64x8_ALIGNMENT VUINT64x4_ALIGNMENT

#ifndef VEC_SUPPRESS_HW

// IIRC `__VEC__' is also defined, but I don't know for sure.
// IBM says that `__ALTIVEC__' is standard though.
#ifdef __ALTIVEC__
# include <altivec.h>
# define VEC_COMPILER_HAS_ALTIVEC
# if defined(__POWER8__) && defined(__VSX__)
#  define VEC_COMPILER_HAS_ALTIVEC_VSX
# endif
# if VINT8x16_ALIGNMENT < VEC_ALTIVEC_ALIGNMENT
#  undef VINT8x16_ALIGNMENT
#  define VINT8x16_ALIGNMENT VEC_ALTIVEC_ALIGNMENT
# endif
# if VINT16x8_ALIGNMENT < VEC_ALTIVEC_ALIGNMENT
#  undef VINT16x8_ALIGNMENT
#  define VINT16x8_ALIGNMENT VEC_ALTIVEC_ALIGNMENT
# endif
# if VINT32x4_ALIGNMENT < VEC_ALTIVEC_ALIGNMENT
#  undef VINT32x4_ALIGNMENT
#  define VINT32x4_ALIGNMENT VEC_ALTIVEC_ALIGNMENT
# endif
# if VINT64x2_ALIGNMENT < VEC_ALTIVEC_ALIGNMENT
#  undef VINT64x2_ALIGNMENT
#  define VINT64x2_ALIGNMENT VEC_ALTIVEC_ALIGNMENT
# endif
# if VUINT8x16_ALIGNMENT < VEC_ALTIVEC_ALIGNMENT
#  undef VUINT8x16_ALIGNMENT
#  define VUINT8x16_ALIGNMENT VEC_ALTIVEC_ALIGNMENT
# endif
# if VUINT16x8_ALIGNMENT < VEC_ALTIVEC_ALIGNMENT
#  undef VUINT16x8_ALIGNMENT
#  define VUINT16x8_ALIGNMENT VEC_ALTIVEC_ALIGNMENT
# endif
# if VUINT32x4_ALIGNMENT < VEC_ALTIVEC_ALIGNMENT
#  undef VUINT32x4_ALIGNMENT
#  define VUINT32x4_ALIGNMENT VEC_ALTIVEC_ALIGNMENT
# endif
# if VUINT64x2_ALIGNMENT < VEC_ALTIVEC_ALIGNMENT
#  undef VUINT64x2_ALIGNMENT
#  define VUINT64x2_ALIGNMENT VEC_ALTIVEC_ALIGNMENT
# endif
#endif

#ifdef __ARM_NEON
# include <arm_neon.h>
# define VEC_COMPILER_HAS_NEON
# if VINT8x8_ALIGNMENT < VEC_NEON_ALIGNMENT
#  undef VINT8x8_ALIGNMENT
#  define VINT8x8_ALIGNMENT VEC_NEON_ALIGNMENT
# endif
# if VINT16x4_ALIGNMENT < VEC_NEON_ALIGNMENT
#  undef VINT16x4_ALIGNMENT
#  define VINT16x4_ALIGNMENT VEC_NEON_ALIGNMENT
# endif
# if VINT32x4_ALIGNMENT < VEC_NEON_ALIGNMENT
#  undef VINT32x4_ALIGNMENT
#  define VINT32x4_ALIGNMENT VEC_NEON_ALIGNMENT
# endif
# if VUINT8x8_ALIGNMENT < VEC_NEON_ALIGNMENT
#  undef VUINT8x8_ALIGNMENT
#  define VUINT8x8_ALIGNMENT VEC_NEON_ALIGNMENT
# endif
# if VUINT16x4_ALIGNMENT < VEC_NEON_ALIGNMENT
#  undef VUINT16x4_ALIGNMENT
#  define VUINT16x4_ALIGNMENT VEC_NEON_ALIGNMENT
# endif
# if VUINT32x4_ALIGNMENT < VEC_NEON_ALIGNMENT
#  undef VUINT32x4_ALIGNMENT
#  define VUINT32x4_ALIGNMENT VEC_NEON_ALIGNMENT
# endif
# if VINT8x16_ALIGNMENT < VEC_NEON_ALIGNMENT
#  undef VINT8x16_ALIGNMENT
#  define VINT8x16_ALIGNMENT VEC_NEON_ALIGNMENT
# endif
# if VINT16x8_ALIGNMENT < VEC_NEON_ALIGNMENT
#  undef VINT16x8_ALIGNMENT
#  define VINT16x8_ALIGNMENT VEC_NEON_ALIGNMENT
# endif
# if VINT32x4_ALIGNMENT < VEC_NEON_ALIGNMENT
#  undef VINT32x4_ALIGNMENT
#  define VINT32x4_ALIGNMENT VEC_NEON_ALIGNMENT
# endif
# if VINT64x2_ALIGNMENT < VEC_NEON_ALIGNMENT
#  undef VINT64x2_ALIGNMENT
#  define VINT64x2_ALIGNMENT VEC_NEON_ALIGNMENT
# endif
# if VUINT8x16_ALIGNMENT < VEC_NEON_ALIGNMENT
#  undef VUINT8x16_ALIGNMENT
#  define VUINT8x16_ALIGNMENT VEC_NEON_ALIGNMENT
# endif
# if VUINT16x8_ALIGNMENT < VEC_NEON_ALIGNMENT
#  undef VUINT16x8_ALIGNMENT
#  define VUINT16x8_ALIGNMENT VEC_NEON_ALIGNMENT
# endif
# if VUINT32x4_ALIGNMENT < VEC_NEON_ALIGNMENT
#  undef VUINT32x4_ALIGNMENT
#  define VUINT32x4_ALIGNMENT VEC_NEON_ALIGNMENT
# endif
# if VUINT64x2_ALIGNMENT < VEC_NEON_ALIGNMENT
#  undef VUINT64x2_ALIGNMENT
#  define VUINT64x2_ALIGNMENT VEC_NEON_ALIGNMENT
# endif
#endif

#ifdef __MMX__
# include <mmintrin.h>
# define VEC_COMPILER_HAS_MMX
#endif

#ifdef __SSE2__
# include <emmintrin.h>
# define VEC_COMPILER_HAS_SSE2
# ifdef __SSE4_1__
#  define VEC_COMPILER_HAS_SSE41
# endif
# if VINT8x16_ALIGNMENT < VEC_SSE2_ALIGNMENT
#  undef VINT8x16_ALIGNMENT
#  define VINT8x16_ALIGNMENT VEC_SSE2_ALIGNMENT
# endif
# if VINT16x8_ALIGNMENT < VEC_SSE2_ALIGNMENT
#  undef VINT16x8_ALIGNMENT
#  define VINT16x8_ALIGNMENT VEC_SSE2_ALIGNMENT
# endif
# if VINT32x4_ALIGNMENT < VEC_SSE2_ALIGNMENT
#  undef VINT32x4_ALIGNMENT
#  define VINT32x4_ALIGNMENT VEC_SSE2_ALIGNMENT
# endif
# if VINT64x2_ALIGNMENT < VEC_SSE2_ALIGNMENT
#  undef VINT64x2_ALIGNMENT
#  define VINT64x2_ALIGNMENT VEC_SSE2_ALIGNMENT
# endif
# if VUINT8x16_ALIGNMENT < VEC_SSE2_ALIGNMENT
#  undef VUINT8x16_ALIGNMENT
#  define VUINT8x16_ALIGNMENT VEC_SSE2_ALIGNMENT
# endif
# if VUINT16x8_ALIGNMENT < VEC_SSE2_ALIGNMENT
#  undef VUINT16x8_ALIGNMENT
#  define VUINT16x8_ALIGNMENT VEC_SSE2_ALIGNMENT
# endif
# if VUINT32x4_ALIGNMENT < VEC_SSE2_ALIGNMENT
#  undef VUINT32x4_ALIGNMENT
#  define VUINT32x4_ALIGNMENT VEC_SSE2_ALIGNMENT
# endif
# if VUINT64x2_ALIGNMENT < VEC_SSE2_ALIGNMENT
#  undef VUINT64x2_ALIGNMENT
#  define VUINT64x2_ALIGNMENT VEC_SSE2_ALIGNMENT
# endif
#endif

#ifdef __AVX2__
# include <immintrin.h>
# define VEC_COMPILER_HAS_AVX2
# if VINT8x32_ALIGNMENT < VEC_AVX2_ALIGNMENT
#  undef VINT8x32_ALIGNMENT
#  define VINT8x32_ALIGNMENT VEC_AVX2_ALIGNMENT
# endif
# if VINT16x16_ALIGNMENT < VEC_AVX2_ALIGNMENT
#  undef VINT16x16_ALIGNMENT
#  define VINT16x16_ALIGNMENT VEC_AVX2_ALIGNMENT
# endif
# if VINT32x8_ALIGNMENT < VEC_AVX2_ALIGNMENT
#  undef VINT32x8_ALIGNMENT
#  define VINT32x8_ALIGNMENT VEC_AVX2_ALIGNMENT
# endif
# if VINT64x4_ALIGNMENT < VEC_AVX2_ALIGNMENT
#  undef VINT64x4_ALIGNMENT
#  define VINT64x4_ALIGNMENT VEC_AVX2_ALIGNMENT
# endif
# if VUINT8x32_ALIGNMENT < VEC_AVX2_ALIGNMENT
#  undef VUINT8x32_ALIGNMENT
#  define VUINT8x32_ALIGNMENT VEC_AVX2_ALIGNMENT
# endif
# if VUINT16x16_ALIGNMENT < VEC_AVX2_ALIGNMENT
#  undef VUINT16x16_ALIGNMENT
#  define VUINT16x16_ALIGNMENT VEC_AVX2_ALIGNMENT
# endif
# if VUINT32x8_ALIGNMENT < VEC_AVX2_ALIGNMENT
#  undef VUINT32x8_ALIGNMENT
#  define VUINT32x8_ALIGNMENT VEC_AVX2_ALIGNMENT
# endif
# if VUINT64x4_ALIGNMENT < VEC_AVX2_ALIGNMENT
#  undef VUINT64x4_ALIGNMENT
#  define VUINT64x4_ALIGNMENT VEC_AVX2_ALIGNMENT
# endif
#endif

#ifdef __AVX512F__
# include <immintrin.h>
# define VEC_COMPILER_HAS_AVX512F
# if VINT8x64_ALIGNMENT < VEC_AVX512F_ALIGNMENT
#  undef VINT8x64_ALIGNMENT
#  define VINT8x64_ALIGNMENT VEC_AVX512F_ALIGNMENT
# endif
# if VINT16x32_ALIGNMENT < VEC_AVX512F_ALIGNMENT
#  undef VINT16x32_ALIGNMENT
#  define VINT16x32_ALIGNMENT VEC_AVX512F_ALIGNMENT
# endif
# if VINT32x16_ALIGNMENT < VEC_AVX512F_ALIGNMENT
#  undef VINT32x16_ALIGNMENT
#  define VINT32x16_ALIGNMENT VEC_AVX512F_ALIGNMENT
# endif
# if VINT64x8_ALIGNMENT < VEC_AVX512F_ALIGNMENT
#  undef VINT64x8_ALIGNMENT
#  define VINT64x8_ALIGNMENT VEC_AVX512F_ALIGNMENT
# endif
# if VUINT8x64_ALIGNMENT < VEC_AVX512F_ALIGNMENT
#  undef VUINT8x64_ALIGNMENT
#  define VUINT8x64_ALIGNMENT VEC_AVX512F_ALIGNMENT
# endif
# if VUINT16x32_ALIGNMENT < VEC_AVX512F_ALIGNMENT
#  undef VUINT16x32_ALIGNMENT
#  define VUINT16x32_ALIGNMENT VEC_AVX512F_ALIGNMENT
# endif
# if VUINT32x16_ALIGNMENT < VEC_AVX512F_ALIGNMENT
#  undef VUINT32x16_ALIGNMENT
#  define VUINT32x16_ALIGNMENT VEC_AVX512F_ALIGNMENT
# endif
# if VUINT64x8_ALIGNMENT < VEC_AVX512F_ALIGNMENT
#  undef VUINT64x8_ALIGNMENT
#  define VUINT64x8_ALIGNMENT VEC_AVX512F_ALIGNMENT
# endif
#endif

#endif

/* --------------------------------------------------------------- */
/* bit shift */

inline vec_uintmax vec_ulrshift(vec_uintmax x, unsigned int y)
{
	return x >> y;
}

inline vec_uintmax vec_ullshift(vec_uintmax x, unsigned int y)
{
	return x << y;
}

inline vec_intmax vec_lrshift(vec_intmax x, unsigned int y)
{
	// reinterpret as unsigned integer and then shift
	union {
		vec_intmax d;
		vec_uintmax u;
	} xx;

	xx.d = x;
	xx.u >>= y;
	return xx.d;
}

inline vec_intmax vec_llshift(vec_intmax x, unsigned int y)
{
	// reinterpret as unsigned integer and then shift
	union {
		vec_intmax d;
		vec_uintmax u;
	} xx;

	xx.d = x;
	xx.u <<= y;
	return xx.d;
}

inline vec_uintmax vec_urshift(vec_uintmax x, unsigned int y)
{
	return x >> y;
}

inline vec_uintmax vec_ulshift(vec_uintmax x, unsigned int y)
{
	return x << y;
}

/**
 * Arithmetic shifts; based off code from OpenMPT, which is under
 * the Boost Software License:
 *
 * Permission is hereby granted, free of charge, to any person or organization
 * obtaining a copy of the software and accompanying documentation covered by
 * this license (the "Software") to use, reproduce, display, distribute,
 * execute, and transmit the Software, and to prepare derivative works of the
 * Software, and to permit third-parties to whom the Software is furnished to
 * do so, all subject to the following:
 * 
 * The copyright notices in the Software and this entire statement, including
 * the above license grant, this restriction and the following disclaimer,
 * must be included in all copies of the Software, in whole or in part, and
 * all derivative works of the Software, unless such copies or derivative
 * works are solely in the form of machine-executable object code generated by
 * a source language processor.
 * 
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
 * SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
 * FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
**/
inline vec_intmax vec_rshift(vec_intmax x, unsigned int y)
{
	static const vec_uintmax roffset = ((vec_uintmax)1) << ((sizeof(vec_intmax) * 8) - 1);

	union {
		vec_intmax d;
		vec_uintmax u;
	} xx;

	xx.d = x;

	xx.u += roffset;
	xx.u >>= y;
	xx.u -= roffset >> y;

	return xx.d;
}

inline vec_intmax vec_lshift(vec_intmax x, unsigned int y)
{
	static const vec_uintmax roffset = ((vec_uintmax)1) << ((sizeof(vec_intmax) * 8) - 1);

	union {
		vec_intmax d;
		vec_uintmax u;
	} xx;

	xx.d = x;

	xx.u += roffset;
	xx.u <<= y;
	xx.u -= roffset << y;

	return xx.d;
}

#ifdef VEC_IMPLEMENTATION
extern inline vec_uintmax vec_ulrshift(vec_uintmax x, unsigned int y);
extern inline vec_uintmax vec_ullshift(vec_uintmax x, unsigned int y);
extern inline vec_intmax vec_lrshift(vec_intmax x, unsigned int y);
extern inline vec_intmax vec_llshift(vec_intmax x, unsigned int y);
extern inline vec_uintmax vec_urshift(vec_uintmax x, unsigned int y);
extern inline vec_uintmax vec_ulshift(vec_uintmax x, unsigned int y);
extern inline vec_intmax vec_rshift(vec_intmax x, unsigned int y);
extern inline vec_intmax vec_lshift(vec_intmax x, unsigned int y);
#endif

/* --------------------------------------------------------------- */

#include "impl/align.h"

/* --------------------------------------------------------------- */
/* Defines the structures for each vector type */

// 16-bit
typedef union {
	vec_uint8 generic[2];
} vuint8x2;

typedef union {
	vec_int8 generic[2];
} vint8x2;

// 32-bit
typedef union {
	vuint8x2 generic[2];
} vuint8x4;

typedef union {
	vec_uint16 generic[2];
} vuint16x2;

typedef union {
	vint8x2 generic[2];
} vint8x4;

typedef union {
	vec_int16 generic[2];
} vint16x2;

// 64-bit
typedef union {
#ifdef VEC_COMPILER_HAS_MMX
	__m64 mmx;
#endif
#ifdef VEC_COMPILER_HAS_NEON
	uint8x8_t neon;
#endif

	vuint8x4 generic[2];
} vuint8x8;

typedef union {
#ifdef VEC_COMPILER_HAS_MMX
	__m64 mmx;
#endif
#ifdef VEC_COMPILER_HAS_NEON
	uint16x4_t neon;
#endif

	vuint16x2 generic[2];
} vuint16x4;

typedef union {
#ifdef VEC_COMPILER_HAS_MMX
	__m64 mmx;
#endif
#ifdef VEC_COMPILER_HAS_NEON
	uint32x2_t neon;
#endif

	vec_uint32 generic[2];
} vuint32x2;

typedef union {
#ifdef VEC_COMPILER_HAS_MMX
	__m64 mmx;
#endif
#ifdef VEC_COMPILER_HAS_NEON
	int8x8_t neon;
#endif

	vint8x4 generic[2];
} vint8x8;

typedef union {
#ifdef VEC_COMPILER_HAS_MMX
	__m64 mmx;
#endif
#ifdef VEC_COMPILER_HAS_NEON
	int16x4_t neon;
#endif

	vint16x2 generic[2];
} vint16x4;

typedef union {
#ifdef VEC_COMPILER_HAS_MMX
	__m64 mmx;
#endif
#ifdef VEC_COMPILER_HAS_NEON
	int32x2_t neon;
#endif

	vec_int32 generic[2];
} vint32x2;

// 128-bit
typedef union {
#ifdef VEC_COMPILER_HAS_SSE2
	__m128i sse;
#endif
#ifdef VEC_COMPILER_HAS_ALTIVEC
	vector unsigned char altivec;
#endif
#ifdef VEC_COMPILER_HAS_NEON
	uint8x16_t neon;
#endif
	vuint8x8 generic[2];
} vuint8x16;

typedef union {
#ifdef VEC_COMPILER_HAS_SSE2
	__m128i sse;
#endif
#ifdef VEC_COMPILER_HAS_ALTIVEC
	vector unsigned short altivec;
#endif
#ifdef VEC_COMPILER_HAS_NEON
	uint16x8_t neon;
#endif
	vuint16x4 generic[2];
} vuint16x8;

typedef union {
#ifdef VEC_COMPILER_HAS_SSE2
	__m128i sse;
#endif
#ifdef VEC_COMPILER_HAS_ALTIVEC
	vector unsigned int altivec;
#endif
#ifdef VEC_COMPILER_HAS_NEON
	uint32x4_t neon;
#endif
	vuint32x2 generic[2];
} vuint32x4;

typedef union {
#ifdef VEC_COMPILER_HAS_SSE2
	__m128i sse;
#endif
#ifdef VEC_COMPILER_HAS_ALTIVEC_VSX
	vector unsigned long long altivec;
#endif
#ifdef VEC_COMPILER_HAS_NEON
	uint64x2_t neon;
#endif
	vec_uint64 generic[2];
} vuint64x2;

typedef union {
#ifdef VEC_COMPILER_HAS_SSE2
	__m128i sse;
#endif
#ifdef VEC_COMPILER_HAS_ALTIVEC
	vector signed char altivec;
#endif
#ifdef VEC_COMPILER_HAS_NEON
	int8x16_t neon;
#endif
	vint8x8 generic[2];
} vint8x16;

typedef union {
#ifdef VEC_COMPILER_HAS_SSE2
	__m128i sse;
#endif
#ifdef VEC_COMPILER_HAS_ALTIVEC
	vector signed short altivec;
#endif
#ifdef VEC_COMPILER_HAS_NEON
	int16x8_t neon;
#endif
	vint16x4 generic[2];
} vint16x8;

typedef union {
#ifdef VEC_COMPILER_HAS_SSE2
	__m128i sse;
#endif
#ifdef VEC_COMPILER_HAS_ALTIVEC
	vector signed int altivec;
#endif
#ifdef VEC_COMPILER_HAS_NEON
	int32x4_t neon;
#endif
	vint32x2 generic[2];
} vint32x4;

typedef union {
#ifdef VEC_COMPILER_HAS_SSE2
	__m128i sse;
#endif
#ifdef VEC_COMPILER_HAS_ALTIVEC_VSX
	vector signed long long altivec;
#endif
#ifdef VEC_COMPILER_HAS_NEON
	int64x2_t neon;
#endif
	vec_int64 generic[2];
} vint64x2;

// 256-bit
typedef union {
#ifdef VEC_COMPILER_HAS_AVX2
	__m256i avx2;
#endif
	vuint8x16 generic[2];
} vuint8x32;

typedef union {
#ifdef VEC_COMPILER_HAS_AVX2
	__m256i avx2;
#endif
	vuint16x8 generic[2];
} vuint16x16;

typedef union {
#ifdef VEC_COMPILER_HAS_AVX2
	__m256i avx2;
#endif
	vuint32x4 generic[2];
} vuint32x8;

typedef union {
#ifdef VEC_COMPILER_HAS_AVX2
	__m256i avx2;
#endif
	vuint64x2 generic[2];
} vuint64x4;

typedef union {
#ifdef VEC_COMPILER_HAS_AVX2
	__m256i avx2;
#endif
	vint8x16 generic[2];
} vint8x32;

typedef union {
#ifdef VEC_COMPILER_HAS_AVX2
	__m256i avx2;
#endif
	vint16x8 generic[2];
} vint16x16;

typedef union {
#ifdef VEC_COMPILER_HAS_AVX2
	__m256i avx2;
#endif
	vint32x4 generic[2];
} vint32x8;

typedef union {
#ifdef VEC_COMPILER_HAS_AVX2
	__m256i avx2;
#endif
	vint64x2 generic[2];
} vint64x4;

// 512-bit
typedef union {
#ifdef VEC_COMPILER_HAS_AVX512F
	__m512i avx512f;
#endif
	vuint8x32 generic[2];
} vuint8x64;

typedef union {
#ifdef VEC_COMPILER_HAS_AVX512F
	__m512i avx512f;
#endif
	vuint16x16 generic[2];
} vuint16x32;

typedef union {
#ifdef VEC_COMPILER_HAS_AVX512F
	__m512i avx512f;
#endif
	vuint32x8 generic[2];
} vuint32x16;

typedef union {
#ifdef VEC_COMPILER_HAS_AVX512F
	__m512i avx512f;
#endif
	vuint64x4 generic[2];
} vuint64x8;

typedef union {
#ifdef VEC_COMPILER_HAS_AVX512F
	__m512i avx512f;
#endif
	vint8x32 generic[2];
} vint8x64;

typedef union {
#ifdef VEC_COMPILER_HAS_AVX512F
	__m512i avx512f;
#endif
	vint16x16 generic[2];
} vint16x32;

typedef union {
#ifdef VEC_COMPILER_HAS_AVX512F
	__m512i avx512f;
#endif
	vint32x8 generic[2];
} vint32x16;

typedef union {
#ifdef VEC_COMPILER_HAS_AVX512F
	__m512i avx512f;
#endif
	vint64x4 generic[2];
} vint64x8;

// ---------------------------------------------------------------------------------
// function declarations

int vec_init(void);

#define VEC_DECLARE_OPERATIONS_SIGN(sign, bits, size) \
	v##sign##int##bits##x##size v##sign##int##bits##x##size##_splat(vec_##sign##int##bits x); \
	v##sign##int##bits##x##size v##sign##int##bits##x##size##_load_aligned(const vec_##sign##int##bits in[size]); \
	v##sign##int##bits##x##size v##sign##int##bits##x##size##_load(const vec_##sign##int##bits in[size]); \
	void v##sign##int##bits##x##size##_store_aligned(v##sign##int##bits##x##size vec, vec_##sign##int##bits out[size]); \
	void v##sign##int##bits##x##size##_store(v##sign##int##bits##x##size vec, vec_##sign##int##bits out[size]); \
	v##sign##int##bits##x##size v##sign##int##bits##x##size##_add(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2); \
	v##sign##int##bits##x##size v##sign##int##bits##x##size##_sub(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2); \
	v##sign##int##bits##x##size v##sign##int##bits##x##size##_mul(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2); \
	v##sign##int##bits##x##size v##sign##int##bits##x##size##_div(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2); \
	v##sign##int##bits##x##size v##sign##int##bits##x##size##_avg(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2); \
	v##sign##int##bits##x##size v##sign##int##bits##x##size##_and(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2); \
	v##sign##int##bits##x##size v##sign##int##bits##x##size##_or(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2); \
	v##sign##int##bits##x##size v##sign##int##bits##x##size##_xor(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2); \
	v##sign##int##bits##x##size v##sign##int##bits##x##size##_cmplt(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2); \
	v##sign##int##bits##x##size v##sign##int##bits##x##size##_cmple(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2); \
	v##sign##int##bits##x##size v##sign##int##bits##x##size##_cmpeq(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2); \
	v##sign##int##bits##x##size v##sign##int##bits##x##size##_cmpge(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2); \
	v##sign##int##bits##x##size v##sign##int##bits##x##size##_cmpgt(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2); \
	v##sign##int##bits##x##size v##sign##int##bits##x##size##_lshift(v##sign##int##bits##x##size vec1, vuint##bits##x##size vec2); \
	v##sign##int##bits##x##size v##sign##int##bits##x##size##_rshift(v##sign##int##bits##x##size vec1, vuint##bits##x##size vec2); \
	v##sign##int##bits##x##size v##sign##int##bits##x##size##_lrshift(v##sign##int##bits##x##size vec1, vuint##bits##x##size vec2);

#define VEC_DECLARE_OPERATIONS(bits, size) \
	VEC_DECLARE_OPERATIONS_SIGN( , bits, size) \
	VEC_DECLARE_OPERATIONS_SIGN(u, bits, size)

// 16-bit
VEC_DECLARE_OPERATIONS(8, 2)

// 32-bit
VEC_DECLARE_OPERATIONS(8, 4)
VEC_DECLARE_OPERATIONS(16, 2)

// 64-bit
VEC_DECLARE_OPERATIONS(8, 8)
VEC_DECLARE_OPERATIONS(16, 4)
VEC_DECLARE_OPERATIONS(32, 2)

// 128-bit
VEC_DECLARE_OPERATIONS(8, 16)
VEC_DECLARE_OPERATIONS(16, 8)
VEC_DECLARE_OPERATIONS(32, 4)
VEC_DECLARE_OPERATIONS(64, 2)

// 256-bit
VEC_DECLARE_OPERATIONS(8, 32)
VEC_DECLARE_OPERATIONS(16, 16)
VEC_DECLARE_OPERATIONS(32, 8)
VEC_DECLARE_OPERATIONS(64, 4)

// 512-bit
VEC_DECLARE_OPERATIONS(8, 64)
VEC_DECLARE_OPERATIONS(16, 32)
VEC_DECLARE_OPERATIONS(32, 16)
VEC_DECLARE_OPERATIONS(64, 8)

#undef VEC_DECLARE_OPERATIONS
#undef VEC_DECLARE_OPERATIONS_SIGN

// ---------------------------------------------------------------------------------
// okay, now we can actually implement the functions

#ifdef VEC_IMPLEMENTATION

// Fallback functions, need to be defined before everything else.
#include "impl/fallback.h"

// okay, these are filled in for each supported backend.
// `and', `or', `xor', and `nor' have to be prefixed with
// `b' because of <iso646.h>
#define VEC_DEFINE_IMPL_STRUCT_SIGN(sign, bits, size) \
	typedef struct { \
		v##sign##int##bits##x##size (*splat)(vec_##sign##int##bits x); \
		v##sign##int##bits##x##size (*load_aligned)(const vec_##sign##int##bits in[size]); \
		v##sign##int##bits##x##size (*load)(const vec_##sign##int##bits in[size]); \
		void (*store_aligned)(v##sign##int##bits##x##size vec, vec_##sign##int##bits out[size]); \
		void (*store)(v##sign##int##bits##x##size vec, vec_##sign##int##bits out[size]); \
		v##sign##int##bits##x##size (*add)(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2); \
		v##sign##int##bits##x##size (*sub)(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2); \
		v##sign##int##bits##x##size (*mul)(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2); \
		v##sign##int##bits##x##size (*div)(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2); \
		v##sign##int##bits##x##size (*avg)(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2); \
		v##sign##int##bits##x##size (*band)(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2); \
		v##sign##int##bits##x##size (*bor)(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2); \
		v##sign##int##bits##x##size (*bxor)(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2); \
		v##sign##int##bits##x##size (*bnot)(v##sign##int##bits##x##size vec); \
		v##sign##int##bits##x##size (*lshift)(v##sign##int##bits##x##size vec1, vuint##bits##x##size vec2); \
		v##sign##int##bits##x##size (*rshift)(v##sign##int##bits##x##size vec1, vuint##bits##x##size vec2); \
		v##sign##int##bits##x##size (*lrshift)(v##sign##int##bits##x##size vec1, vuint##bits##x##size vec2); \
		v##sign##int##bits##x##size (*cmplt)(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2); \
		v##sign##int##bits##x##size (*cmple)(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2); \
		v##sign##int##bits##x##size (*cmpeq)(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2); \
		v##sign##int##bits##x##size (*cmpge)(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2); \
		v##sign##int##bits##x##size (*cmpgt)(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2); \
	} v##sign##int##bits##x##size##_impl;

#define VEC_DEFINE_IMPL_STRUCT(bits, size) \
	VEC_DEFINE_IMPL_STRUCT_SIGN( , bits, size) \
	VEC_DEFINE_IMPL_STRUCT_SIGN(u, bits, size)

// 16-bit
VEC_DEFINE_IMPL_STRUCT(8, 2)

// 32-bit
VEC_DEFINE_IMPL_STRUCT(8, 4)
VEC_DEFINE_IMPL_STRUCT(16, 2)

// 64-bit
VEC_DEFINE_IMPL_STRUCT(8, 8)
VEC_DEFINE_IMPL_STRUCT(16, 4)
VEC_DEFINE_IMPL_STRUCT(32, 2)

// 128-bit
VEC_DEFINE_IMPL_STRUCT(8, 16)
VEC_DEFINE_IMPL_STRUCT(16, 8)
VEC_DEFINE_IMPL_STRUCT(32, 4)
VEC_DEFINE_IMPL_STRUCT(64, 2)

// 256-bit
VEC_DEFINE_IMPL_STRUCT(8, 32)
VEC_DEFINE_IMPL_STRUCT(16, 16)
VEC_DEFINE_IMPL_STRUCT(32, 8)
VEC_DEFINE_IMPL_STRUCT(64, 4)

// 512-bit
VEC_DEFINE_IMPL_STRUCT(8, 64)
VEC_DEFINE_IMPL_STRUCT(16, 32)
VEC_DEFINE_IMPL_STRUCT(32, 16)
VEC_DEFINE_IMPL_STRUCT(64, 8)

#undef VEC_DEFINE_IMPL_STRUCT
#undef VEC_DEFINE_IMPL_STRUCT_SIGN

// ------------------------------------------------------------------------

#ifdef VEC_COMPILER_HAS_ALTIVEC
# include "impl/ppc/altivec.h"
#endif

#ifdef VEC_COMPILER_HAS_AVX512F
# include "impl/x86/avx512f.h"
#endif

#ifdef VEC_COMPILER_HAS_AVX2
# include "impl/x86/avx2.h"
#endif

#ifdef VEC_COMPILER_HAS_SSE2
# include "impl/x86/sse2.h"
#endif

// depends on SSE2 functions; the only thing SSE4.1 provides for us
// is a native 32-bit multiply
#ifdef VEC_COMPILER_HAS_SSE41
# include "impl/x86/sse41.h"
#endif

#ifdef VEC_COMPILER_HAS_MMX
# include "impl/x86/mmx.h"
#endif

#ifdef VEC_COMPILER_HAS_NEON
# include "impl/arm/neon.h"
#endif

#include "impl/generic.h"

/* ---------------------------------------------------------------- */

#include "impl/cpu.h" // CPU detection crap

// 16-bit
static vint8x2_impl   *vint8x2_impl_cpu   = &vint8x2_impl_generic;
static vuint8x2_impl  *vuint8x2_impl_cpu  = &vuint8x2_impl_generic;

// 32-bit
static vint8x4_impl   *vint8x4_impl_cpu   = &vint8x4_impl_generic;
static vuint8x4_impl  *vuint8x4_impl_cpu  = &vuint8x4_impl_generic;
static vint16x2_impl  *vint16x2_impl_cpu  = &vint16x2_impl_generic;
static vuint16x2_impl *vuint16x2_impl_cpu = &vuint16x2_impl_generic;

// 64-bit
static vint8x8_impl   *vint8x8_impl_cpu   = &vint8x8_impl_generic;
static vuint8x8_impl  *vuint8x8_impl_cpu  = &vuint8x8_impl_generic;
static vint16x4_impl  *vint16x4_impl_cpu  = &vint16x4_impl_generic;
static vuint16x4_impl *vuint16x4_impl_cpu = &vuint16x4_impl_generic;
static vint32x2_impl  *vint32x2_impl_cpu  = &vint32x2_impl_generic;
static vuint32x2_impl *vuint32x2_impl_cpu = &vuint32x2_impl_generic;

// 128-bit
static vint8x16_impl  *vint8x16_impl_cpu  = &vint8x16_impl_generic;
static vuint8x16_impl *vuint8x16_impl_cpu = &vuint8x16_impl_generic;
static vint16x8_impl  *vint16x8_impl_cpu  = &vint16x8_impl_generic;
static vuint16x8_impl *vuint16x8_impl_cpu = &vuint16x8_impl_generic;
static vint32x4_impl  *vint32x4_impl_cpu  = &vint32x4_impl_generic;
static vuint32x4_impl *vuint32x4_impl_cpu = &vuint32x4_impl_generic;
static vint64x2_impl  *vint64x2_impl_cpu  = &vint64x2_impl_generic;
static vuint64x2_impl *vuint64x2_impl_cpu = &vuint64x2_impl_generic;

// 256-bit
static vint8x32_impl  *vint8x32_impl_cpu  = &vint8x32_impl_generic;
static vuint8x32_impl *vuint8x32_impl_cpu = &vuint8x32_impl_generic;
static vint16x16_impl  *vint16x16_impl_cpu  = &vint16x16_impl_generic;
static vuint16x16_impl *vuint16x16_impl_cpu = &vuint16x16_impl_generic;
static vint32x8_impl  *vint32x8_impl_cpu  = &vint32x8_impl_generic;
static vuint32x8_impl *vuint32x8_impl_cpu = &vuint32x8_impl_generic;
static vint64x4_impl  *vint64x4_impl_cpu  = &vint64x4_impl_generic;
static vuint64x4_impl *vuint64x4_impl_cpu = &vuint64x4_impl_generic;

// 512-bit
static vint8x64_impl  *vint8x64_impl_cpu  = &vint8x64_impl_generic;
static vuint8x64_impl *vuint8x64_impl_cpu = &vuint8x64_impl_generic;
static vint16x32_impl  *vint16x32_impl_cpu  = &vint16x32_impl_generic;
static vuint16x32_impl *vuint16x32_impl_cpu = &vuint16x32_impl_generic;
static vint32x16_impl  *vint32x16_impl_cpu  = &vint32x16_impl_generic;
static vuint32x16_impl *vuint32x16_impl_cpu = &vuint32x16_impl_generic;
static vint64x8_impl  *vint64x8_impl_cpu  = &vint64x8_impl_generic;
static vuint64x8_impl *vuint64x8_impl_cpu = &vuint64x8_impl_generic;

// returns 0 or a negative error code on failure
int vec_init(void)
{
	// This function is NOT thread safe. However, once vec
	// is initialized, all of the vector functions are thread-safe.
	//
	// In fact, it's possible to use vec without calling
	// vec_init() at all, but it would be completely useless since
	// it would just use a generic implementation without any
	// vectorization whatsoever (unless maybe the compiler is
	// smart enough to optimize it into vectors)

	vec_get_CPU_features();

#ifdef VEC_COMPILER_HAS_ALTIVEC
	if (vec_CPU_have_ALTIVEC()) {
		vint8x16_impl_cpu  = &vint8x16_impl_altivec;
		vuint8x16_impl_cpu = &vuint8x16_impl_altivec;
		vint16x8_impl_cpu  = &vint16x8_impl_altivec;
		vuint16x8_impl_cpu = &vuint16x8_impl_altivec;
		vint32x4_impl_cpu  = &vint32x4_impl_altivec;
		vuint32x4_impl_cpu = &vuint32x4_impl_altivec;
#ifdef VEC_COMPILER_HAS_ALTIVEC_VSX
		if (vec_CPU_have_ALTIVEC_VSX()) {
			vint64x2_impl_cpu  = &vint64x2_impl_altivec;
			vuint64x2_impl_cpu = &vuint64x2_impl_altivec;
		}
#endif
	}
#endif
#ifdef VEC_COMPILER_HAS_AVX512F
	if (vec_CPU_have_AVX512F()) {
		vint8x64_impl_cpu  = &vint8x64_impl_avx512f;
		vuint8x64_impl_cpu = &vuint8x64_impl_avx512f;
		vint16x32_impl_cpu  = &vint16x32_impl_avx512f;
		vuint16x32_impl_cpu = &vuint16x32_impl_avx512f;
		vint32x16_impl_cpu  = &vint32x16_impl_avx512f;
		vuint32x16_impl_cpu = &vuint32x16_impl_avx512f;
		vint64x8_impl_cpu  = &vint64x8_impl_avx512f;
		vuint64x8_impl_cpu = &vuint64x8_impl_avx512f;
	}
#endif
#ifdef VEC_COMPILER_HAS_AVX2
	if (vec_CPU_have_AVX2()) {
		vint8x32_impl_cpu  = &vint8x32_impl_avx2;
		vuint8x32_impl_cpu = &vuint8x32_impl_avx2;
		vint16x16_impl_cpu  = &vint16x16_impl_avx2;
		vuint16x16_impl_cpu = &vuint16x16_impl_avx2;
		vint32x8_impl_cpu  = &vint32x8_impl_avx2;
		vuint32x8_impl_cpu = &vuint32x8_impl_avx2;
		vint64x4_impl_cpu  = &vint64x4_impl_avx2;
		vuint64x4_impl_cpu = &vuint64x4_impl_avx2;
	}
#endif
#ifdef VEC_COMPILER_HAS_SSE2
	if (vec_CPU_have_SSE2()) {
		vint8x16_impl_cpu  = &vint8x16_impl_sse2;
		vuint8x16_impl_cpu = &vuint8x16_impl_sse2;
		vint16x8_impl_cpu  = &vint16x8_impl_sse2;
		vuint16x8_impl_cpu = &vuint16x8_impl_sse2;
# ifdef VEC_COMPILER_HAS_SSE41
		if (vec_CPU_have_SSE41()) {
			vint32x4_impl_cpu  = &vint32x4_impl_sse41;
			vuint32x4_impl_cpu = &vuint32x4_impl_sse41;
		} else
# endif
		{
			vint32x4_impl_cpu  = &vint32x4_impl_sse2;
			vuint32x4_impl_cpu = &vuint32x4_impl_sse2;
		}
		vint64x2_impl_cpu  = &vint64x2_impl_sse2;
		vuint64x2_impl_cpu = &vuint64x2_impl_sse2;
	}
#endif
#ifdef VEC_COMPILER_HAS_MMX
	if (vec_CPU_have_MMX()) {
		vint8x8_impl_cpu  = &vint8x8_impl_mmx;
		vuint8x8_impl_cpu = &vuint8x8_impl_mmx;
		vint16x4_impl_cpu  = &vint16x4_impl_mmx;
		vuint16x4_impl_cpu = &vuint16x4_impl_mmx;
		vint32x2_impl_cpu  = &vint32x2_impl_mmx;
		vuint32x2_impl_cpu = &vuint32x2_impl_mmx;
	}
#endif
#ifdef VEC_COMPILER_HAS_NEON
	if (vec_CPU_have_NEON()) {
		// 64-bit
		vint8x8_impl_cpu  = &vint8x8_impl_neon;
		vuint8x8_impl_cpu = &vuint8x8_impl_neon;
		vint16x4_impl_cpu  = &vint16x4_impl_neon;
		vuint16x4_impl_cpu = &vuint16x4_impl_neon;
		vint32x2_impl_cpu  = &vint32x2_impl_neon;
		vuint32x2_impl_cpu = &vuint32x2_impl_neon;

		// 128-bit
		vint8x16_impl_cpu  = &vint8x16_impl_neon;
		vuint8x16_impl_cpu = &vuint8x16_impl_neon;
		vint16x8_impl_cpu  = &vint16x8_impl_neon;
		vuint16x8_impl_cpu = &vuint16x8_impl_neon;
		vint32x4_impl_cpu  = &vint32x4_impl_neon;
		vuint32x4_impl_cpu = &vuint32x4_impl_neon;
		vint64x2_impl_cpu  = &vint64x2_impl_neon;
		vuint64x2_impl_cpu = &vuint64x2_impl_neon;
	}
#endif
	{
		// do nothing, they're already set to generics
	}

	return 0;
}

/* ---------------------------------------------------------------- */

#define VEC_DEFINE_OPERATIONS_SIGN(sign, bits, size) \
	v##sign##int##bits##x##size v##sign##int##bits##x##size##_splat(vec_##sign##int##bits x) \
	{ \
		if (v##sign##int##bits##x##size##_impl_cpu->splat) \
			return v##sign##int##bits##x##size##_impl_cpu->splat(x); \
	\
		return v##sign##int##bits##x##size##_fallback_splat(x); \
	} \
	\
	v##sign##int##bits##x##size v##sign##int##bits##x##size##_load_aligned(const vec_##sign##int##bits in[size]) \
	{ \
		v##sign##int##bits##x##size err = {0}; \
	\
		if (v##sign##int##bits##x##size##_impl_cpu->load_aligned) \
			return v##sign##int##bits##x##size##_impl_cpu->load_aligned(in); \
	\
		VEC_ASSERT(0, "vec: load_aligned is required to be implemented"); \
	\
		return err; \
	} \
	\
	v##sign##int##bits##x##size v##sign##int##bits##x##size##_load(const vec_##sign##int##bits in[size]) \
	{ \
		if (v##sign##int##bits##x##size##_impl_cpu->load) \
			return v##sign##int##bits##x##size##_impl_cpu->load(in); \
	\
		return v##sign##int##bits##x##size##_fallback_load(in); \
	} \
	\
	void v##sign##int##bits##x##size##_store_aligned(v##sign##int##bits##x##size vec, vec_##sign##int##bits out[size]) \
	{ \
		if (v##sign##int##bits##x##size##_impl_cpu->store_aligned) { \
			v##sign##int##bits##x##size##_impl_cpu->store_aligned(vec, out); \
			return; \
		} \
	\
		VEC_ASSERT(0, "vec: store_aligned is required to be implemented"); \
	} \
	\
	void v##sign##int##bits##x##size##_store(v##sign##int##bits##x##size vec, vec_##sign##int##bits out[size]) \
	{ \
		if (v##sign##int##bits##x##size##_impl_cpu->store) { \
			v##sign##int##bits##x##size##_impl_cpu->store(vec, out); \
			return; \
		} \
	\
		v##sign##int##bits##x##size##_fallback_store(vec, out); \
	} \
	\
	v##sign##int##bits##x##size v##sign##int##bits##x##size##_add(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2) \
	{ \
		if (v##sign##int##bits##x##size##_impl_cpu->add) \
			v##sign##int##bits##x##size##_impl_cpu->add(vec1, vec2); \
	\
		return v##sign##int##bits##x##size##_fallback_add(vec1, vec2); \
	} \
	\
	v##sign##int##bits##x##size v##sign##int##bits##x##size##_sub(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2) \
	{ \
		if (v##sign##int##bits##x##size##_impl_cpu->sub) \
			v##sign##int##bits##x##size##_impl_cpu->sub(vec1, vec2); \
	\
		return v##sign##int##bits##x##size##_fallback_sub(vec1, vec2); \
	} \
	\
	v##sign##int##bits##x##size v##sign##int##bits##x##size##_mul(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2) \
	{ \
		if (v##sign##int##bits##x##size##_impl_cpu->mul) \
			v##sign##int##bits##x##size##_impl_cpu->mul(vec1, vec2); \
	\
		return v##sign##int##bits##x##size##_fallback_mul(vec1, vec2); \
	} \
	\
	v##sign##int##bits##x##size v##sign##int##bits##x##size##_div(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2) \
	{ \
		if (v##sign##int##bits##x##size##_impl_cpu->div) \
			v##sign##int##bits##x##size##_impl_cpu->div(vec1, vec2); \
	\
		return v##sign##int##bits##x##size##_fallback_div(vec1, vec2); \
	} \
	\
	v##sign##int##bits##x##size v##sign##int##bits##x##size##_avg(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2) \
	{ \
		if (v##sign##int##bits##x##size##_impl_cpu->avg) \
			v##sign##int##bits##x##size##_impl_cpu->avg(vec1, vec2); \
	\
		return v##sign##int##bits##x##size##_fallback_avg(vec1, vec2); \
	} \
	\
	v##sign##int##bits##x##size v##sign##int##bits##x##size##_and(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2) \
	{ \
		if (v##sign##int##bits##x##size##_impl_cpu->band) \
			v##sign##int##bits##x##size##_impl_cpu->band(vec1, vec2); \
	\
		return v##sign##int##bits##x##size##_fallback_and(vec1, vec2); \
	} \
	\
	v##sign##int##bits##x##size v##sign##int##bits##x##size##_or(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2) \
	{ \
		if (v##sign##int##bits##x##size##_impl_cpu->bor) \
			v##sign##int##bits##x##size##_impl_cpu->bor(vec1, vec2); \
	\
		return v##sign##int##bits##x##size##_fallback_or(vec1, vec2); \
	} \
	\
	v##sign##int##bits##x##size v##sign##int##bits##x##size##_xor(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2) \
	{ \
		if (v##sign##int##bits##x##size##_impl_cpu->bxor) \
			v##sign##int##bits##x##size##_impl_cpu->bxor(vec1, vec2); \
	\
		return v##sign##int##bits##x##size##_fallback_xor(vec1, vec2); \
	} \
	\
	v##sign##int##bits##x##size v##sign##int##bits##x##size##_not(v##sign##int##bits##x##size vec) \
	{ \
		if (v##sign##int##bits##x##size##_impl_cpu->bnot) \
			v##sign##int##bits##x##size##_impl_cpu->bnot(vec); \
	\
		return v##sign##int##bits##x##size##_fallback_not(vec); \
	} \
	\
	v##sign##int##bits##x##size v##sign##int##bits##x##size##_cmplt(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2) \
	{ \
		if (v##sign##int##bits##x##size##_impl_cpu->cmplt) \
			v##sign##int##bits##x##size##_impl_cpu->cmplt(vec1, vec2); \
	\
		return v##sign##int##bits##x##size##_fallback_cmplt(vec1, vec2); \
	} \
	\
	v##sign##int##bits##x##size v##sign##int##bits##x##size##_cmple(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2) \
	{ \
		if (v##sign##int##bits##x##size##_impl_cpu->cmple) \
			v##sign##int##bits##x##size##_impl_cpu->cmple(vec1, vec2); \
	\
		return v##sign##int##bits##x##size##_fallback_cmple(vec1, vec2); \
	} \
	\
	v##sign##int##bits##x##size v##sign##int##bits##x##size##_cmpeq(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2) \
	{ \
		if (v##sign##int##bits##x##size##_impl_cpu->cmpeq) \
			v##sign##int##bits##x##size##_impl_cpu->cmpeq(vec1, vec2); \
	\
		return v##sign##int##bits##x##size##_fallback_cmpeq(vec1, vec2); \
	} \
	\
	v##sign##int##bits##x##size v##sign##int##bits##x##size##_cmpge(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2) \
	{ \
		if (v##sign##int##bits##x##size##_impl_cpu->cmpge) \
			v##sign##int##bits##x##size##_impl_cpu->cmpge(vec1, vec2); \
	\
		return v##sign##int##bits##x##size##_fallback_cmpge(vec1, vec2); \
	} \
	\
	v##sign##int##bits##x##size v##sign##int##bits##x##size##_cmpgt(v##sign##int##bits##x##size vec1, v##sign##int##bits##x##size vec2) \
	{ \
		if (v##sign##int##bits##x##size##_impl_cpu->cmpgt) \
			v##sign##int##bits##x##size##_impl_cpu->cmpgt(vec1, vec2); \
	\
		return v##sign##int##bits##x##size##_fallback_cmpgt(vec1, vec2); \
	} \
	\
	v##sign##int##bits##x##size v##sign##int##bits##x##size##_lshift(v##sign##int##bits##x##size vec1, vuint##bits##x##size vec2) \
	{ \
		if (v##sign##int##bits##x##size##_impl_cpu->lshift) \
			v##sign##int##bits##x##size##_impl_cpu->lshift(vec1, vec2); \
	\
		return v##sign##int##bits##x##size##_fallback_lshift(vec1, vec2); \
	} \
	\
	v##sign##int##bits##x##size v##sign##int##bits##x##size##_rshift(v##sign##int##bits##x##size vec1, vuint##bits##x##size vec2) \
	{ \
		if (v##sign##int##bits##x##size##_impl_cpu->rshift) \
			v##sign##int##bits##x##size##_impl_cpu->rshift(vec1, vec2); \
	\
		return v##sign##int##bits##x##size##_fallback_rshift(vec1, vec2); \
	} \
	\
	v##sign##int##bits##x##size v##sign##int##bits##x##size##_lrshift(v##sign##int##bits##x##size vec1, vuint##bits##x##size vec2) \
	{ \
		if (v##sign##int##bits##x##size##_impl_cpu->lrshift) \
			v##sign##int##bits##x##size##_impl_cpu->lrshift(vec1, vec2); \
	\
		return v##sign##int##bits##x##size##_fallback_lrshift(vec1, vec2); \
	}

#define VEC_DEFINE_OPERATIONS(bits, size) \
	VEC_DEFINE_OPERATIONS_SIGN( , bits, size) \
	VEC_DEFINE_OPERATIONS_SIGN(u, bits, size)

// 16-bit
VEC_DEFINE_OPERATIONS(8, 2)

// 32-bit
VEC_DEFINE_OPERATIONS(8, 4)
VEC_DEFINE_OPERATIONS(16, 2)

// 64-bit
VEC_DEFINE_OPERATIONS(8, 8)
VEC_DEFINE_OPERATIONS(16, 4)
VEC_DEFINE_OPERATIONS(32, 2)

// 128-bit
VEC_DEFINE_OPERATIONS(8, 16)
VEC_DEFINE_OPERATIONS(16, 8)
VEC_DEFINE_OPERATIONS(32, 4)
VEC_DEFINE_OPERATIONS(64, 2)

// 256-bit
VEC_DEFINE_OPERATIONS(8, 32)
VEC_DEFINE_OPERATIONS(16, 16)
VEC_DEFINE_OPERATIONS(32, 8)
VEC_DEFINE_OPERATIONS(64, 4)

// 512-bit
VEC_DEFINE_OPERATIONS(8, 64)
VEC_DEFINE_OPERATIONS(16, 32)
VEC_DEFINE_OPERATIONS(32, 16)
VEC_DEFINE_OPERATIONS(64, 8)

#undef VEC_DEFINE_OPERATIONS
#undef VEC_DEFINE_OPERATIONS_SIGN

#endif /* VEC_IMPLEMENTATION */

#ifdef __cplusplus
}
#endif

#endif /* VEC_VEC_H_ */
