diff options
Diffstat (limited to 'thirdparty/embree-aarch64/common/simd')
29 files changed, 0 insertions, 9526 deletions
diff --git a/thirdparty/embree-aarch64/common/simd/avx.h b/thirdparty/embree-aarch64/common/simd/avx.h deleted file mode 100644 index c840e41805..0000000000 --- a/thirdparty/embree-aarch64/common/simd/avx.h +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2009-2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 - -#pragma once - -#include "sse.h" - -#if defined(__AVX512VL__) -#include "vboolf8_avx512.h" -#include "vboold4_avx512.h" -#else -#include "vboolf8_avx.h" -#include "vboold4_avx.h" -#endif - -#if defined(__AVX2__) -#include "vint8_avx2.h" -#include "vuint8_avx2.h" -#if defined(__X86_64__) -#include "vllong4_avx2.h" -#endif -#else -#include "vint8_avx.h" -#include "vuint8_avx.h" -#endif -#include "vfloat8_avx.h" -#if defined(__X86_64__) -#include "vdouble4_avx.h" -#endif - -#if defined(__AVX512F__) -#include "avx512.h" -#endif - diff --git a/thirdparty/embree-aarch64/common/simd/avx512.h b/thirdparty/embree-aarch64/common/simd/avx512.h deleted file mode 100644 index 25414ab5b1..0000000000 --- a/thirdparty/embree-aarch64/common/simd/avx512.h +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2009-2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 - -#pragma once - -#include "../sys/platform.h" -#include "../sys/intrinsics.h" -#include "../math/constants.h" -#include "../sys/alloc.h" -#include "varying.h" - -#include "vboolf16_avx512.h" -#include "vint16_avx512.h" -#include "vuint16_avx512.h" -#include "vfloat16_avx512.h" - -#include "vboold8_avx512.h" -#include "vllong8_avx512.h" -#include "vdouble8_avx512.h" - -namespace embree -{ - //////////////////////////////////////////////////////////////////////////////// - /// Prefetching - //////////////////////////////////////////////////////////////////////////////// - -#define PFHINT_L1 0 -#define PFHINT_L2 1 -#define PFHINT_NT 2 - - template<const unsigned int mode> - __forceinline void prefetch(const void * __restrict__ const m) - { - if (mode == PFHINT_L1) - _mm_prefetch((const char*)m,_MM_HINT_T0); - else if (mode == PFHINT_L2) - _mm_prefetch((const char*)m,_MM_HINT_T1); - else if (mode == PFHINT_NT) - _mm_prefetch((const char*)m,_MM_HINT_NTA); - } -} diff --git a/thirdparty/embree-aarch64/common/simd/simd.h b/thirdparty/embree-aarch64/common/simd/simd.h deleted file mode 100644 index 647851110b..0000000000 --- a/thirdparty/embree-aarch64/common/simd/simd.h +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright 2009-2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 - -#pragma once - -#include "../math/math.h" - -/* include SSE wrapper classes */ -#if defined(__SSE__) || defined(__ARM_NEON) -# include "sse.h" -#endif - -/* include AVX wrapper classes */ -#if defined(__AVX__) -# include "avx.h" -#endif - -/* include AVX512 wrapper classes */ -#if defined (__AVX512F__) -# include "avx512.h" -#endif - -namespace embree -{ - template <int N> - __forceinline vbool<N> isfinite(const vfloat<N>& v) - { - return (v >= vfloat<N>(-std::numeric_limits<float>::max())) - & (v <= vfloat<N>( std::numeric_limits<float>::max())); - } - - /* foreach unique */ - template<typename vbool, typename vint, typename Closure> - __forceinline void foreach_unique(const vbool& valid0, const vint& vi, const Closure& closure) - { - vbool valid1 = valid0; - while (any(valid1)) { - const int j = int(bsf(movemask(valid1))); - const int i = vi[j]; - const vbool valid2 = valid1 & (i == vi); - valid1 = andn(valid1, valid2); - closure(valid2, i); - } - } - - /* returns the next unique value i in vi and the corresponding valid_i mask */ - template<typename vbool, typename vint> - __forceinline int next_unique(vbool& valid, const vint& vi, /*out*/ vbool& valid_i) - { - assert(any(valid)); - const int j = int(bsf(movemask(valid))); - const int i = vi[j]; - valid_i = valid & (i == vi); - valid = andn(valid, valid_i); - return i; - } - - /* foreach unique index */ - template<typename vbool, typename vint, typename Closure> - __forceinline void foreach_unique_index(const vbool& valid0, const vint& vi, const Closure& closure) - { - vbool valid1 = valid0; - while (any(valid1)) { - const int j = int(bsf(movemask(valid1))); - const int i = vi[j]; - const vbool valid2 = valid1 & (i == vi); - valid1 = andn(valid1, valid2); - closure(valid2, i, j); - } - } - - /* returns the index of the next unique value i in vi and the corresponding valid_i mask */ - template<typename vbool, typename vint> - __forceinline int next_unique_index(vbool& valid, const vint& vi, /*out*/ vbool& valid_i) - { - assert(any(valid)); - const int j = int(bsf(movemask(valid))); - const int i = vi[j]; - valid_i = valid & (i == vi); - valid = andn(valid, valid_i); - return j; - } - - template<typename Closure> - __forceinline void foreach2(int x0, int x1, int y0, int y1, const Closure& closure) - { - __aligned(64) int U[2*VSIZEX]; - __aligned(64) int V[2*VSIZEX]; - int index = 0; - for (int y=y0; y<y1; y++) { - const bool lasty = y+1>=y1; - const vintx vy = y; - for (int x=x0; x<x1; ) { //x+=VSIZEX) { - const bool lastx = x+VSIZEX >= x1; - vintx vx = x+vintx(step); - vintx::storeu(&U[index], vx); - vintx::storeu(&V[index], vy); - const int dx = min(x1-x,VSIZEX); - index += dx; - x += dx; - if (index >= VSIZEX || (lastx && lasty)) { - const vboolx valid = vintx(step) < vintx(index); - closure(valid, vintx::load(U), vintx::load(V)); - x-= max(0, index-VSIZEX); - index = 0; - } - } - } - } -} diff --git a/thirdparty/embree-aarch64/common/simd/sse.cpp b/thirdparty/embree-aarch64/common/simd/sse.cpp deleted file mode 100644 index 1732cfa421..0000000000 --- a/thirdparty/embree-aarch64/common/simd/sse.cpp +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2009-2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 - -#include "sse.h" - -namespace embree -{ - const __m128 mm_lookupmask_ps[16] = { - _mm_castsi128_ps(_mm_set_epi32( 0, 0, 0, 0)), - _mm_castsi128_ps(_mm_set_epi32( 0, 0, 0,-1)), - _mm_castsi128_ps(_mm_set_epi32( 0, 0,-1, 0)), - _mm_castsi128_ps(_mm_set_epi32( 0, 0,-1,-1)), - _mm_castsi128_ps(_mm_set_epi32( 0,-1, 0, 0)), - _mm_castsi128_ps(_mm_set_epi32( 0,-1, 0,-1)), - _mm_castsi128_ps(_mm_set_epi32( 0,-1,-1, 0)), - _mm_castsi128_ps(_mm_set_epi32( 0,-1,-1,-1)), - _mm_castsi128_ps(_mm_set_epi32(-1, 0, 0, 0)), - _mm_castsi128_ps(_mm_set_epi32(-1, 0, 0,-1)), - _mm_castsi128_ps(_mm_set_epi32(-1, 0,-1, 0)), - _mm_castsi128_ps(_mm_set_epi32(-1, 0,-1,-1)), - _mm_castsi128_ps(_mm_set_epi32(-1,-1, 0, 0)), - _mm_castsi128_ps(_mm_set_epi32(-1,-1, 0,-1)), - _mm_castsi128_ps(_mm_set_epi32(-1,-1,-1, 0)), - _mm_castsi128_ps(_mm_set_epi32(-1,-1,-1,-1)) - }; - - const __m128d mm_lookupmask_pd[4] = { - _mm_castsi128_pd(_mm_set_epi32( 0, 0, 0, 0)), - _mm_castsi128_pd(_mm_set_epi32( 0, 0,-1,-1)), - _mm_castsi128_pd(_mm_set_epi32(-1,-1, 0, 0)), - _mm_castsi128_pd(_mm_set_epi32(-1,-1,-1,-1)) - }; - -} diff --git a/thirdparty/embree-aarch64/common/simd/sse.h b/thirdparty/embree-aarch64/common/simd/sse.h deleted file mode 100644 index 6bc818b55b..0000000000 --- a/thirdparty/embree-aarch64/common/simd/sse.h +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2009-2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 - -#pragma once - -#include "../sys/platform.h" -#include "../sys/intrinsics.h" -#include "../sys/alloc.h" -#include "../math/constants.h" -#include "varying.h" - -namespace embree -{ -#if (defined(__aarch64__) && defined(BUILD_IOS)) || defined(__SSE4_1__) - __forceinline __m128 blendv_ps(__m128 f, __m128 t, __m128 mask) { - return _mm_blendv_ps(f,t,mask); - } -#else - __forceinline __m128 blendv_ps(__m128 f, __m128 t, __m128 mask) { - return _mm_or_ps(_mm_and_ps(mask, t), _mm_andnot_ps(mask, f)); - } -#endif - - extern const __m128 mm_lookupmask_ps[16]; - extern const __m128d mm_lookupmask_pd[4]; -} - -#if defined(__AVX512VL__) -#include "vboolf4_avx512.h" -#else -#include "vboolf4_sse2.h" -#endif -#include "vint4_sse2.h" -#include "vuint4_sse2.h" -#include "vfloat4_sse2.h" diff --git a/thirdparty/embree-aarch64/common/simd/varying.h b/thirdparty/embree-aarch64/common/simd/varying.h deleted file mode 100644 index 9a46817da9..0000000000 --- a/thirdparty/embree-aarch64/common/simd/varying.h +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright 2009-2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 - -#pragma once - -#include "../sys/platform.h" - -namespace embree -{ - /* Varying numeric types */ - template<int N> - struct vfloat - { - union { float f[N]; int i[N]; }; - __forceinline const float& operator [](size_t index) const { assert(index < N); return f[index]; } - __forceinline float& operator [](size_t index) { assert(index < N); return f[index]; } - }; - - template<int N> - struct vdouble - { - union { double f[N]; long long i[N]; }; - __forceinline const double& operator [](size_t index) const { assert(index < N); return f[index]; } - __forceinline double& operator [](size_t index) { assert(index < N); return f[index]; } - }; - - template<int N> - struct vint - { - int i[N]; - __forceinline const int& operator [](size_t index) const { assert(index < N); return i[index]; } - __forceinline int& operator [](size_t index) { assert(index < N); return i[index]; } - }; - - template<int N> - struct vuint - { - unsigned int i[N]; - __forceinline const unsigned int& operator [](size_t index) const { assert(index < N); return i[index]; } - __forceinline unsigned int& operator [](size_t index) { assert(index < N); return i[index]; } - }; - - template<int N> - struct vllong - { - long long i[N]; - __forceinline const long long& operator [](size_t index) const { assert(index < N); return i[index]; } - __forceinline long long& operator [](size_t index) { assert(index < N); return i[index]; } - }; - - /* Varying bool types */ - template<int N> struct vboolf { int i[N]; }; // for float/int - template<int N> struct vboold { long long i[N]; }; // for double/long long - - /* Aliases to default types */ - template<int N> using vreal = vfloat<N>; - template<int N> using vbool = vboolf<N>; - - /* Varying size constants */ -#if defined(__AVX512VL__) // SKX - const int VSIZEX = 8; // default size - const int VSIZEL = 16; // large size -#elif defined(__AVX512F__) // KNL - const int VSIZEX = 16; - const int VSIZEL = 16; -#elif defined(__AVX__) - const int VSIZEX = 8; - const int VSIZEL = 8; -#else - const int VSIZEX = 4; - const int VSIZEL = 4; -#endif - - /* Extends varying size N to optimal or up to max(N, N2) */ - template<int N, int N2 = VSIZEX> - struct vextend - { -#if defined(__AVX512F__) && !defined(__AVX512VL__) // KNL - /* use 16-wide SIMD calculations on KNL even for 4 and 8 wide SIMD */ - static const int size = (N2 == VSIZEX) ? VSIZEX : N; - #define SIMD_MODE(N) N, 16 -#else - /* calculate with same SIMD width otherwise */ - static const int size = N; - #define SIMD_MODE(N) N, N -#endif - }; - - /* 4-wide shortcuts */ - typedef vfloat<4> vfloat4; - typedef vdouble<4> vdouble4; - typedef vreal<4> vreal4; - typedef vint<4> vint4; - typedef vuint<4> vuint4; - typedef vllong<4> vllong4; - typedef vbool<4> vbool4; - typedef vboolf<4> vboolf4; - typedef vboold<4> vboold4; - - /* 8-wide shortcuts */ - typedef vfloat<8> vfloat8; - typedef vdouble<8> vdouble8; - typedef vreal<8> vreal8; - typedef vint<8> vint8; - typedef vuint<8> vuint8; - typedef vllong<8> vllong8; - typedef vbool<8> vbool8; - typedef vboolf<8> vboolf8; - typedef vboold<8> vboold8; - - /* 16-wide shortcuts */ - typedef vfloat<16> vfloat16; - typedef vdouble<16> vdouble16; - typedef vreal<16> vreal16; - typedef vint<16> vint16; - typedef vuint<16> vuint16; - typedef vllong<16> vllong16; - typedef vbool<16> vbool16; - typedef vboolf<16> vboolf16; - typedef vboold<16> vboold16; - - /* Default shortcuts */ - typedef vfloat<VSIZEX> vfloatx; - typedef vdouble<VSIZEX> vdoublex; - typedef vreal<VSIZEX> vrealx; - typedef vint<VSIZEX> vintx; - typedef vuint<VSIZEX> vuintx; - typedef vllong<VSIZEX> vllongx; - typedef vbool<VSIZEX> vboolx; - typedef vboolf<VSIZEX> vboolfx; - typedef vboold<VSIZEX> vbooldx; -} diff --git a/thirdparty/embree-aarch64/common/simd/vboold4_avx.h b/thirdparty/embree-aarch64/common/simd/vboold4_avx.h deleted file mode 100644 index 6505ee56f3..0000000000 --- a/thirdparty/embree-aarch64/common/simd/vboold4_avx.h +++ /dev/null @@ -1,160 +0,0 @@ -// Copyright 2009-2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 - -#pragma once - -namespace embree -{ - /* 4-wide AVX bool type for 64bit data types*/ - template<> - struct vboold<4> - { - ALIGNED_STRUCT_(32); - - typedef vboold4 Bool; - - enum { size = 4 }; // number of SIMD elements - union { // data - __m256d v; - struct { __m128d vl,vh; }; - long long i[4]; - }; - - //////////////////////////////////////////////////////////////////////////////// - /// Constructors, Assignment & Cast Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboold() {} - __forceinline vboold(const vboold4& a) { v = a.v; } - __forceinline vboold4& operator =(const vboold4& a) { v = a.v; return *this; } - - __forceinline vboold(__m256d a) : v(a) {} - __forceinline vboold(__m256i a) : v(_mm256_castsi256_pd(a)) {} - - __forceinline operator const __m256() const { return _mm256_castpd_ps(v); } - __forceinline operator const __m256i() const { return _mm256_castpd_si256(v); } - __forceinline operator const __m256d() const { return v; } - - __forceinline vboold(int a) - { - assert(a >= 0 && a <= 255); -#if defined (__AVX2__) - const __m256i mask = _mm256_set_epi64x(0x8, 0x4, 0x2, 0x1); - const __m256i b = _mm256_set1_epi64x(a); - const __m256i c = _mm256_and_si256(b,mask); - v = _mm256_castsi256_pd(_mm256_cmpeq_epi64(c,mask)); -#else - vl = mm_lookupmask_pd[a & 0x3]; - vh = mm_lookupmask_pd[a >> 2]; -#endif - } - - __forceinline vboold(__m128d a, __m128d b) : vl(a), vh(b) {} - - //////////////////////////////////////////////////////////////////////////////// - /// Constants - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboold(FalseTy) : v(_mm256_setzero_pd()) {} -#if !defined(__aarch64__) - __forceinline vboold(TrueTy) : v(_mm256_cmp_pd(_mm256_setzero_pd(), _mm256_setzero_pd(), _CMP_EQ_OQ)) {} -#else - __forceinline vboold(TrueTy) : v(_mm256_cmpeq_pd(_mm256_setzero_pd(), _mm256_setzero_pd())) {} -#endif - - //////////////////////////////////////////////////////////////////////////////// - /// Array Access - //////////////////////////////////////////////////////////////////////////////// - - __forceinline bool operator [](size_t index) const { assert(index < 4); return (_mm256_movemask_pd(v) >> index) & 1; } - __forceinline long long& operator [](size_t index) { assert(index < 4); return i[index]; } - }; - - //////////////////////////////////////////////////////////////////////////////// - /// Unary Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboold4 operator !(const vboold4& a) { return _mm256_xor_pd(a, vboold4(embree::True)); } - - //////////////////////////////////////////////////////////////////////////////// - /// Binary Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboold4 operator &(const vboold4& a, const vboold4& b) { return _mm256_and_pd(a, b); } - __forceinline vboold4 operator |(const vboold4& a, const vboold4& b) { return _mm256_or_pd (a, b); } - __forceinline vboold4 operator ^(const vboold4& a, const vboold4& b) { return _mm256_xor_pd(a, b); } - - __forceinline vboold4 andn(const vboold4& a, const vboold4& b) { return _mm256_andnot_pd(b, a); } - - __forceinline vboold4& operator &=(vboold4& a, const vboold4& b) { return a = a & b; } - __forceinline vboold4& operator |=(vboold4& a, const vboold4& b) { return a = a | b; } - __forceinline vboold4& operator ^=(vboold4& a, const vboold4& b) { return a = a ^ b; } - - //////////////////////////////////////////////////////////////////////////////// - /// Comparison Operators + Select - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboold4 operator !=(const vboold4& a, const vboold4& b) { return _mm256_xor_pd(a, b); } - __forceinline vboold4 operator ==(const vboold4& a, const vboold4& b) { return _mm256_xor_pd(_mm256_xor_pd(a,b),vboold4(embree::True)); } - - __forceinline vboold4 select(const vboold4& mask, const vboold4& t, const vboold4& f) { - return _mm256_blendv_pd(f, t, mask); - } - - //////////////////////////////////////////////////////////////////////////////// - /// Movement/Shifting/Shuffling Functions - //////////////////////////////////////////////////////////////////////////////// - -#if !defined(__aarch64__) - __forceinline vboold4 unpacklo(const vboold4& a, const vboold4& b) { return _mm256_unpacklo_pd(a, b); } - __forceinline vboold4 unpackhi(const vboold4& a, const vboold4& b) { return _mm256_unpackhi_pd(a, b); } -#endif - -#if defined(__AVX2__) - template<int i0, int i1, int i2, int i3> - __forceinline vboold4 shuffle(const vboold4& v) { - return _mm256_permute4x64_pd(v, _MM_SHUFFLE(i3, i2, i1, i0)); - } - - template<int i> - __forceinline vboold4 shuffle(const vboold4& v) { - return _mm256_permute4x64_pd(v, _MM_SHUFFLE(i, i, i, i)); - } -#endif - - - //////////////////////////////////////////////////////////////////////////////// - /// Reduction Operations - //////////////////////////////////////////////////////////////////////////////// - - __forceinline bool reduce_and(const vboold4& a) { return _mm256_movemask_pd(a) == (unsigned int)0xf; } - __forceinline bool reduce_or (const vboold4& a) { return !_mm256_testz_pd(a,a); } - - __forceinline bool all (const vboold4& a) { return _mm256_movemask_pd(a) == (unsigned int)0xf; } - __forceinline bool any (const vboold4& a) { return !_mm256_testz_pd(a,a); } - __forceinline bool none(const vboold4& a) { return _mm256_testz_pd(a,a) != 0; } - - __forceinline bool all (const vboold4& valid, const vboold4& b) { return all((!valid) | b); } - __forceinline bool any (const vboold4& valid, const vboold4& b) { return any(valid & b); } - __forceinline bool none(const vboold4& valid, const vboold4& b) { return none(valid & b); } - - __forceinline unsigned int movemask(const vboold4& a) { return _mm256_movemask_pd(a); } - __forceinline size_t popcnt (const vboold4& a) { return popcnt((size_t)_mm256_movemask_pd(a)); } - - //////////////////////////////////////////////////////////////////////////////// - /// Get/Set Functions - //////////////////////////////////////////////////////////////////////////////// - - __forceinline bool get(const vboold4& a, size_t index) { return a[index]; } - __forceinline void set (vboold4& a, size_t index) { a[index] = -1; } - __forceinline void clear(vboold4& a, size_t index) { a[index] = 0; } - - //////////////////////////////////////////////////////////////////////////////// - /// Output Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline embree_ostream operator <<(embree_ostream cout, const vboold4& a) { - return cout << "<" << a[0] << ", " << a[1] << ", " << a[2] << ", " << a[3] << ", " - << a[4] << ", " << a[5] << ", " << a[6] << ", " << a[7] << ">"; - } -} diff --git a/thirdparty/embree-aarch64/common/simd/vboold4_avx512.h b/thirdparty/embree-aarch64/common/simd/vboold4_avx512.h deleted file mode 100644 index 4fe730d713..0000000000 --- a/thirdparty/embree-aarch64/common/simd/vboold4_avx512.h +++ /dev/null @@ -1,140 +0,0 @@ -// Copyright 2009-2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 - -#pragma once - -namespace embree -{ - /* 4-wide AVX-512 bool type */ - template<> - struct vboold<4> - { - typedef vboold4 Bool; - typedef vint4 Int; - - enum { size = 4 }; // number of SIMD elements - __mmask8 v; // data - - //////////////////////////////////////////////////////////////////////////////// - /// Constructors, Assignment & Cast Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboold() {} - __forceinline vboold(const vboold4& t) { v = t.v; } - __forceinline vboold4& operator =(const vboold4& f) { v = f.v; return *this; } - - __forceinline vboold(const __mmask8 &t) { v = t; } - __forceinline operator __mmask8() const { return v; } - - __forceinline vboold(bool b) { v = b ? 0xf : 0x0; } - __forceinline vboold(int t) { v = (__mmask8)t; } - __forceinline vboold(unsigned int t) { v = (__mmask8)t; } - - /* return int8 mask */ - __forceinline __m128i mask8() const { - return _mm_movm_epi8(v); - } - - /* return int32 mask */ - __forceinline __m128i mask32() const { - return _mm_movm_epi32(v); - } - - /* return int64 mask */ - __forceinline __m256i mask64() const { - return _mm256_movm_epi64(v); - } - - //////////////////////////////////////////////////////////////////////////////// - /// Constants - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboold(FalseTy) : v(0x0) {} - __forceinline vboold(TrueTy) : v(0xf) {} - - //////////////////////////////////////////////////////////////////////////////// - /// Array Access - //////////////////////////////////////////////////////////////////////////////// - - __forceinline bool operator [](size_t index) const { - assert(index < 4); return (mm512_mask2int(v) >> index) & 1; - } - }; - - //////////////////////////////////////////////////////////////////////////////// - /// Unary Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboold4 operator !(const vboold4& a) { return _mm512_kandn(a, 0xf); } - - //////////////////////////////////////////////////////////////////////////////// - /// Binary Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboold4 operator &(const vboold4& a, const vboold4& b) { return _mm512_kand(a, b); } - __forceinline vboold4 operator |(const vboold4& a, const vboold4& b) { return _mm512_kor(a, b); } - __forceinline vboold4 operator ^(const vboold4& a, const vboold4& b) { return _mm512_kxor(a, b); } - - __forceinline vboold4 andn(const vboold4& a, const vboold4& b) { return _mm512_kandn(b, a); } - - //////////////////////////////////////////////////////////////////////////////// - /// Assignment Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboold4& operator &=(vboold4& a, const vboold4& b) { return a = a & b; } - __forceinline vboold4& operator |=(vboold4& a, const vboold4& b) { return a = a | b; } - __forceinline vboold4& operator ^=(vboold4& a, const vboold4& b) { return a = a ^ b; } - - //////////////////////////////////////////////////////////////////////////////// - /// Comparison Operators + Select - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboold4 operator !=(const vboold4& a, const vboold4& b) { return _mm512_kxor(a, b); } - __forceinline vboold4 operator ==(const vboold4& a, const vboold4& b) { return _mm512_kand(_mm512_kxnor(a, b), 0xf); } - - __forceinline vboold4 select(const vboold4& s, const vboold4& a, const vboold4& b) { - return _mm512_kor(_mm512_kand(s, a), _mm512_kandn(s, b)); - } - - //////////////////////////////////////////////////////////////////////////////// - /// Reduction Operations - //////////////////////////////////////////////////////////////////////////////// - - __forceinline int all (const vboold4& a) { return a.v == 0xf; } - __forceinline int any (const vboold4& a) { return _mm512_kortestz(a, a) == 0; } - __forceinline int none(const vboold4& a) { return _mm512_kortestz(a, a) != 0; } - - __forceinline int all (const vboold4& valid, const vboold4& b) { return all((!valid) | b); } - __forceinline int any (const vboold4& valid, const vboold4& b) { return any(valid & b); } - __forceinline int none(const vboold4& valid, const vboold4& b) { return none(valid & b); } - - __forceinline size_t movemask(const vboold4& a) { return _mm512_kmov(a); } - __forceinline size_t popcnt (const vboold4& a) { return popcnt(a.v); } - - //////////////////////////////////////////////////////////////////////////////// - /// Conversion Operations - //////////////////////////////////////////////////////////////////////////////// - - __forceinline unsigned int toInt(const vboold4& a) { return mm512_mask2int(a); } - - //////////////////////////////////////////////////////////////////////////////// - /// Get/Set Functions - //////////////////////////////////////////////////////////////////////////////// - - __forceinline bool get(const vboold4& a, size_t index) { assert(index < 4); return (toInt(a) >> index) & 1; } - __forceinline void set(vboold4& a, size_t index) { assert(index < 4); a |= 1 << index; } - __forceinline void clear(vboold4& a, size_t index) { assert(index < 4); a = andn(a, 1 << index); } - - //////////////////////////////////////////////////////////////////////////////// - /// Output Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline embree_ostream operator <<(embree_ostream cout, const vboold4& a) - { - cout << "<"; - for (size_t i=0; i<4; i++) { - if ((a.v >> i) & 1) cout << "1"; else cout << "0"; - } - return cout << ">"; - } -} diff --git a/thirdparty/embree-aarch64/common/simd/vboold8_avx512.h b/thirdparty/embree-aarch64/common/simd/vboold8_avx512.h deleted file mode 100644 index fdf3f00de5..0000000000 --- a/thirdparty/embree-aarch64/common/simd/vboold8_avx512.h +++ /dev/null @@ -1,148 +0,0 @@ -// Copyright 2009-2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 - -#pragma once - -namespace embree -{ - /* 8-wide AVX-512 bool type */ - template<> - struct vboold<8> - { - typedef vboold8 Bool; - typedef vint8 Int; - - enum { size = 8 }; // number of SIMD elements - __mmask8 v; // data - - //////////////////////////////////////////////////////////////////////////////// - /// Constructors, Assignment & Cast Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboold() {} - __forceinline vboold(const vboold8& t) { v = t.v; } - __forceinline vboold8& operator =(const vboold8& f) { v = f.v; return *this; } - - __forceinline vboold(const __mmask8& t) { v = t; } - __forceinline operator __mmask8() const { return v; } - - __forceinline vboold(bool b) { v = b ? 0xff : 0x00; } - __forceinline vboold(int t) { v = (__mmask8)t; } - __forceinline vboold(unsigned int t) { v = (__mmask8)t; } - - /* return int8 mask */ - __forceinline __m128i mask8() const { -#if defined(__AVX512BW__) - return _mm_movm_epi8(v); -#else - const __m512i f = _mm512_set1_epi64(0); - const __m512i t = _mm512_set1_epi64(-1); - const __m512i m = _mm512_mask_or_epi64(f,v,t,t); - return _mm512_cvtepi64_epi8(m); -#endif - } - - /* return int64 mask */ - __forceinline __m512i mask64() const { -#if defined(__AVX512DQ__) - return _mm512_movm_epi64(v); -#else - const __m512i f = _mm512_set1_epi64(0); - const __m512i t = _mm512_set1_epi64(-1); - return _mm512_mask_or_epi64(f,v,t,t); -#endif - } - - //////////////////////////////////////////////////////////////////////////////// - /// Constants - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboold(FalseTy) : v(0x00) {} - __forceinline vboold(TrueTy) : v(0xff) {} - - //////////////////////////////////////////////////////////////////////////////// - /// Array Access - //////////////////////////////////////////////////////////////////////////////// - - __forceinline bool operator [](size_t index) const { - assert(index < 8); return (mm512_mask2int(v) >> index) & 1; - } - }; - - //////////////////////////////////////////////////////////////////////////////// - /// Unary Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboold8 operator !(const vboold8& a) { return _mm512_knot(a); } - - //////////////////////////////////////////////////////////////////////////////// - /// Binary Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboold8 operator &(const vboold8& a, const vboold8& b) { return _mm512_kand(a, b); } - __forceinline vboold8 operator |(const vboold8& a, const vboold8& b) { return _mm512_kor(a, b); } - __forceinline vboold8 operator ^(const vboold8& a, const vboold8& b) { return _mm512_kxor(a, b); } - - __forceinline vboold8 andn(const vboold8& a, const vboold8& b) { return _mm512_kandn(b, a); } - - //////////////////////////////////////////////////////////////////////////////// - /// Assignment Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboold8& operator &=(vboold8& a, const vboold8& b) { return a = a & b; } - __forceinline vboold8& operator |=(vboold8& a, const vboold8& b) { return a = a | b; } - __forceinline vboold8& operator ^=(vboold8& a, const vboold8& b) { return a = a ^ b; } - - //////////////////////////////////////////////////////////////////////////////// - /// Comparison Operators + Select - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboold8 operator !=(const vboold8& a, const vboold8& b) { return _mm512_kxor(a, b); } - __forceinline vboold8 operator ==(const vboold8& a, const vboold8& b) { return _mm512_kxnor(a, b); } - - __forceinline vboold8 select(const vboold8& s, const vboold8& a, const vboold8& b) { - return _mm512_kor(_mm512_kand(s, a), _mm512_kandn(s, b)); - } - - //////////////////////////////////////////////////////////////////////////////// - /// Reduction Operations - //////////////////////////////////////////////////////////////////////////////// - - __forceinline int all (const vboold8& a) { return a.v == 0xff; } - __forceinline int any (const vboold8& a) { return _mm512_kortestz(a, a) == 0; } - __forceinline int none(const vboold8& a) { return _mm512_kortestz(a, a) != 0; } - - __forceinline int all (const vboold8& valid, const vboold8& b) { return all((!valid) | b); } - __forceinline int any (const vboold8& valid, const vboold8& b) { return any(valid & b); } - __forceinline int none(const vboold8& valid, const vboold8& b) { return none(valid & b); } - - __forceinline size_t movemask(const vboold8& a) { return _mm512_kmov(a); } - __forceinline size_t popcnt (const vboold8& a) { return popcnt(a.v); } - - //////////////////////////////////////////////////////////////////////////////// - /// Conversion Operations - //////////////////////////////////////////////////////////////////////////////// - - __forceinline unsigned int toInt(const vboold8& a) { return mm512_mask2int(a); } - - //////////////////////////////////////////////////////////////////////////////// - /// Get/Set Functions - //////////////////////////////////////////////////////////////////////////////// - - __forceinline bool get(const vboold8& a, size_t index) { assert(index < 8); return (toInt(a) >> index) & 1; } - __forceinline void set(vboold8& a, size_t index) { assert(index < 8); a |= 1 << index; } - __forceinline void clear(vboold8& a, size_t index) { assert(index < 8); a = andn(a, 1 << index); } - - //////////////////////////////////////////////////////////////////////////////// - /// Output Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline embree_ostream operator <<(embree_ostream cout, const vboold8& a) - { - cout << "<"; - for (size_t i=0; i<8; i++) { - if ((a.v >> i) & 1) cout << "1"; else cout << "0"; - } - return cout << ">"; - } -} diff --git a/thirdparty/embree-aarch64/common/simd/vboolf16_avx512.h b/thirdparty/embree-aarch64/common/simd/vboolf16_avx512.h deleted file mode 100644 index 238cdc8eb9..0000000000 --- a/thirdparty/embree-aarch64/common/simd/vboolf16_avx512.h +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright 2009-2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 - -#pragma once - -namespace embree -{ - /* 16-wide AVX-512 bool type */ - template<> - struct vboolf<16> - { - typedef vboolf16 Bool; - typedef vint16 Int; - typedef vfloat16 Float; - - enum { size = 16 }; // number of SIMD elements - __mmask16 v; // data - - //////////////////////////////////////////////////////////////////////////////// - /// Constructors, Assignment & Cast Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboolf() {} - __forceinline vboolf(const vboolf16& t) { v = t.v; } - __forceinline vboolf16& operator =(const vboolf16& f) { v = f.v; return *this; } - - __forceinline vboolf(const __mmask16& t) { v = t; } - __forceinline operator __mmask16() const { return v; } - - __forceinline vboolf(bool b) { v = b ? 0xFFFF : 0x0000; } - __forceinline vboolf(int t) { v = (__mmask16)t; } - __forceinline vboolf(unsigned int t) { v = (__mmask16)t; } - - /* return int8 mask */ - __forceinline __m128i mask8() const { -#if defined(__AVX512BW__) - return _mm_movm_epi8(v); -#else - const __m512i f = _mm512_set1_epi32(0); - const __m512i t = _mm512_set1_epi32(-1); - const __m512i m = _mm512_mask_or_epi32(f,v,t,t); - return _mm512_cvtepi32_epi8(m); -#endif - } - - /* return int32 mask */ - __forceinline __m512i mask32() const { -#if defined(__AVX512DQ__) - return _mm512_movm_epi32(v); -#else - const __m512i f = _mm512_set1_epi32(0); - const __m512i t = _mm512_set1_epi32(-1); - return _mm512_mask_or_epi32(f,v,t,t); -#endif - } - - //////////////////////////////////////////////////////////////////////////////// - /// Constants - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboolf(FalseTy) : v(0x0000) {} - __forceinline vboolf(TrueTy) : v(0xffff) {} - - //////////////////////////////////////////////////////////////////////////////// - /// Array Access - //////////////////////////////////////////////////////////////////////////////// - - __forceinline bool operator [](size_t index) const { - assert(index < 16); return (mm512_mask2int(v) >> index) & 1; - } - }; - - //////////////////////////////////////////////////////////////////////////////// - /// Unary Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboolf16 operator !(const vboolf16& a) { return _mm512_knot(a); } - - //////////////////////////////////////////////////////////////////////////////// - /// Binary Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboolf16 operator &(const vboolf16& a, const vboolf16& b) { return _mm512_kand(a,b); } - __forceinline vboolf16 operator |(const vboolf16& a, const vboolf16& b) { return _mm512_kor(a,b); } - __forceinline vboolf16 operator ^(const vboolf16& a, const vboolf16& b) { return _mm512_kxor(a,b); } - - __forceinline vboolf16 andn(const vboolf16& a, const vboolf16& b) { return _mm512_kandn(b,a); } - - //////////////////////////////////////////////////////////////////////////////// - /// Assignment Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboolf16& operator &=(vboolf16& a, const vboolf16& b) { return a = a & b; } - __forceinline vboolf16& operator |=(vboolf16& a, const vboolf16& b) { return a = a | b; } - __forceinline vboolf16& operator ^=(vboolf16& a, const vboolf16& b) { return a = a ^ b; } - - //////////////////////////////////////////////////////////////////////////////// - /// Comparison Operators + Select - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboolf16 operator !=(const vboolf16& a, const vboolf16& b) { return _mm512_kxor(a, b); } - __forceinline vboolf16 operator ==(const vboolf16& a, const vboolf16& b) { return _mm512_kxnor(a, b); } - - __forceinline vboolf16 select(const vboolf16& s, const vboolf16& a, const vboolf16& b) { - return _mm512_kor(_mm512_kand(s,a),_mm512_kandn(s,b)); - } - - //////////////////////////////////////////////////////////////////////////////// - /// Reduction Operations - //////////////////////////////////////////////////////////////////////////////// - - __forceinline int all (const vboolf16& a) { return _mm512_kortestc(a,a) != 0; } - __forceinline int any (const vboolf16& a) { return _mm512_kortestz(a,a) == 0; } - __forceinline int none(const vboolf16& a) { return _mm512_kortestz(a,a) != 0; } - - __forceinline int all (const vboolf16& valid, const vboolf16& b) { return all((!valid) | b); } - __forceinline int any (const vboolf16& valid, const vboolf16& b) { return any(valid & b); } - __forceinline int none(const vboolf16& valid, const vboolf16& b) { return none(valid & b); } - - __forceinline size_t movemask(const vboolf16& a) { return _mm512_kmov(a); } - __forceinline size_t popcnt (const vboolf16& a) { return popcnt(a.v); } - - //////////////////////////////////////////////////////////////////////////////// - /// Convertion Operations - //////////////////////////////////////////////////////////////////////////////// - - __forceinline unsigned int toInt (const vboolf16& a) { return mm512_mask2int(a); } - __forceinline vboolf16 toMask(const int& a) { return mm512_int2mask(a); } - - //////////////////////////////////////////////////////////////////////////////// - /// Get/Set Functions - //////////////////////////////////////////////////////////////////////////////// - - __forceinline bool get(const vboolf16& a, size_t index) { assert(index < 16); return (toInt(a) >> index) & 1; } - __forceinline void set(vboolf16& a, size_t index) { assert(index < 16); a |= 1 << index; } - __forceinline void clear(vboolf16& a, size_t index) { assert(index < 16); a = andn(a, 1 << index); } - - //////////////////////////////////////////////////////////////////////////////// - /// Output Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline embree_ostream operator <<(embree_ostream cout, const vboolf16& a) - { - cout << "<"; - for (size_t i=0; i<16; i++) { - if ((a.v >> i) & 1) cout << "1"; else cout << "0"; - } - return cout << ">"; - } -} diff --git a/thirdparty/embree-aarch64/common/simd/vboolf4_avx512.h b/thirdparty/embree-aarch64/common/simd/vboolf4_avx512.h deleted file mode 100644 index 2ae4c4470e..0000000000 --- a/thirdparty/embree-aarch64/common/simd/vboolf4_avx512.h +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright 2009-2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 - -#pragma once - -namespace embree -{ - /* 4-wide AVX-512 bool type */ - template<> - struct vboolf<4> - { - typedef vboolf4 Bool; - typedef vint4 Int; - - enum { size = 4 }; // number of SIMD elements - __mmask8 v; // data - - //////////////////////////////////////////////////////////////////////////////// - /// Constructors, Assignment & Cast Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboolf() {} - __forceinline vboolf(const vboolf4& t) { v = t.v; } - __forceinline vboolf4& operator =(const vboolf4& f) { v = f.v; return *this; } - - __forceinline vboolf(const __mmask8 &t) { v = t; } - __forceinline operator __mmask8() const { return v; } - - __forceinline vboolf(bool b) { v = b ? 0xf : 0x0; } - __forceinline vboolf(int t) { v = (__mmask8)t; } - __forceinline vboolf(unsigned int t) { v = (__mmask8)t; } - - __forceinline vboolf(bool a, bool b, bool c, bool d) - : v((__mmask8)((int(d) << 3) | (int(c) << 2) | (int(b) << 1) | int(a))) {} - - /* return int8 mask */ - __forceinline __m128i mask8() const { - return _mm_movm_epi8(v); - } - - /* return int32 mask */ - __forceinline __m128i mask32() const { - return _mm_movm_epi32(v); - } - - /* return int64 mask */ - __forceinline __m256i mask64() const { - return _mm256_movm_epi64(v); - } - - //////////////////////////////////////////////////////////////////////////////// - /// Constants - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboolf(FalseTy) : v(0x0) {} - __forceinline vboolf(TrueTy) : v(0xf) {} - - //////////////////////////////////////////////////////////////////////////////// - /// Array Access - //////////////////////////////////////////////////////////////////////////////// - - __forceinline bool operator [](size_t index) const { - assert(index < 4); return (mm512_mask2int(v) >> index) & 1; - } - }; - - //////////////////////////////////////////////////////////////////////////////// - /// Unary Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboolf4 operator !(const vboolf4& a) { return _mm512_kandn(a, 0xf); } - - //////////////////////////////////////////////////////////////////////////////// - /// Binary Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboolf4 operator &(const vboolf4& a, const vboolf4& b) { return _mm512_kand(a, b); } - __forceinline vboolf4 operator |(const vboolf4& a, const vboolf4& b) { return _mm512_kor(a, b); } - __forceinline vboolf4 operator ^(const vboolf4& a, const vboolf4& b) { return _mm512_kxor(a, b); } - - __forceinline vboolf4 andn(const vboolf4& a, const vboolf4& b) { return _mm512_kandn(b, a); } - - //////////////////////////////////////////////////////////////////////////////// - /// Assignment Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboolf4& operator &=(vboolf4& a, const vboolf4& b) { return a = a & b; } - __forceinline vboolf4& operator |=(vboolf4& a, const vboolf4& b) { return a = a | b; } - __forceinline vboolf4& operator ^=(vboolf4& a, const vboolf4& b) { return a = a ^ b; } - - //////////////////////////////////////////////////////////////////////////////// - /// Comparison Operators + Select - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboolf4 operator !=(const vboolf4& a, const vboolf4& b) { return _mm512_kxor(a, b); } - __forceinline vboolf4 operator ==(const vboolf4& a, const vboolf4& b) { return _mm512_kand(_mm512_kxnor(a, b), 0xf); } - - __forceinline vboolf4 select(const vboolf4& s, const vboolf4& a, const vboolf4& b) { - return _mm512_kor(_mm512_kand(s, a), _mm512_kandn(s, b)); - } - - //////////////////////////////////////////////////////////////////////////////// - /// Reduction Operations - //////////////////////////////////////////////////////////////////////////////// - - __forceinline int all (const vboolf4& a) { return a.v == 0xf; } - __forceinline int any (const vboolf4& a) { return _mm512_kortestz(a, a) == 0; } - __forceinline int none(const vboolf4& a) { return _mm512_kortestz(a, a) != 0; } - - __forceinline int all (const vboolf4& valid, const vboolf4& b) { return all((!valid) | b); } - __forceinline int any (const vboolf4& valid, const vboolf4& b) { return any(valid & b); } - __forceinline int none(const vboolf4& valid, const vboolf4& b) { return none(valid & b); } - - __forceinline size_t movemask(const vboolf4& a) { return _mm512_kmov(a); } - __forceinline size_t popcnt (const vboolf4& a) { return popcnt(a.v); } - - //////////////////////////////////////////////////////////////////////////////// - /// Conversion Operations - //////////////////////////////////////////////////////////////////////////////// - - __forceinline unsigned int toInt(const vboolf4& a) { return mm512_mask2int(a); } - - //////////////////////////////////////////////////////////////////////////////// - /// Get/Set Functions - //////////////////////////////////////////////////////////////////////////////// - - __forceinline bool get(const vboolf4& a, size_t index) { assert(index < 4); return (toInt(a) >> index) & 1; } - __forceinline void set(vboolf4& a, size_t index) { assert(index < 4); a |= 1 << index; } - __forceinline void clear(vboolf4& a, size_t index) { assert(index < 4); a = andn(a, 1 << index); } - - //////////////////////////////////////////////////////////////////////////////// - /// Output Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline embree_ostream operator <<(embree_ostream cout, const vboolf4& a) - { - cout << "<"; - for (size_t i=0; i<4; i++) { - if ((a.v >> i) & 1) cout << "1"; else cout << "0"; - } - return cout << ">"; - } -} diff --git a/thirdparty/embree-aarch64/common/simd/vboolf4_sse2.h b/thirdparty/embree-aarch64/common/simd/vboolf4_sse2.h deleted file mode 100644 index ed53b3c783..0000000000 --- a/thirdparty/embree-aarch64/common/simd/vboolf4_sse2.h +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright 2009-2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 - -#pragma once - -namespace embree -{ - /* 4-wide SSE bool type */ - template<> - struct vboolf<4> - { - ALIGNED_STRUCT_(16); - - typedef vboolf4 Bool; - typedef vint4 Int; - typedef vfloat4 Float; - - enum { size = 4 }; // number of SIMD elements - union { __m128 v; int i[4]; }; // data - - //////////////////////////////////////////////////////////////////////////////// - /// Constructors, Assignment & Cast Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboolf() {} - __forceinline vboolf(const vboolf4& other) { v = other.v; } - __forceinline vboolf4& operator =(const vboolf4& other) { v = other.v; return *this; } - - __forceinline vboolf(__m128 input) : v(input) {} - __forceinline operator const __m128&() const { return v; } - __forceinline operator const __m128i() const { return _mm_castps_si128(v); } - __forceinline operator const __m128d() const { return _mm_castps_pd(v); } - - __forceinline vboolf(bool a) - : v(mm_lookupmask_ps[(size_t(a) << 3) | (size_t(a) << 2) | (size_t(a) << 1) | size_t(a)]) {} - __forceinline vboolf(bool a, bool b) - : v(mm_lookupmask_ps[(size_t(b) << 3) | (size_t(a) << 2) | (size_t(b) << 1) | size_t(a)]) {} - __forceinline vboolf(bool a, bool b, bool c, bool d) - : v(mm_lookupmask_ps[(size_t(d) << 3) | (size_t(c) << 2) | (size_t(b) << 1) | size_t(a)]) {} -#if defined(__aarch64__) && defined(BUILD_IOS) - __forceinline vboolf(int mask) { v = mm_lookupmask_ps[mask]; } - __forceinline vboolf(unsigned int mask) { v = mm_lookupmask_ps[mask]; } -#else - __forceinline vboolf(int mask) { assert(mask >= 0 && mask < 16); v = mm_lookupmask_ps[mask]; } - __forceinline vboolf(unsigned int mask) { assert(mask < 16); v = mm_lookupmask_ps[mask]; } -#endif - /* return int32 mask */ - __forceinline __m128i mask32() const { - return _mm_castps_si128(v); - } - - //////////////////////////////////////////////////////////////////////////////// - /// Constants - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboolf(FalseTy) : v(_mm_setzero_ps()) {} - __forceinline vboolf(TrueTy) : v(_mm_castsi128_ps(_mm_cmpeq_epi32(_mm_setzero_si128(), _mm_setzero_si128()))) {} - - //////////////////////////////////////////////////////////////////////////////// - /// Array Access - //////////////////////////////////////////////////////////////////////////////// - -#if defined(__aarch64__) && defined(BUILD_IOS) - __forceinline bool operator [](size_t index) const { return (_mm_movemask_ps(v) >> index) & 1; } - __forceinline int& operator [](size_t index) { return i[index]; } -#else - __forceinline bool operator [](size_t index) const { assert(index < 4); return (_mm_movemask_ps(v) >> index) & 1; } - __forceinline int& operator [](size_t index) { assert(index < 4); return i[index]; } -#endif - }; - - //////////////////////////////////////////////////////////////////////////////// - /// Unary Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboolf4 operator !(const vboolf4& a) { return _mm_xor_ps(a, vboolf4(embree::True)); } - - //////////////////////////////////////////////////////////////////////////////// - /// Binary Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboolf4 operator &(const vboolf4& a, const vboolf4& b) { return _mm_and_ps(a, b); } - __forceinline vboolf4 operator |(const vboolf4& a, const vboolf4& b) { return _mm_or_ps (a, b); } - __forceinline vboolf4 operator ^(const vboolf4& a, const vboolf4& b) { return _mm_xor_ps(a, b); } - - __forceinline vboolf4 andn(const vboolf4& a, const vboolf4& b) { return _mm_andnot_ps(b, a); } - - //////////////////////////////////////////////////////////////////////////////// - /// Assignment Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboolf4& operator &=(vboolf4& a, const vboolf4& b) { return a = a & b; } - __forceinline vboolf4& operator |=(vboolf4& a, const vboolf4& b) { return a = a | b; } - __forceinline vboolf4& operator ^=(vboolf4& a, const vboolf4& b) { return a = a ^ b; } - - //////////////////////////////////////////////////////////////////////////////// - /// Comparison Operators + Select - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboolf4 operator !=(const vboolf4& a, const vboolf4& b) { return _mm_xor_ps(a, b); } - __forceinline vboolf4 operator ==(const vboolf4& a, const vboolf4& b) { return _mm_castsi128_ps(_mm_cmpeq_epi32(a, b)); } - - __forceinline vboolf4 select(const vboolf4& m, const vboolf4& t, const vboolf4& f) { -#if (defined(__aarch64__) && defined(BUILD_IOS)) || defined(__SSE4_1__) - return _mm_blendv_ps(f, t, m); -#else - return _mm_or_ps(_mm_and_ps(m, t), _mm_andnot_ps(m, f)); -#endif - } - - //////////////////////////////////////////////////////////////////////////////// - /// Movement/Shifting/Shuffling Functions - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboolf4 unpacklo(const vboolf4& a, const vboolf4& b) { return _mm_unpacklo_ps(a, b); } - __forceinline vboolf4 unpackhi(const vboolf4& a, const vboolf4& b) { return _mm_unpackhi_ps(a, b); } - -#if defined(__aarch64__) - template<int i0, int i1, int i2, int i3> - __forceinline vboolf4 shuffle(const vboolf4& v) { - return vreinterpretq_f32_u8(vqtbl1q_u8( vreinterpretq_u8_s32(v), _MN_SHUFFLE(i0, i1, i2, i3))); - } - - template<int i0, int i1, int i2, int i3> - __forceinline vboolf4 shuffle(const vboolf4& a, const vboolf4& b) { - return vreinterpretq_f32_u8(vqtbl2q_u8( (uint8x16x2_t){(uint8x16_t)a.v, (uint8x16_t)b.v}, _MF_SHUFFLE(i0, i1, i2, i3))); - } -#else - template<int i0, int i1, int i2, int i3> - __forceinline vboolf4 shuffle(const vboolf4& v) { - return _mm_castsi128_ps(_mm_shuffle_epi32(v, _MM_SHUFFLE(i3, i2, i1, i0))); - } - - template<int i0, int i1, int i2, int i3> - __forceinline vboolf4 shuffle(const vboolf4& a, const vboolf4& b) { - return _mm_shuffle_ps(a, b, _MM_SHUFFLE(i3, i2, i1, i0)); - } -#endif - - template<int i0> - __forceinline vboolf4 shuffle(const vboolf4& v) { - return shuffle<i0,i0,i0,i0>(v); - } - -#if defined(__SSE3__) - template<> __forceinline vboolf4 shuffle<0, 0, 2, 2>(const vboolf4& v) { return _mm_moveldup_ps(v); } - template<> __forceinline vboolf4 shuffle<1, 1, 3, 3>(const vboolf4& v) { return _mm_movehdup_ps(v); } - template<> __forceinline vboolf4 shuffle<0, 1, 0, 1>(const vboolf4& v) { return _mm_castpd_ps(_mm_movedup_pd(v)); } -#endif - -#if defined(__SSE4_1__) && !defined(__aarch64__) - template<int dst, int src, int clr> __forceinline vboolf4 insert(const vboolf4& a, const vboolf4& b) { return _mm_insert_ps(a, b, (dst << 4) | (src << 6) | clr); } - template<int dst, int src> __forceinline vboolf4 insert(const vboolf4& a, const vboolf4& b) { return insert<dst, src, 0>(a, b); } - template<int dst> __forceinline vboolf4 insert(const vboolf4& a, const bool b) { return insert<dst, 0>(a, vboolf4(b)); } -#endif - - //////////////////////////////////////////////////////////////////////////////// - /// Reduction Operations - //////////////////////////////////////////////////////////////////////////////// - - __forceinline bool reduce_and(const vboolf4& a) { return _mm_movemask_ps(a) == 0xf; } - __forceinline bool reduce_or (const vboolf4& a) { return _mm_movemask_ps(a) != 0x0; } - - __forceinline bool all (const vboolf4& b) { return _mm_movemask_ps(b) == 0xf; } - __forceinline bool any (const vboolf4& b) { return _mm_movemask_ps(b) != 0x0; } - __forceinline bool none(const vboolf4& b) { return _mm_movemask_ps(b) == 0x0; } - - __forceinline bool all (const vboolf4& valid, const vboolf4& b) { return all((!valid) | b); } - __forceinline bool any (const vboolf4& valid, const vboolf4& b) { return any(valid & b); } - __forceinline bool none(const vboolf4& valid, const vboolf4& b) { return none(valid & b); } - - __forceinline size_t movemask(const vboolf4& a) { return _mm_movemask_ps(a); } -#if defined(__aarch64__) && defined(BUILD_IOS) -__forceinline size_t popcnt(const vboolf4& a) { return _mm_movemask_popcnt_ps(a); } -#else -#if defined(__SSE4_2__) - __forceinline size_t popcnt(const vboolf4& a) { return popcnt((size_t)_mm_movemask_ps(a)); } -#else - __forceinline size_t popcnt(const vboolf4& a) { return bool(a[0])+bool(a[1])+bool(a[2])+bool(a[3]); } -#endif -#endif - - //////////////////////////////////////////////////////////////////////////////// - /// Get/Set Functions - //////////////////////////////////////////////////////////////////////////////// - - __forceinline bool get(const vboolf4& a, size_t index) { return a[index]; } - __forceinline void set(vboolf4& a, size_t index) { a[index] = -1; } - __forceinline void clear(vboolf4& a, size_t index) { a[index] = 0; } - - //////////////////////////////////////////////////////////////////////////////// - /// Output Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline embree_ostream operator <<(embree_ostream cout, const vboolf4& a) { - return cout << "<" << a[0] << ", " << a[1] << ", " << a[2] << ", " << a[3] << ">"; - } -} diff --git a/thirdparty/embree-aarch64/common/simd/vboolf8_avx.h b/thirdparty/embree-aarch64/common/simd/vboolf8_avx.h deleted file mode 100644 index 4f64741b55..0000000000 --- a/thirdparty/embree-aarch64/common/simd/vboolf8_avx.h +++ /dev/null @@ -1,189 +0,0 @@ -// Copyright 2009-2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 - -#pragma once - -namespace embree -{ - /* 8-wide AVX bool type */ - template<> - struct vboolf<8> - { - ALIGNED_STRUCT_(32); - - typedef vboolf8 Bool; - typedef vint8 Int; - typedef vfloat8 Float; - - enum { size = 8 }; // number of SIMD elements - union { // data - __m256 v; - struct { __m128 vl,vh; }; - int i[8]; - }; - - //////////////////////////////////////////////////////////////////////////////// - /// Constructors, Assignment & Cast Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboolf() {} - __forceinline vboolf(const vboolf8& a) { v = a.v; } - __forceinline vboolf8& operator =(const vboolf8& a) { v = a.v; return *this; } - - __forceinline vboolf(__m256 a) : v(a) {} - __forceinline operator const __m256&() const { return v; } - __forceinline operator const __m256i() const { return _mm256_castps_si256(v); } - __forceinline operator const __m256d() const { return _mm256_castps_pd(v); } - - __forceinline vboolf(int a) - { - assert(a >= 0 && a <= 255); -#if defined (__AVX2__) - const __m256i mask = _mm256_set_epi32(0x80, 0x40, 0x20, 0x10, 0x8, 0x4, 0x2, 0x1); - const __m256i b = _mm256_set1_epi32(a); - const __m256i c = _mm256_and_si256(b,mask); - v = _mm256_castsi256_ps(_mm256_cmpeq_epi32(c,mask)); -#else - vl = mm_lookupmask_ps[a & 0xF]; - vh = mm_lookupmask_ps[a >> 4]; -#endif - } - - __forceinline vboolf(const vboolf4& a) : v(_mm256_insertf128_ps(_mm256_castps128_ps256(a),a,1)) {} - __forceinline vboolf(const vboolf4& a, const vboolf4& b) : v(_mm256_insertf128_ps(_mm256_castps128_ps256(a),b,1)) {} - __forceinline vboolf(__m128 a, __m128 b) : vl(a), vh(b) {} - - __forceinline vboolf(bool a) : v(vboolf8(vboolf4(a), vboolf4(a))) {} - __forceinline vboolf(bool a, bool b) : v(vboolf8(vboolf4(a), vboolf4(b))) {} - __forceinline vboolf(bool a, bool b, bool c, bool d) : v(vboolf8(vboolf4(a,b), vboolf4(c,d))) {} - __forceinline vboolf(bool a, bool b, bool c, bool d, bool e, bool f, bool g, bool h) : v(vboolf8(vboolf4(a,b,c,d), vboolf4(e,f,g,h))) {} - - /* return int32 mask */ - __forceinline __m256i mask32() const { - return _mm256_castps_si256(v); - } - - //////////////////////////////////////////////////////////////////////////////// - /// Constants - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboolf(FalseTy) : v(_mm256_setzero_ps()) {} -#if !defined(__aarch64__) - __forceinline vboolf(TrueTy) : v(_mm256_cmp_ps(_mm256_setzero_ps(), _mm256_setzero_ps(), _CMP_EQ_OQ)) {} -#else - __forceinline vboolf(TrueTy) : v(_mm256_cmpeq_ps(_mm256_setzero_ps(), _mm256_setzero_ps())) {} -#endif - //////////////////////////////////////////////////////////////////////////////// - /// Array Access - //////////////////////////////////////////////////////////////////////////////// - - __forceinline bool operator [](size_t index) const { assert(index < 8); return (_mm256_movemask_ps(v) >> index) & 1; } - __forceinline int& operator [](size_t index) { assert(index < 8); return i[index]; } - }; - - //////////////////////////////////////////////////////////////////////////////// - /// Unary Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboolf8 operator !(const vboolf8& a) { return _mm256_xor_ps(a, vboolf8(embree::True)); } - - //////////////////////////////////////////////////////////////////////////////// - /// Binary Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboolf8 operator &(const vboolf8& a, const vboolf8& b) { return _mm256_and_ps(a, b); } - __forceinline vboolf8 operator |(const vboolf8& a, const vboolf8& b) { return _mm256_or_ps (a, b); } - __forceinline vboolf8 operator ^(const vboolf8& a, const vboolf8& b) { return _mm256_xor_ps(a, b); } - - __forceinline vboolf8 andn(const vboolf8& a, const vboolf8& b) { return _mm256_andnot_ps(b, a); } - - __forceinline vboolf8& operator &=(vboolf8& a, const vboolf8& b) { return a = a & b; } - __forceinline vboolf8& operator |=(vboolf8& a, const vboolf8& b) { return a = a | b; } - __forceinline vboolf8& operator ^=(vboolf8& a, const vboolf8& b) { return a = a ^ b; } - - //////////////////////////////////////////////////////////////////////////////// - /// Comparison Operators + Select - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboolf8 operator !=(const vboolf8& a, const vboolf8& b) { return _mm256_xor_ps(a, b); } - __forceinline vboolf8 operator ==(const vboolf8& a, const vboolf8& b) { return _mm256_xor_ps(_mm256_xor_ps(a,b),vboolf8(embree::True)); } - - __forceinline vboolf8 select(const vboolf8& mask, const vboolf8& t, const vboolf8& f) { - return _mm256_blendv_ps(f, t, mask); - } - - //////////////////////////////////////////////////////////////////////////////// - /// Movement/Shifting/Shuffling Functions - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboolf8 unpacklo(const vboolf8& a, const vboolf8& b) { return _mm256_unpacklo_ps(a, b); } - __forceinline vboolf8 unpackhi(const vboolf8& a, const vboolf8& b) { return _mm256_unpackhi_ps(a, b); } - - template<int i> - __forceinline vboolf8 shuffle(const vboolf8& v) { - return _mm256_permute_ps(v, _MM_SHUFFLE(i, i, i, i)); - } - - template<int i0, int i1> - __forceinline vboolf8 shuffle4(const vboolf8& v) { - return _mm256_permute2f128_ps(v, v, (i1 << 4) | (i0 << 0)); - } - - template<int i0, int i1> - __forceinline vboolf8 shuffle4(const vboolf8& a, const vboolf8& b) { - return _mm256_permute2f128_ps(a, b, (i1 << 4) | (i0 << 0)); - } - - template<int i0, int i1, int i2, int i3> - __forceinline vboolf8 shuffle(const vboolf8& v) { - return _mm256_permute_ps(v, _MM_SHUFFLE(i3, i2, i1, i0)); - } - - template<int i0, int i1, int i2, int i3> - __forceinline vboolf8 shuffle(const vboolf8& a, const vboolf8& b) { - return _mm256_shuffle_ps(a, b, _MM_SHUFFLE(i3, i2, i1, i0)); - } - - template<> __forceinline vboolf8 shuffle<0, 0, 2, 2>(const vboolf8& v) { return _mm256_moveldup_ps(v); } - template<> __forceinline vboolf8 shuffle<1, 1, 3, 3>(const vboolf8& v) { return _mm256_movehdup_ps(v); } - template<> __forceinline vboolf8 shuffle<0, 1, 0, 1>(const vboolf8& v) { return _mm256_castpd_ps(_mm256_movedup_pd(_mm256_castps_pd(v))); } - - template<int i> __forceinline vboolf8 insert4(const vboolf8& a, const vboolf4& b) { return _mm256_insertf128_ps(a, b, i); } - template<int i> __forceinline vboolf4 extract4 (const vboolf8& a) { return _mm256_extractf128_ps(a, i); } - template<> __forceinline vboolf4 extract4<0>(const vboolf8& a) { return _mm256_castps256_ps128(a); } - - //////////////////////////////////////////////////////////////////////////////// - /// Reduction Operations - //////////////////////////////////////////////////////////////////////////////// - - __forceinline bool reduce_and(const vboolf8& a) { return _mm256_movemask_ps(a) == (unsigned int)0xff; } - __forceinline bool reduce_or (const vboolf8& a) { return !_mm256_testz_ps(a,a); } - - __forceinline bool all (const vboolf8& a) { return _mm256_movemask_ps(a) == (unsigned int)0xff; } - __forceinline bool any (const vboolf8& a) { return !_mm256_testz_ps(a,a); } - __forceinline bool none(const vboolf8& a) { return _mm256_testz_ps(a,a) != 0; } - - __forceinline bool all (const vboolf8& valid, const vboolf8& b) { return all((!valid) | b); } - __forceinline bool any (const vboolf8& valid, const vboolf8& b) { return any(valid & b); } - __forceinline bool none(const vboolf8& valid, const vboolf8& b) { return none(valid & b); } - - __forceinline unsigned int movemask(const vboolf8& a) { return _mm256_movemask_ps(a); } - __forceinline size_t popcnt (const vboolf8& a) { return popcnt((size_t)_mm256_movemask_ps(a)); } - - //////////////////////////////////////////////////////////////////////////////// - /// Get/Set Functions - //////////////////////////////////////////////////////////////////////////////// - - __forceinline bool get(const vboolf8& a, size_t index) { return a[index]; } - __forceinline void set(vboolf8& a, size_t index) { a[index] = -1; } - __forceinline void clear(vboolf8& a, size_t index) { a[index] = 0; } - - //////////////////////////////////////////////////////////////////////////////// - /// Output Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline embree_ostream operator <<(embree_ostream cout, const vboolf8& a) { - return cout << "<" << a[0] << ", " << a[1] << ", " << a[2] << ", " << a[3] << ", " - << a[4] << ", " << a[5] << ", " << a[6] << ", " << a[7] << ">"; - } -} diff --git a/thirdparty/embree-aarch64/common/simd/vboolf8_avx512.h b/thirdparty/embree-aarch64/common/simd/vboolf8_avx512.h deleted file mode 100644 index 2a52b554c7..0000000000 --- a/thirdparty/embree-aarch64/common/simd/vboolf8_avx512.h +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright 2009-2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 - -#pragma once - -namespace embree -{ - /* 8-wide AVX-512 bool type */ - template<> - struct vboolf<8> - { - typedef vboolf8 Bool; - typedef vint8 Int; - - enum { size = 8 }; // number of SIMD elements - __mmask8 v; // data - - //////////////////////////////////////////////////////////////////////////////// - /// Constructors, Assignment & Cast Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboolf() {} - __forceinline vboolf(const vboolf8& t) { v = t.v; } - __forceinline vboolf8& operator =(const vboolf8& f) { v = f.v; return *this; } - - __forceinline vboolf(const __mmask8 &t) { v = t; } - __forceinline operator __mmask8() const { return v; } - - __forceinline vboolf(bool b) { v = b ? 0xff : 0x00; } - __forceinline vboolf(int t) { v = (__mmask8)t; } - __forceinline vboolf(unsigned int t) { v = (__mmask8)t; } - - __forceinline vboolf(bool a, bool b, bool c, bool d, bool e, bool f, bool g, bool h) - : v((__mmask8)((int(h) << 7) | (int(g) << 6) | (int(f) << 5) | (int(e) << 4) | (int(d) << 3) | (int(c) << 2) | (int(b) << 1) | int(a))) {} - - /* return int8 mask */ - __forceinline __m128i mask8() const { - return _mm_movm_epi8(v); - } - - /* return int32 mask */ - __forceinline __m256i mask32() const { - return _mm256_movm_epi32(v); - } - - /* return int64 mask */ - __forceinline __m512i mask64() const { - return _mm512_movm_epi64(v); - } - - //////////////////////////////////////////////////////////////////////////////// - /// Constants - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboolf(FalseTy) : v(0x00) {} - __forceinline vboolf(TrueTy) : v(0xff) {} - - //////////////////////////////////////////////////////////////////////////////// - /// Array Access - //////////////////////////////////////////////////////////////////////////////// - - __forceinline bool operator [](size_t index) const { - assert(index < 8); return (mm512_mask2int(v) >> index) & 1; - } - }; - - //////////////////////////////////////////////////////////////////////////////// - /// Unary Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboolf8 operator !(const vboolf8& a) { return _mm512_knot(a); } - - //////////////////////////////////////////////////////////////////////////////// - /// Binary Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboolf8 operator &(const vboolf8& a, const vboolf8& b) { return _mm512_kand(a, b); } - __forceinline vboolf8 operator |(const vboolf8& a, const vboolf8& b) { return _mm512_kor(a, b); } - __forceinline vboolf8 operator ^(const vboolf8& a, const vboolf8& b) { return _mm512_kxor(a, b); } - - __forceinline vboolf8 andn(const vboolf8& a, const vboolf8& b) { return _mm512_kandn(b, a); } - - //////////////////////////////////////////////////////////////////////////////// - /// Assignment Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboolf8& operator &=(vboolf8& a, const vboolf8& b) { return a = a & b; } - __forceinline vboolf8& operator |=(vboolf8& a, const vboolf8& b) { return a = a | b; } - __forceinline vboolf8& operator ^=(vboolf8& a, const vboolf8& b) { return a = a ^ b; } - - //////////////////////////////////////////////////////////////////////////////// - /// Comparison Operators + Select - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboolf8 operator !=(const vboolf8& a, const vboolf8& b) { return _mm512_kxor(a, b); } - __forceinline vboolf8 operator ==(const vboolf8& a, const vboolf8& b) { return _mm512_kxnor(a, b); } - - __forceinline vboolf8 select(const vboolf8& s, const vboolf8& a, const vboolf8& b) { - return _mm512_kor(_mm512_kand(s, a), _mm512_kandn(s, b)); - } - - //////////////////////////////////////////////////////////////////////////////// - /// Reduction Operations - //////////////////////////////////////////////////////////////////////////////// - - __forceinline int all (const vboolf8& a) { return a.v == 0xff; } - __forceinline int any (const vboolf8& a) { return _mm512_kortestz(a, a) == 0; } - __forceinline int none(const vboolf8& a) { return _mm512_kortestz(a, a) != 0; } - - __forceinline int all (const vboolf8& valid, const vboolf8& b) { return all((!valid) | b); } - __forceinline int any (const vboolf8& valid, const vboolf8& b) { return any(valid & b); } - __forceinline int none(const vboolf8& valid, const vboolf8& b) { return none(valid & b); } - - __forceinline size_t movemask(const vboolf8& a) { return _mm512_kmov(a); } - __forceinline size_t popcnt (const vboolf8& a) { return popcnt(a.v); } - - //////////////////////////////////////////////////////////////////////////////// - /// Conversion Operations - //////////////////////////////////////////////////////////////////////////////// - - __forceinline unsigned int toInt(const vboolf8& a) { return mm512_mask2int(a); } - - //////////////////////////////////////////////////////////////////////////////// - /// Get/Set Functions - //////////////////////////////////////////////////////////////////////////////// - - __forceinline bool get(const vboolf8& a, size_t index) { assert(index < 8); return (toInt(a) >> index) & 1; } - __forceinline void set(vboolf8& a, size_t index) { assert(index < 8); a |= 1 << index; } - __forceinline void clear(vboolf8& a, size_t index) { assert(index < 8); a = andn(a, 1 << index); } - - //////////////////////////////////////////////////////////////////////////////// - /// Output Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline embree_ostream operator <<(embree_ostream cout, const vboolf8& a) - { - cout << "<"; - for (size_t i=0; i<8; i++) { - if ((a.v >> i) & 1) cout << "1"; else cout << "0"; - } - return cout << ">"; - } -} diff --git a/thirdparty/embree-aarch64/common/simd/vdouble4_avx.h b/thirdparty/embree-aarch64/common/simd/vdouble4_avx.h deleted file mode 100644 index 1f65b45d7e..0000000000 --- a/thirdparty/embree-aarch64/common/simd/vdouble4_avx.h +++ /dev/null @@ -1,324 +0,0 @@ -// Copyright 2009-2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 - -#pragma once - -namespace embree -{ - /* 4-wide AVX 64-bit double type */ - template<> - struct vdouble<4> - { - ALIGNED_STRUCT_(32); - - typedef vboold4 Bool; - - enum { size = 4 }; // number of SIMD elements - union { // data - __m256d v; - double i[4]; - }; - - //////////////////////////////////////////////////////////////////////////////// - /// Constructors, Assignment & Cast Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vdouble() {} - __forceinline vdouble(const vdouble4& t) { v = t.v; } - __forceinline vdouble4& operator =(const vdouble4& f) { v = f.v; return *this; } - - __forceinline vdouble(const __m256d& t) { v = t; } - __forceinline operator __m256d() const { return v; } - - __forceinline vdouble(double i) { - v = _mm256_set1_pd(i); - } - - __forceinline vdouble(double a, double b, double c, double d) { - v = _mm256_set_pd(d,c,b,a); - } - - - //////////////////////////////////////////////////////////////////////////////// - /// Constants - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vdouble(ZeroTy) : v(_mm256_setzero_pd()) {} - __forceinline vdouble(OneTy) : v(_mm256_set1_pd(1)) {} - __forceinline vdouble(StepTy) : v(_mm256_set_pd(3.0,2.0,1.0,0.0)) {} - __forceinline vdouble(ReverseStepTy) : v(_mm256_setr_pd(3.0,2.0,1.0,0.0)) {} - - //////////////////////////////////////////////////////////////////////////////// - /// Loads and Stores - //////////////////////////////////////////////////////////////////////////////// - - static __forceinline void store_nt(double *__restrict__ ptr, const vdouble4& a) { - _mm256_stream_pd(ptr, a); - } - - static __forceinline vdouble4 loadu(const double* addr) { - return _mm256_loadu_pd(addr); - } - - static __forceinline vdouble4 load(const vdouble4* addr) { - return _mm256_load_pd((double*)addr); - } - - static __forceinline vdouble4 load(const double* addr) { - return _mm256_load_pd(addr); - } - - static __forceinline void store(double* ptr, const vdouble4& v) { - _mm256_store_pd(ptr, v); - } - - static __forceinline void storeu(double* ptr, const vdouble4& v) { - _mm256_storeu_pd(ptr, v); - } - - static __forceinline vdouble4 broadcast(const void* a) { return _mm256_set1_pd(*(double*)a); } - - //////////////////////////////////////////////////////////////////////////////// - /// Array Access - //////////////////////////////////////////////////////////////////////////////// - - __forceinline double& operator [](size_t index) { assert(index < 4); return i[index]; } - __forceinline const double& operator [](size_t index) const { assert(index < 4); return i[index]; } - }; - - //////////////////////////////////////////////////////////////////////////////// - /// Unary Operators - //////////////////////////////////////////////////////////////////////////////// - -#if defined(__AVX2__) - __forceinline vdouble4 asDouble(const vllong4& a) { return _mm256_castsi256_pd(a); } - __forceinline vllong4 asLLong (const vdouble4& a) { return _mm256_castpd_si256(a); } -#endif - - __forceinline vdouble4 operator +(const vdouble4& a) { return a; } - __forceinline vdouble4 operator -(const vdouble4& a) { return _mm256_sub_pd(_mm256_setzero_pd(), a); } - - //////////////////////////////////////////////////////////////////////////////// - /// Binary Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vdouble4 operator +(const vdouble4& a, const vdouble4& b) { return _mm256_add_pd(a, b); } - __forceinline vdouble4 operator +(const vdouble4& a, double b) { return a + vdouble4(b); } - __forceinline vdouble4 operator +(double a, const vdouble4& b) { return vdouble4(a) + b; } - - __forceinline vdouble4 operator -(const vdouble4& a, const vdouble4& b) { return _mm256_sub_pd(a, b); } - __forceinline vdouble4 operator -(const vdouble4& a, double b) { return a - vdouble4(b); } - __forceinline vdouble4 operator -(double a, const vdouble4& b) { return vdouble4(a) - b; } - - __forceinline vdouble4 operator *(const vdouble4& a, const vdouble4& b) { return _mm256_mul_pd(a, b); } - __forceinline vdouble4 operator *(const vdouble4& a, double b) { return a * vdouble4(b); } - __forceinline vdouble4 operator *(double a, const vdouble4& b) { return vdouble4(a) * b; } - - __forceinline vdouble4 operator &(const vdouble4& a, const vdouble4& b) { return _mm256_and_pd(a, b); } - __forceinline vdouble4 operator &(const vdouble4& a, double b) { return a & vdouble4(b); } - __forceinline vdouble4 operator &(double a, const vdouble4& b) { return vdouble4(a) & b; } - - __forceinline vdouble4 operator |(const vdouble4& a, const vdouble4& b) { return _mm256_or_pd(a, b); } - __forceinline vdouble4 operator |(const vdouble4& a, double b) { return a | vdouble4(b); } - __forceinline vdouble4 operator |(double a, const vdouble4& b) { return vdouble4(a) | b; } - - __forceinline vdouble4 operator ^(const vdouble4& a, const vdouble4& b) { return _mm256_xor_pd(a, b); } - __forceinline vdouble4 operator ^(const vdouble4& a, double b) { return a ^ vdouble4(b); } - __forceinline vdouble4 operator ^(double a, const vdouble4& b) { return vdouble4(a) ^ b; } - - __forceinline vdouble4 min(const vdouble4& a, const vdouble4& b) { return _mm256_min_pd(a, b); } - __forceinline vdouble4 min(const vdouble4& a, double b) { return min(a,vdouble4(b)); } - __forceinline vdouble4 min(double a, const vdouble4& b) { return min(vdouble4(a),b); } - - __forceinline vdouble4 max(const vdouble4& a, const vdouble4& b) { return _mm256_max_pd(a, b); } - __forceinline vdouble4 max(const vdouble4& a, double b) { return max(a,vdouble4(b)); } - __forceinline vdouble4 max(double a, const vdouble4& b) { return max(vdouble4(a),b); } - - //////////////////////////////////////////////////////////////////////////////// - /// Ternary Operators - //////////////////////////////////////////////////////////////////////////////// - -#if defined(__FMA__) - __forceinline vdouble4 madd (const vdouble4& a, const vdouble4& b, const vdouble4& c) { return _mm256_fmadd_pd(a,b,c); } - __forceinline vdouble4 msub (const vdouble4& a, const vdouble4& b, const vdouble4& c) { return _mm256_fmsub_pd(a,b,c); } - __forceinline vdouble4 nmadd(const vdouble4& a, const vdouble4& b, const vdouble4& c) { return _mm256_fnmadd_pd(a,b,c); } - __forceinline vdouble4 nmsub(const vdouble4& a, const vdouble4& b, const vdouble4& c) { return _mm256_fnmsub_pd(a,b,c); } -#else - __forceinline vdouble4 madd (const vdouble4& a, const vdouble4& b, const vdouble4& c) { return a*b+c; } - __forceinline vdouble4 msub (const vdouble4& a, const vdouble4& b, const vdouble4& c) { return a*b-c; } - __forceinline vdouble4 nmadd(const vdouble4& a, const vdouble4& b, const vdouble4& c) { return -a*b+c;} - __forceinline vdouble4 nmsub(const vdouble4& a, const vdouble4& b, const vdouble4& c) { return -a*b-c; } -#endif - - //////////////////////////////////////////////////////////////////////////////// - /// Assignment Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vdouble4& operator +=(vdouble4& a, const vdouble4& b) { return a = a + b; } - __forceinline vdouble4& operator +=(vdouble4& a, double b) { return a = a + b; } - - __forceinline vdouble4& operator -=(vdouble4& a, const vdouble4& b) { return a = a - b; } - __forceinline vdouble4& operator -=(vdouble4& a, double b) { return a = a - b; } - - __forceinline vdouble4& operator *=(vdouble4& a, const vdouble4& b) { return a = a * b; } - __forceinline vdouble4& operator *=(vdouble4& a, double b) { return a = a * b; } - - __forceinline vdouble4& operator &=(vdouble4& a, const vdouble4& b) { return a = a & b; } - __forceinline vdouble4& operator &=(vdouble4& a, double b) { return a = a & b; } - - __forceinline vdouble4& operator |=(vdouble4& a, const vdouble4& b) { return a = a | b; } - __forceinline vdouble4& operator |=(vdouble4& a, double b) { return a = a | b; } - - - //////////////////////////////////////////////////////////////////////////////// - /// Comparison Operators + Select - //////////////////////////////////////////////////////////////////////////////// - -#if defined(__AVX512VL__) - __forceinline vboold4 operator ==(const vdouble4& a, const vdouble4& b) { return _mm256_cmp_pd_mask(a, b, _MM_CMPINT_EQ); } - __forceinline vboold4 operator !=(const vdouble4& a, const vdouble4& b) { return _mm256_cmp_pd_mask(a, b, _MM_CMPINT_NE); } - __forceinline vboold4 operator < (const vdouble4& a, const vdouble4& b) { return _mm256_cmp_pd_mask(a, b, _MM_CMPINT_LT); } - __forceinline vboold4 operator >=(const vdouble4& a, const vdouble4& b) { return _mm256_cmp_pd_mask(a, b, _MM_CMPINT_GE); } - __forceinline vboold4 operator > (const vdouble4& a, const vdouble4& b) { return _mm256_cmp_pd_mask(a, b, _MM_CMPINT_GT); } - __forceinline vboold4 operator <=(const vdouble4& a, const vdouble4& b) { return _mm256_cmp_pd_mask(a, b, _MM_CMPINT_LE); } -#elif !defined(__aarch64__) - __forceinline vboold4 operator ==(const vdouble4& a, const vdouble4& b) { return _mm256_cmp_pd(a, b, _CMP_EQ_OQ); } - __forceinline vboold4 operator !=(const vdouble4& a, const vdouble4& b) { return _mm256_cmp_pd(a, b, _CMP_NEQ_UQ); } - __forceinline vboold4 operator < (const vdouble4& a, const vdouble4& b) { return _mm256_cmp_pd(a, b, _CMP_LT_OS); } - __forceinline vboold4 operator >=(const vdouble4& a, const vdouble4& b) { return _mm256_cmp_pd(a, b, _CMP_NLT_US); } - __forceinline vboold4 operator > (const vdouble4& a, const vdouble4& b) { return _mm256_cmp_pd(a, b, _CMP_NLE_US); } - __forceinline vboold4 operator <=(const vdouble4& a, const vdouble4& b) { return _mm256_cmp_pd(a, b, _CMP_LE_OS); } -#else - __forceinline vboold4 operator ==(const vdouble4& a, const vdouble4& b) { return _mm256_cmpeq_pd(a, b); } - __forceinline vboold4 operator !=(const vdouble4& a, const vdouble4& b) { return _mm256_cmpneq_pd(a, b); } - __forceinline vboold4 operator < (const vdouble4& a, const vdouble4& b) { return _mm256_cmplt_pd(a, b); } - __forceinline vboold4 operator >=(const vdouble4& a, const vdouble4& b) { return _mm256_cmpnlt_pd(a, b); } - __forceinline vboold4 operator > (const vdouble4& a, const vdouble4& b) { return _mm256_cmpnle_pd(a, b); } - __forceinline vboold4 operator <=(const vdouble4& a, const vdouble4& b) { return _mm256_cmple_pd(a, b); } -#endif - - __forceinline vboold4 operator ==(const vdouble4& a, double b) { return a == vdouble4(b); } - __forceinline vboold4 operator ==(double a, const vdouble4& b) { return vdouble4(a) == b; } - - __forceinline vboold4 operator !=(const vdouble4& a, double b) { return a != vdouble4(b); } - __forceinline vboold4 operator !=(double a, const vdouble4& b) { return vdouble4(a) != b; } - - __forceinline vboold4 operator < (const vdouble4& a, double b) { return a < vdouble4(b); } - __forceinline vboold4 operator < (double a, const vdouble4& b) { return vdouble4(a) < b; } - - __forceinline vboold4 operator >=(const vdouble4& a, double b) { return a >= vdouble4(b); } - __forceinline vboold4 operator >=(double a, const vdouble4& b) { return vdouble4(a) >= b; } - - __forceinline vboold4 operator > (const vdouble4& a, double b) { return a > vdouble4(b); } - __forceinline vboold4 operator > (double a, const vdouble4& b) { return vdouble4(a) > b; } - - __forceinline vboold4 operator <=(const vdouble4& a, double b) { return a <= vdouble4(b); } - __forceinline vboold4 operator <=(double a, const vdouble4& b) { return vdouble4(a) <= b; } - - __forceinline vboold4 eq(const vdouble4& a, const vdouble4& b) { return a == b; } - __forceinline vboold4 ne(const vdouble4& a, const vdouble4& b) { return a != b; } - __forceinline vboold4 lt(const vdouble4& a, const vdouble4& b) { return a < b; } - __forceinline vboold4 ge(const vdouble4& a, const vdouble4& b) { return a >= b; } - __forceinline vboold4 gt(const vdouble4& a, const vdouble4& b) { return a > b; } - __forceinline vboold4 le(const vdouble4& a, const vdouble4& b) { return a <= b; } - -#if defined(__AVX512VL__) - __forceinline vboold4 eq(const vboold4& mask, const vdouble4& a, const vdouble4& b) { return _mm256_mask_cmp_pd_mask(mask, a, b, _MM_CMPINT_EQ); } - __forceinline vboold4 ne(const vboold4& mask, const vdouble4& a, const vdouble4& b) { return _mm256_mask_cmp_pd_mask(mask, a, b, _MM_CMPINT_NE); } - __forceinline vboold4 lt(const vboold4& mask, const vdouble4& a, const vdouble4& b) { return _mm256_mask_cmp_pd_mask(mask, a, b, _MM_CMPINT_LT); } - __forceinline vboold4 ge(const vboold4& mask, const vdouble4& a, const vdouble4& b) { return _mm256_mask_cmp_pd_mask(mask, a, b, _MM_CMPINT_GE); } - __forceinline vboold4 gt(const vboold4& mask, const vdouble4& a, const vdouble4& b) { return _mm256_mask_cmp_pd_mask(mask, a, b, _MM_CMPINT_GT); } - __forceinline vboold4 le(const vboold4& mask, const vdouble4& a, const vdouble4& b) { return _mm256_mask_cmp_pd_mask(mask, a, b, _MM_CMPINT_LE); } -#else - __forceinline vboold4 eq(const vboold4& mask, const vdouble4& a, const vdouble4& b) { return mask & (a == b); } - __forceinline vboold4 ne(const vboold4& mask, const vdouble4& a, const vdouble4& b) { return mask & (a != b); } - __forceinline vboold4 lt(const vboold4& mask, const vdouble4& a, const vdouble4& b) { return mask & (a < b); } - __forceinline vboold4 ge(const vboold4& mask, const vdouble4& a, const vdouble4& b) { return mask & (a >= b); } - __forceinline vboold4 gt(const vboold4& mask, const vdouble4& a, const vdouble4& b) { return mask & (a > b); } - __forceinline vboold4 le(const vboold4& mask, const vdouble4& a, const vdouble4& b) { return mask & (a <= b); } -#endif - - __forceinline vdouble4 select(const vboold4& m, const vdouble4& t, const vdouble4& f) { -#if defined(__AVX512VL__) - return _mm256_mask_blend_pd(m, f, t); -#else - return _mm256_blendv_pd(f, t, m); -#endif - } - - __forceinline void xchg(const vboold4& m, vdouble4& a, vdouble4& b) { - const vdouble4 c = a; a = select(m,b,a); b = select(m,c,b); - } - - __forceinline vboold4 test(const vdouble4& a, const vdouble4& b) { -#if defined(__AVX512VL__) - return _mm256_test_epi64_mask(_mm256_castpd_si256(a),_mm256_castpd_si256(b)); -#else - return _mm256_testz_si256(_mm256_castpd_si256(a),_mm256_castpd_si256(b)); -#endif - } - - //////////////////////////////////////////////////////////////////////////////// - // Movement/Shifting/Shuffling Functions - //////////////////////////////////////////////////////////////////////////////// - - template<int i0, int i1> - __forceinline vdouble4 shuffle(const vdouble4& v) { - return _mm256_permute_pd(v, (i1 << 3) | (i0 << 2) | (i1 << 1) | i0); - } - - template<int i> - __forceinline vdouble4 shuffle(const vdouble4& v) { - return shuffle<i, i>(v); - } - - template<int i0, int i1> - __forceinline vdouble4 shuffle2(const vdouble4& v) { - return _mm256_permute2f128_pd(v, v, (i1 << 4) | i0); - } - - __forceinline double toScalar(const vdouble4& v) { - return _mm_cvtsd_f64(_mm256_castpd256_pd128(v)); - } - - //////////////////////////////////////////////////////////////////////////////// - /// Reductions - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vdouble4 vreduce_min2(const vdouble4& x) { return min(x, shuffle<1,0>(x)); } - __forceinline vdouble4 vreduce_min (const vdouble4& y) { const vdouble4 x = vreduce_min2(y); return min(x, shuffle2<1,0>(x)); } - - __forceinline vdouble4 vreduce_max2(const vdouble4& x) { return max(x,shuffle<1,0>(x)); } - __forceinline vdouble4 vreduce_max (const vdouble4& y) { const vdouble4 x = vreduce_max2(y); return max(x, shuffle2<1,0>(x)); } - - __forceinline vdouble4 vreduce_and2(const vdouble4& x) { return x & shuffle<1,0>(x); } - __forceinline vdouble4 vreduce_and (const vdouble4& y) { const vdouble4 x = vreduce_and2(y); return x & shuffle2<1,0>(x); } - - __forceinline vdouble4 vreduce_or2(const vdouble4& x) { return x | shuffle<1,0>(x); } - __forceinline vdouble4 vreduce_or (const vdouble4& y) { const vdouble4 x = vreduce_or2(y); return x | shuffle2<1,0>(x); } - - __forceinline vdouble4 vreduce_add2(const vdouble4& x) { return x + shuffle<1,0>(x); } - __forceinline vdouble4 vreduce_add (const vdouble4& y) { const vdouble4 x = vreduce_add2(y); return x + shuffle2<1,0>(x); } - - __forceinline double reduce_add(const vdouble4& a) { return toScalar(vreduce_add(a)); } - __forceinline double reduce_min(const vdouble4& a) { return toScalar(vreduce_min(a)); } - __forceinline double reduce_max(const vdouble4& a) { return toScalar(vreduce_max(a)); } - - //////////////////////////////////////////////////////////////////////////////// - /// Memory load and store operations - //////////////////////////////////////////////////////////////////////////////// - - - - //////////////////////////////////////////////////////////////////////////////// - /// Output Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline embree_ostream operator <<(embree_ostream cout, const vdouble4& v) - { - cout << "<" << v[0]; - for (size_t i=1; i<4; i++) cout << ", " << v[i]; - cout << ">"; - return cout; - } -} diff --git a/thirdparty/embree-aarch64/common/simd/vdouble8_avx512.h b/thirdparty/embree-aarch64/common/simd/vdouble8_avx512.h deleted file mode 100644 index 4eec7d2f6a..0000000000 --- a/thirdparty/embree-aarch64/common/simd/vdouble8_avx512.h +++ /dev/null @@ -1,356 +0,0 @@ -// Copyright 2009-2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 - -#pragma once - -namespace embree -{ - /* 8-wide AVX-512 64-bit double type */ - template<> - struct vdouble<8> - { - ALIGNED_STRUCT_(64); - - typedef vboold8 Bool; - - enum { size = 8 }; // number of SIMD elements - union { // data - __m512d v; - double i[8]; - }; - - //////////////////////////////////////////////////////////////////////////////// - /// Constructors, Assignment & Cast Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vdouble() {} - __forceinline vdouble(const vdouble8& t) { v = t.v; } - __forceinline vdouble8& operator =(const vdouble8& f) { v = f.v; return *this; } - - __forceinline vdouble(const __m512d& t) { v = t; } - __forceinline operator __m512d() const { return v; } - __forceinline operator __m256d() const { return _mm512_castpd512_pd256(v); } - - __forceinline vdouble(double i) { - v = _mm512_set1_pd(i); - } - - __forceinline vdouble(double a, double b, double c, double d) { - v = _mm512_set4_pd(d,c,b,a); - } - - __forceinline vdouble(double a0, double a1, double a2, double a3, - double a4, double a5, double a6, double a7) - { - v = _mm512_set_pd(a7,a6,a5,a4,a3,a2,a1,a0); - } - - - //////////////////////////////////////////////////////////////////////////////// - /// Constants - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vdouble(ZeroTy) : v(_mm512_setzero_pd()) {} - __forceinline vdouble(OneTy) : v(_mm512_set1_pd(1)) {} - __forceinline vdouble(StepTy) : v(_mm512_set_pd(7.0,6.0,5.0,4.0,3.0,2.0,1.0,0.0)) {} - __forceinline vdouble(ReverseStepTy) : v(_mm512_setr_pd(7.0,6.0,5.0,4.0,3.0,2.0,1.0,0.0)) {} - - //////////////////////////////////////////////////////////////////////////////// - /// Loads and Stores - //////////////////////////////////////////////////////////////////////////////// - - static __forceinline void store_nt(void *__restrict__ ptr, const vdouble8& a) { - _mm512_stream_pd((double*)ptr, a); - } - - static __forceinline vdouble8 loadu(const void* addr) { - return _mm512_loadu_pd((double*)addr); - } - - static __forceinline vdouble8 load(const vdouble8* addr) { - return _mm512_load_pd((double*)addr); - } - - static __forceinline vdouble8 load(const double* addr) { - return _mm512_load_pd(addr); - } - - static __forceinline void store(void* ptr, const vdouble8& v) { - _mm512_store_pd(ptr, v); - } - - static __forceinline void storeu(void* ptr, const vdouble8& v) { - _mm512_storeu_pd(ptr, v); - } - - static __forceinline void storeu(const vboold8& mask, double* ptr, const vdouble8& f) { - _mm512_mask_storeu_pd(ptr, mask, f); - } - - static __forceinline void store(const vboold8& mask, void* addr, const vdouble8& v2) { - _mm512_mask_store_pd(addr, mask, v2); - } - - /* pass by value to avoid compiler generating inefficient code */ - static __forceinline void storeu_compact(const vboold8 mask,void * addr, const vdouble8& reg) { - _mm512_mask_compressstoreu_pd(addr, mask, reg); - } - - static __forceinline vdouble8 compact64bit(const vboold8& mask, vdouble8& v) { - return _mm512_mask_compress_pd(v, mask, v); - } - - static __forceinline vdouble8 compact(const vboold8& mask, vdouble8& v) { - return _mm512_mask_compress_pd(v, mask, v); - } - - static __forceinline vdouble8 compact(const vboold8& mask, const vdouble8& a, vdouble8& b) { - return _mm512_mask_compress_pd(a, mask, b); - } - - static __forceinline vdouble8 broadcast(const void* a) { return _mm512_set1_pd(*(double*)a); } - - //////////////////////////////////////////////////////////////////////////////// - /// Array Access - //////////////////////////////////////////////////////////////////////////////// - - __forceinline double& operator [](size_t index) { assert(index < 8); return i[index]; } - __forceinline const double& operator [](size_t index) const { assert(index < 8); return i[index]; } - - }; - - //////////////////////////////////////////////////////////////////////////////// - /// Unary Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vdouble8 asDouble(const vllong8& a) { return _mm512_castsi512_pd(a); } - __forceinline vllong8 asLLong (const vdouble8& a) { return _mm512_castpd_si512(a); } - - __forceinline vdouble8 operator +(const vdouble8& a) { return a; } - __forceinline vdouble8 operator -(const vdouble8& a) { return _mm512_sub_pd(_mm512_setzero_pd(), a); } - - //////////////////////////////////////////////////////////////////////////////// - /// Binary Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vdouble8 operator +(const vdouble8& a, const vdouble8& b) { return _mm512_add_pd(a, b); } - __forceinline vdouble8 operator +(const vdouble8& a, double b) { return a + vdouble8(b); } - __forceinline vdouble8 operator +(double a, const vdouble8& b) { return vdouble8(a) + b; } - - __forceinline vdouble8 operator -(const vdouble8& a, const vdouble8& b) { return _mm512_sub_pd(a, b); } - __forceinline vdouble8 operator -(const vdouble8& a, double b) { return a - vdouble8(b); } - __forceinline vdouble8 operator -(double a, const vdouble8& b) { return vdouble8(a) - b; } - - __forceinline vdouble8 operator *(const vdouble8& a, const vdouble8& b) { return _mm512_mul_pd(a, b); } - __forceinline vdouble8 operator *(const vdouble8& a, double b) { return a * vdouble8(b); } - __forceinline vdouble8 operator *(double a, const vdouble8& b) { return vdouble8(a) * b; } - - __forceinline vdouble8 operator &(const vdouble8& a, const vdouble8& b) { return _mm512_and_pd(a, b); } - __forceinline vdouble8 operator &(const vdouble8& a, double b) { return a & vdouble8(b); } - __forceinline vdouble8 operator &(double a, const vdouble8& b) { return vdouble8(a) & b; } - - __forceinline vdouble8 operator |(const vdouble8& a, const vdouble8& b) { return _mm512_or_pd(a, b); } - __forceinline vdouble8 operator |(const vdouble8& a, double b) { return a | vdouble8(b); } - __forceinline vdouble8 operator |(double a, const vdouble8& b) { return vdouble8(a) | b; } - - __forceinline vdouble8 operator ^(const vdouble8& a, const vdouble8& b) { return _mm512_xor_pd(a, b); } - __forceinline vdouble8 operator ^(const vdouble8& a, double b) { return a ^ vdouble8(b); } - __forceinline vdouble8 operator ^(double a, const vdouble8& b) { return vdouble8(a) ^ b; } - - __forceinline vdouble8 operator <<(const vdouble8& a, const unsigned int n) { return _mm512_castsi512_pd(_mm512_slli_epi64(_mm512_castpd_si512(a), n)); } - __forceinline vdouble8 operator >>(const vdouble8& a, const unsigned int n) { return _mm512_castsi512_pd(_mm512_srai_epi64(_mm512_castpd_si512(a), n)); } - - __forceinline vdouble8 operator <<(const vdouble8& a, const vllong8& n) { return _mm512_castsi512_pd(_mm512_sllv_epi64(_mm512_castpd_si512(a), n)); } - __forceinline vdouble8 operator >>(const vdouble8& a, const vllong8& n) { return _mm512_castsi512_pd(_mm512_srav_epi64(_mm512_castpd_si512(a), n)); } - - __forceinline vdouble8 sll (const vdouble8& a, const unsigned int b) { return _mm512_castsi512_pd(_mm512_slli_epi64(_mm512_castpd_si512(a), b)); } - __forceinline vdouble8 sra (const vdouble8& a, const unsigned int b) { return _mm512_castsi512_pd(_mm512_srai_epi64(_mm512_castpd_si512(a), b)); } - __forceinline vdouble8 srl (const vdouble8& a, const unsigned int b) { return _mm512_castsi512_pd(_mm512_srli_epi64(_mm512_castpd_si512(a), b)); } - - __forceinline vdouble8 min(const vdouble8& a, const vdouble8& b) { return _mm512_min_pd(a, b); } - __forceinline vdouble8 min(const vdouble8& a, double b) { return min(a,vdouble8(b)); } - __forceinline vdouble8 min(double a, const vdouble8& b) { return min(vdouble8(a),b); } - - __forceinline vdouble8 max(const vdouble8& a, const vdouble8& b) { return _mm512_max_pd(a, b); } - __forceinline vdouble8 max(const vdouble8& a, double b) { return max(a,vdouble8(b)); } - __forceinline vdouble8 max(double a, const vdouble8& b) { return max(vdouble8(a),b); } - - __forceinline vdouble8 mask_add(const vboold8& mask, vdouble8& c, const vdouble8& a, const vdouble8& b) { return _mm512_mask_add_pd(c,mask,a,b); } - __forceinline vdouble8 mask_sub(const vboold8& mask, vdouble8& c, const vdouble8& a, const vdouble8& b) { return _mm512_mask_sub_pd(c,mask,a,b); } - - __forceinline vdouble8 mask_and(const vboold8& m,vdouble8& c, const vdouble8& a, const vdouble8& b) { return _mm512_mask_and_pd(c,m,a,b); } - __forceinline vdouble8 mask_or (const vboold8& m,vdouble8& c, const vdouble8& a, const vdouble8& b) { return _mm512_mask_or_pd(c,m,a,b); } - - //////////////////////////////////////////////////////////////////////////////// - /// Ternary Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vdouble8 madd (const vdouble8& a, const vdouble8& b, const vdouble8& c) { return _mm512_fmadd_pd(a,b,c); } - __forceinline vdouble8 msub (const vdouble8& a, const vdouble8& b, const vdouble8& c) { return _mm512_fmsub_pd(a,b,c); } - __forceinline vdouble8 nmadd(const vdouble8& a, const vdouble8& b, const vdouble8& c) { return _mm512_fnmadd_pd(a,b,c); } - __forceinline vdouble8 nmsub(const vdouble8& a, const vdouble8& b, const vdouble8& c) { return _mm512_fnmsub_pd(a,b,c); } - - - //////////////////////////////////////////////////////////////////////////////// - /// Assignment Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vdouble8& operator +=(vdouble8& a, const vdouble8& b) { return a = a + b; } - __forceinline vdouble8& operator +=(vdouble8& a, double b) { return a = a + b; } - - __forceinline vdouble8& operator -=(vdouble8& a, const vdouble8& b) { return a = a - b; } - __forceinline vdouble8& operator -=(vdouble8& a, double b) { return a = a - b; } - - __forceinline vdouble8& operator *=(vdouble8& a, const vdouble8& b) { return a = a * b; } - __forceinline vdouble8& operator *=(vdouble8& a, double b) { return a = a * b; } - - __forceinline vdouble8& operator &=(vdouble8& a, const vdouble8& b) { return a = a & b; } - __forceinline vdouble8& operator &=(vdouble8& a, double b) { return a = a & b; } - - __forceinline vdouble8& operator |=(vdouble8& a, const vdouble8& b) { return a = a | b; } - __forceinline vdouble8& operator |=(vdouble8& a, double b) { return a = a | b; } - - __forceinline vdouble8& operator <<=(vdouble8& a, const double b) { return a = a << b; } - __forceinline vdouble8& operator >>=(vdouble8& a, const double b) { return a = a >> b; } - - - //////////////////////////////////////////////////////////////////////////////// - /// Comparison Operators + Select - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboold8 operator ==(const vdouble8& a, const vdouble8& b) { return _mm512_cmp_pd_mask(a,b,_MM_CMPINT_EQ); } - __forceinline vboold8 operator ==(const vdouble8& a, double b) { return a == vdouble8(b); } - __forceinline vboold8 operator ==(double a, const vdouble8& b) { return vdouble8(a) == b; } - - __forceinline vboold8 operator !=(const vdouble8& a, const vdouble8& b) { return _mm512_cmp_pd_mask(a,b,_MM_CMPINT_NE); } - __forceinline vboold8 operator !=(const vdouble8& a, double b) { return a != vdouble8(b); } - __forceinline vboold8 operator !=(double a, const vdouble8& b) { return vdouble8(a) != b; } - - __forceinline vboold8 operator < (const vdouble8& a, const vdouble8& b) { return _mm512_cmp_pd_mask(a,b,_MM_CMPINT_LT); } - __forceinline vboold8 operator < (const vdouble8& a, double b) { return a < vdouble8(b); } - __forceinline vboold8 operator < (double a, const vdouble8& b) { return vdouble8(a) < b; } - - __forceinline vboold8 operator >=(const vdouble8& a, const vdouble8& b) { return _mm512_cmp_pd_mask(a,b,_MM_CMPINT_GE); } - __forceinline vboold8 operator >=(const vdouble8& a, double b) { return a >= vdouble8(b); } - __forceinline vboold8 operator >=(double a, const vdouble8& b) { return vdouble8(a) >= b; } - - __forceinline vboold8 operator > (const vdouble8& a, const vdouble8& b) { return _mm512_cmp_pd_mask(a,b,_MM_CMPINT_GT); } - __forceinline vboold8 operator > (const vdouble8& a, double b) { return a > vdouble8(b); } - __forceinline vboold8 operator > (double a, const vdouble8& b) { return vdouble8(a) > b; } - - __forceinline vboold8 operator <=(const vdouble8& a, const vdouble8& b) { return _mm512_cmp_pd_mask(a,b,_MM_CMPINT_LE); } - __forceinline vboold8 operator <=(const vdouble8& a, double b) { return a <= vdouble8(b); } - __forceinline vboold8 operator <=(double a, const vdouble8& b) { return vdouble8(a) <= b; } - - __forceinline vboold8 eq(const vdouble8& a, const vdouble8& b) { return _mm512_cmp_pd_mask(a,b,_MM_CMPINT_EQ); } - __forceinline vboold8 ne(const vdouble8& a, const vdouble8& b) { return _mm512_cmp_pd_mask(a,b,_MM_CMPINT_NE); } - __forceinline vboold8 lt(const vdouble8& a, const vdouble8& b) { return _mm512_cmp_pd_mask(a,b,_MM_CMPINT_LT); } - __forceinline vboold8 ge(const vdouble8& a, const vdouble8& b) { return _mm512_cmp_pd_mask(a,b,_MM_CMPINT_GE); } - __forceinline vboold8 gt(const vdouble8& a, const vdouble8& b) { return _mm512_cmp_pd_mask(a,b,_MM_CMPINT_GT); } - __forceinline vboold8 le(const vdouble8& a, const vdouble8& b) { return _mm512_cmp_pd_mask(a,b,_MM_CMPINT_LE); } - - __forceinline vboold8 eq(const vboold8 mask, const vdouble8& a, const vdouble8& b) { return _mm512_mask_cmp_pd_mask(mask,a,b,_MM_CMPINT_EQ); } - __forceinline vboold8 ne(const vboold8 mask, const vdouble8& a, const vdouble8& b) { return _mm512_mask_cmp_pd_mask(mask,a,b,_MM_CMPINT_NE); } - __forceinline vboold8 lt(const vboold8 mask, const vdouble8& a, const vdouble8& b) { return _mm512_mask_cmp_pd_mask(mask,a,b,_MM_CMPINT_LT); } - __forceinline vboold8 ge(const vboold8 mask, const vdouble8& a, const vdouble8& b) { return _mm512_mask_cmp_pd_mask(mask,a,b,_MM_CMPINT_GE); } - __forceinline vboold8 gt(const vboold8 mask, const vdouble8& a, const vdouble8& b) { return _mm512_mask_cmp_pd_mask(mask,a,b,_MM_CMPINT_GT); } - __forceinline vboold8 le(const vboold8 mask, const vdouble8& a, const vdouble8& b) { return _mm512_mask_cmp_pd_mask(mask,a,b,_MM_CMPINT_LE); } - - __forceinline vdouble8 select(const vboold8& m, const vdouble8& t, const vdouble8& f) { - return _mm512_mask_or_pd(f,m,t,t); - } - - __forceinline void xchg(const vboold8& m, vdouble8& a, vdouble8& b) { - const vdouble8 c = a; a = select(m,b,a); b = select(m,c,b); - } - - __forceinline vboold8 test(const vboold8& m, const vdouble8& a, const vdouble8& b) { - return _mm512_mask_test_epi64_mask(m,_mm512_castpd_si512(a),_mm512_castpd_si512(b)); - } - - __forceinline vboold8 test(const vdouble8& a, const vdouble8& b) { - return _mm512_test_epi64_mask(_mm512_castpd_si512(a),_mm512_castpd_si512(b)); - } - - //////////////////////////////////////////////////////////////////////////////// - // Movement/Shifting/Shuffling Functions - //////////////////////////////////////////////////////////////////////////////// - - template<int i0, int i1> - __forceinline vdouble8 shuffle(const vdouble8& v) { - return _mm512_permute_pd(v, (i1 << 7) | (i0 << 6) | (i1 << 5) | (i0 << 4) | (i1 << 3) | (i0 << 2) | (i1 << 1) | i0); - } - - template<int i> - __forceinline vdouble8 shuffle(const vdouble8& v) { - return shuffle<i, i>(v); - } - - template<int i0, int i1, int i2, int i3> - __forceinline vdouble8 shuffle(const vdouble8& v) { - return _mm512_permutex_pd(v, _MM_SHUFFLE(i3, i2, i1, i0)); - } - - template<int i0, int i1> - __forceinline vdouble8 shuffle4(const vdouble8& v) { - return _mm512_shuffle_f64x2(v, v, _MM_SHUFFLE(i1*2+1, i1*2, i0*2+1, i0*2)); - } - - template<int i> - __forceinline vdouble8 shuffle4(const vdouble8& v) { - return shuffle4<i, i>(v); - } - - template<int i> - __forceinline vdouble8 align_shift_right(const vdouble8& a, const vdouble8& b) { - return _mm512_castsi512_pd(_mm512_alignr_epi64(_mm512_castpd_si512(a), _mm512_castpd_si512(b), i)); - } - - __forceinline double toScalar(const vdouble8& v) { - return _mm_cvtsd_f64(_mm512_castpd512_pd128(v)); - } - - //////////////////////////////////////////////////////////////////////////////// - /// Reductions - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vdouble8 vreduce_add2(vdouble8 x) { return x + shuffle<1,0,3,2>(x); } - __forceinline vdouble8 vreduce_add4(vdouble8 x) { x = vreduce_add2(x); return x + shuffle<2,3,0,1>(x); } - __forceinline vdouble8 vreduce_add (vdouble8 x) { x = vreduce_add4(x); return x + shuffle4<1,0>(x); } - - __forceinline vdouble8 vreduce_min2(vdouble8 x) { return min(x, shuffle<1,0,3,2>(x)); } - __forceinline vdouble8 vreduce_min4(vdouble8 x) { x = vreduce_min2(x); return min(x, shuffle<2,3,0,1>(x)); } - __forceinline vdouble8 vreduce_min (vdouble8 x) { x = vreduce_min4(x); return min(x, shuffle4<1,0>(x)); } - - __forceinline vdouble8 vreduce_max2(vdouble8 x) { return max(x, shuffle<1,0,3,2>(x)); } - __forceinline vdouble8 vreduce_max4(vdouble8 x) { x = vreduce_max2(x); return max(x, shuffle<2,3,0,1>(x)); } - __forceinline vdouble8 vreduce_max (vdouble8 x) { x = vreduce_max4(x); return max(x, shuffle4<1,0>(x)); } - - __forceinline double reduce_add(const vdouble8& v) { return toScalar(vreduce_add(v)); } - __forceinline double reduce_min(const vdouble8& v) { return toScalar(vreduce_min(v)); } - __forceinline double reduce_max(const vdouble8& v) { return toScalar(vreduce_max(v)); } - - //////////////////////////////////////////////////////////////////////////////// - /// Memory load and store operations - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vdouble8 permute(const vdouble8& v, const vllong8& index) { - return _mm512_permutexvar_pd(index, v); - } - - __forceinline vdouble8 reverse(const vdouble8& a) { - return permute(a, vllong8(reverse_step)); - } - - //////////////////////////////////////////////////////////////////////////////// - /// Output Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline embree_ostream operator <<(embree_ostream cout, const vdouble8& v) - { - cout << "<" << v[0]; - for (size_t i=1; i<8; i++) cout << ", " << v[i]; - cout << ">"; - return cout; - } -} diff --git a/thirdparty/embree-aarch64/common/simd/vfloat16_avx512.h b/thirdparty/embree-aarch64/common/simd/vfloat16_avx512.h deleted file mode 100644 index aed2419b77..0000000000 --- a/thirdparty/embree-aarch64/common/simd/vfloat16_avx512.h +++ /dev/null @@ -1,771 +0,0 @@ -// Copyright 2009-2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 - -#pragma once - -namespace embree -{ - /* 16-wide AVX-512 float type */ - template<> - struct vfloat<16> - { - ALIGNED_STRUCT_(64); - - typedef vboolf16 Bool; - typedef vint16 Int; - typedef vfloat16 Float; - - enum { size = 16 }; // number of SIMD elements - union { // data - __m512 v; - float f[16]; - int i[16]; - }; - - //////////////////////////////////////////////////////////////////////////////// - /// Constructors, Assignment & Cast Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vfloat() {} - __forceinline vfloat(const vfloat16& t) { v = t; } - __forceinline vfloat16& operator =(const vfloat16& f) { v = f.v; return *this; } - - __forceinline vfloat(const __m512& t) { v = t; } - __forceinline operator __m512() const { return v; } - __forceinline operator __m256() const { return _mm512_castps512_ps256(v); } - __forceinline operator __m128() const { return _mm512_castps512_ps128(v); } - - __forceinline vfloat(float f) { - v = _mm512_set1_ps(f); - } - - __forceinline vfloat(float a, float b, float c, float d) { - v = _mm512_set4_ps(a, b, c, d); - } - - __forceinline vfloat(const vfloat4& i) { - v = _mm512_broadcast_f32x4(i); - } - - __forceinline vfloat(const vfloat4& a, const vfloat4& b, const vfloat4& c, const vfloat4& d) { - v = _mm512_castps128_ps512(a); - v = _mm512_insertf32x4(v, b, 1); - v = _mm512_insertf32x4(v, c, 2); - v = _mm512_insertf32x4(v, d, 3); - } - - __forceinline vfloat(const vboolf16& mask, const vfloat4& a, const vfloat4& b) { - v = _mm512_broadcast_f32x4(a); - v = _mm512_mask_broadcast_f32x4(v,mask,b); - } - - __forceinline vfloat(const vfloat8& i) { - v = _mm512_castpd_ps(_mm512_broadcast_f64x4(_mm256_castps_pd(i))); - } - - __forceinline vfloat(const vfloat8& a, const vfloat8& b) { - v = _mm512_castps256_ps512(a); -#if defined(__AVX512DQ__) - v = _mm512_insertf32x8(v, b, 1); -#else - v = _mm512_castpd_ps(_mm512_insertf64x4(_mm512_castps_pd(v), _mm256_castps_pd(b), 1)); -#endif - } - - /* WARNING: due to f64x4 the mask is considered as an 8bit mask */ - __forceinline vfloat(const vboolf16& mask, const vfloat8& a, const vfloat8& b) { - __m512d aa = _mm512_broadcast_f64x4(_mm256_castps_pd(a)); - aa = _mm512_mask_broadcast_f64x4(aa,mask,_mm256_castps_pd(b)); - v = _mm512_castpd_ps(aa); - } - - __forceinline explicit vfloat(const vint16& a) { - v = _mm512_cvtepi32_ps(a); - } - - __forceinline explicit vfloat(const vuint16& a) { - v = _mm512_cvtepu32_ps(a); - } - - //////////////////////////////////////////////////////////////////////////////// - /// Constants - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vfloat(ZeroTy) : v(_mm512_setzero_ps()) {} - __forceinline vfloat(OneTy) : v(_mm512_set1_ps(1.0f)) {} - __forceinline vfloat(PosInfTy) : v(_mm512_set1_ps(pos_inf)) {} - __forceinline vfloat(NegInfTy) : v(_mm512_set1_ps(neg_inf)) {} - __forceinline vfloat(StepTy) : v(_mm512_set_ps(15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0)) {} - __forceinline vfloat(NaNTy) : v(_mm512_set1_ps(nan)) {} - __forceinline vfloat(UndefinedTy) : v(_mm512_undefined_ps()) {} - - //////////////////////////////////////////////////////////////////////////////// - /// Loads and Stores - //////////////////////////////////////////////////////////////////////////////// - - static __forceinline vfloat16 load (const void* ptr) { return _mm512_load_ps((float*)ptr); } - static __forceinline vfloat16 loadu(const void* ptr) { return _mm512_loadu_ps((float*)ptr); } - - static __forceinline vfloat16 load (const vboolf16& mask, const void* ptr) { return _mm512_mask_load_ps (_mm512_setzero_ps(),mask,(float*)ptr); } - static __forceinline vfloat16 loadu(const vboolf16& mask, const void* ptr) { return _mm512_mask_loadu_ps(_mm512_setzero_ps(),mask,(float*)ptr); } - - static __forceinline void store (void* ptr, const vfloat16& v) { _mm512_store_ps ((float*)ptr,v); } - static __forceinline void storeu(void* ptr, const vfloat16& v) { _mm512_storeu_ps((float*)ptr,v); } - - static __forceinline void store (const vboolf16& mask, void* ptr, const vfloat16& v) { _mm512_mask_store_ps ((float*)ptr,mask,v); } - static __forceinline void storeu(const vboolf16& mask, void* ptr, const vfloat16& v) { _mm512_mask_storeu_ps((float*)ptr,mask,v); } - - static __forceinline void store_nt(void* __restrict__ ptr, const vfloat16& a) { - _mm512_stream_ps((float*)ptr,a); - } - - static __forceinline vfloat16 broadcast(const float* f) { - return _mm512_set1_ps(*f); - } - - static __forceinline vfloat16 compact(const vboolf16& mask, vfloat16 &v) { - return _mm512_mask_compress_ps(v, mask, v); - } - static __forceinline vfloat16 compact(const vboolf16& mask, vfloat16 &a, const vfloat16& b) { - return _mm512_mask_compress_ps(a, mask, b); - } - - static __forceinline vfloat16 expand(const vboolf16& mask, const vfloat16& a, vfloat16& b) { - return _mm512_mask_expand_ps(b, mask, a); - } - - static __forceinline vfloat16 loadu_compact(const vboolf16& mask, const void* ptr) { - return _mm512_mask_expandloadu_ps(_mm512_setzero_ps(), mask, (float*)ptr); - } - - static __forceinline void storeu_compact(const vboolf16& mask, float *addr, const vfloat16 reg) { - _mm512_mask_compressstoreu_ps(addr, mask, reg); - } - - static __forceinline void storeu_compact_single(const vboolf16& mask, float * addr, const vfloat16& reg) { - //_mm512_mask_compressstoreu_ps(addr,mask,reg); - *addr = mm512_cvtss_f32(_mm512_mask_compress_ps(reg, mask, reg)); - } - - template<int scale = 4> - static __forceinline vfloat16 gather(const float* ptr, const vint16& index) { - return _mm512_i32gather_ps(index, ptr, scale); - } - - template<int scale = 4> - static __forceinline vfloat16 gather(const vboolf16& mask, const float* ptr, const vint16& index) { - vfloat16 r = zero; - return _mm512_mask_i32gather_ps(r, mask, index, ptr, scale); - } - - template<int scale = 4> - static __forceinline void scatter(float* ptr, const vint16& index, const vfloat16& v) { - _mm512_i32scatter_ps(ptr, index, v, scale); - } - - template<int scale = 4> - static __forceinline void scatter(const vboolf16& mask, float* ptr, const vint16& index, const vfloat16& v) { - _mm512_mask_i32scatter_ps(ptr, mask, index, v, scale); - } - - //////////////////////////////////////////////////////////////////////////////// - /// Array Access - //////////////////////////////////////////////////////////////////////////////// - - __forceinline float& operator [](size_t index) { assert(index < 16); return f[index]; } - __forceinline const float& operator [](size_t index) const { assert(index < 16); return f[index]; } - }; - - //////////////////////////////////////////////////////////////////////////////// - /// Unary Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vfloat16 asFloat(const vint16& a) { return _mm512_castsi512_ps(a); } - __forceinline vint16 asInt (const vfloat16& a) { return _mm512_castps_si512(a); } - __forceinline vuint16 asUInt (const vfloat16& a) { return _mm512_castps_si512(a); } - - __forceinline vint16 toInt (const vfloat16& a) { return vint16(a); } - __forceinline vfloat16 toFloat(const vint16& a) { return vfloat16(a); } - - __forceinline vfloat16 operator +(const vfloat16& a) { return a; } - __forceinline vfloat16 operator -(const vfloat16& a) { return _mm512_mul_ps(a,vfloat16(-1)); } - - __forceinline vfloat16 abs (const vfloat16& a) { return _mm512_castsi512_ps(_mm512_and_epi32(_mm512_castps_si512(a),_mm512_set1_epi32(0x7FFFFFFF))); } - __forceinline vfloat16 signmsk(const vfloat16& a) { return _mm512_castsi512_ps(_mm512_and_epi32(_mm512_castps_si512(a),_mm512_set1_epi32(0x80000000))); } - - __forceinline vfloat16 rcp(const vfloat16& a) { -#if defined(__AVX512ER__) - return _mm512_rcp28_ps(a); -#else - const vfloat16 r = _mm512_rcp14_ps(a); - return _mm512_mul_ps(r, _mm512_fnmadd_ps(r, a, vfloat16(2.0f))); -#endif - } - - __forceinline vfloat16 sqr (const vfloat16& a) { return _mm512_mul_ps(a,a); } - __forceinline vfloat16 sqrt(const vfloat16& a) { return _mm512_sqrt_ps(a); } - - __forceinline vfloat16 rsqrt(const vfloat16& a) - { -#if defined(__AVX512VL__) - const vfloat16 r = _mm512_rsqrt14_ps(a); - return _mm512_fmadd_ps(_mm512_set1_ps(1.5f), r, - _mm512_mul_ps(_mm512_mul_ps(_mm512_mul_ps(a, _mm512_set1_ps(-0.5f)), r), _mm512_mul_ps(r, r))); -#else - return _mm512_rsqrt28_ps(a); -#endif - } - - //////////////////////////////////////////////////////////////////////////////// - /// Binary Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vfloat16 operator +(const vfloat16& a, const vfloat16& b) { return _mm512_add_ps(a, b); } - __forceinline vfloat16 operator +(const vfloat16& a, float b) { return a + vfloat16(b); } - __forceinline vfloat16 operator +(float a, const vfloat16& b) { return vfloat16(a) + b; } - - __forceinline vfloat16 operator -(const vfloat16& a, const vfloat16& b) { return _mm512_sub_ps(a, b); } - __forceinline vfloat16 operator -(const vfloat16& a, float b) { return a - vfloat16(b); } - __forceinline vfloat16 operator -(float a, const vfloat16& b) { return vfloat16(a) - b; } - - __forceinline vfloat16 operator *(const vfloat16& a, const vfloat16& b) { return _mm512_mul_ps(a, b); } - __forceinline vfloat16 operator *(const vfloat16& a, float b) { return a * vfloat16(b); } - __forceinline vfloat16 operator *(float a, const vfloat16& b) { return vfloat16(a) * b; } - - __forceinline vfloat16 operator /(const vfloat16& a, const vfloat16& b) { return _mm512_div_ps(a,b); } - __forceinline vfloat16 operator /(const vfloat16& a, float b) { return a/vfloat16(b); } - __forceinline vfloat16 operator /(float a, const vfloat16& b) { return vfloat16(a)/b; } - - __forceinline vfloat16 operator &(const vfloat16& a, const vfloat16& b) { return _mm512_and_ps(a,b); } - __forceinline vfloat16 operator |(const vfloat16& a, const vfloat16& b) { return _mm512_or_ps(a,b); } - __forceinline vfloat16 operator ^(const vfloat16& a, const vfloat16& b) { - return _mm512_castsi512_ps(_mm512_xor_epi32(_mm512_castps_si512(a),_mm512_castps_si512(b))); - } - - __forceinline vfloat16 min(const vfloat16& a, const vfloat16& b) { - return _mm512_min_ps(a,b); - } - __forceinline vfloat16 min(const vfloat16& a, float b) { - return _mm512_min_ps(a,vfloat16(b)); - } - __forceinline vfloat16 min(const float& a, const vfloat16& b) { - return _mm512_min_ps(vfloat16(a),b); - } - - __forceinline vfloat16 max(const vfloat16& a, const vfloat16& b) { - return _mm512_max_ps(a,b); - } - __forceinline vfloat16 max(const vfloat16& a, float b) { - return _mm512_max_ps(a,vfloat16(b)); - } - __forceinline vfloat16 max(const float& a, const vfloat16& b) { - return _mm512_max_ps(vfloat16(a),b); - } - - __forceinline vfloat16 mask_add(const vboolf16& mask, const vfloat16& c, const vfloat16& a, const vfloat16& b) { return _mm512_mask_add_ps (c,mask,a,b); } - __forceinline vfloat16 mask_min(const vboolf16& mask, const vfloat16& c, const vfloat16& a, const vfloat16& b) { - return _mm512_mask_min_ps(c,mask,a,b); - }; - __forceinline vfloat16 mask_max(const vboolf16& mask, const vfloat16& c, const vfloat16& a, const vfloat16& b) { - return _mm512_mask_max_ps(c,mask,a,b); - }; - - __forceinline vfloat16 mini(const vfloat16& a, const vfloat16& b) { -#if !defined(__AVX512ER__) // SKX - const vint16 ai = _mm512_castps_si512(a); - const vint16 bi = _mm512_castps_si512(b); - const vint16 ci = _mm512_min_epi32(ai,bi); - return _mm512_castsi512_ps(ci); -#else // KNL - return min(a,b); -#endif - } - - __forceinline vfloat16 maxi(const vfloat16& a, const vfloat16& b) { -#if !defined(__AVX512ER__) // SKX - const vint16 ai = _mm512_castps_si512(a); - const vint16 bi = _mm512_castps_si512(b); - const vint16 ci = _mm512_max_epi32(ai,bi); - return _mm512_castsi512_ps(ci); -#else // KNL - return max(a,b); -#endif - } - - //////////////////////////////////////////////////////////////////////////////// - /// Ternary Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vfloat16 madd (const vfloat16& a, const vfloat16& b, const vfloat16& c) { return _mm512_fmadd_ps(a,b,c); } - __forceinline vfloat16 msub (const vfloat16& a, const vfloat16& b, const vfloat16& c) { return _mm512_fmsub_ps(a,b,c); } - __forceinline vfloat16 nmadd(const vfloat16& a, const vfloat16& b, const vfloat16& c) { return _mm512_fnmadd_ps(a,b,c); } - __forceinline vfloat16 nmsub(const vfloat16& a, const vfloat16& b, const vfloat16& c) { return _mm512_fnmsub_ps(a,b,c); } - - __forceinline vfloat16 mask_msub(const vboolf16& mask,const vfloat16& a, const vfloat16& b, const vfloat16& c) { return _mm512_mask_fmsub_ps(a,mask,b,c); } - - __forceinline vfloat16 madd231 (const vfloat16& a, const vfloat16& b, const vfloat16& c) { return _mm512_fmadd_ps(c,b,a); } - __forceinline vfloat16 msub213 (const vfloat16& a, const vfloat16& b, const vfloat16& c) { return _mm512_fmsub_ps(a,b,c); } - __forceinline vfloat16 msub231 (const vfloat16& a, const vfloat16& b, const vfloat16& c) { return _mm512_fmsub_ps(c,b,a); } - __forceinline vfloat16 msubr231(const vfloat16& a, const vfloat16& b, const vfloat16& c) { return _mm512_fnmadd_ps(c,b,a); } - - - //////////////////////////////////////////////////////////////////////////////// - /// Operators with rounding - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vfloat16 madd_round_down(const vfloat16& a, const vfloat16& b, const vfloat16& c) { return _mm512_fmadd_round_ps(a,b,c,_MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC); } - __forceinline vfloat16 madd_round_up (const vfloat16& a, const vfloat16& b, const vfloat16& c) { return _mm512_fmadd_round_ps(a,b,c,_MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC); } - - __forceinline vfloat16 mul_round_down(const vfloat16& a, const vfloat16& b) { return _mm512_mul_round_ps(a,b,_MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC); } - __forceinline vfloat16 mul_round_up (const vfloat16& a, const vfloat16& b) { return _mm512_mul_round_ps(a,b,_MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC); } - - __forceinline vfloat16 add_round_down(const vfloat16& a, const vfloat16& b) { return _mm512_add_round_ps(a,b,_MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC); } - __forceinline vfloat16 add_round_up (const vfloat16& a, const vfloat16& b) { return _mm512_add_round_ps(a,b,_MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC); } - - __forceinline vfloat16 sub_round_down(const vfloat16& a, const vfloat16& b) { return _mm512_sub_round_ps(a,b,_MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC); } - __forceinline vfloat16 sub_round_up (const vfloat16& a, const vfloat16& b) { return _mm512_sub_round_ps(a,b,_MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC); } - - __forceinline vfloat16 div_round_down(const vfloat16& a, const vfloat16& b) { return _mm512_div_round_ps(a,b,_MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC); } - __forceinline vfloat16 div_round_up (const vfloat16& a, const vfloat16& b) { return _mm512_div_round_ps(a,b,_MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC); } - - __forceinline vfloat16 mask_msub_round_down(const vboolf16& mask,const vfloat16& a, const vfloat16& b, const vfloat16& c) { return _mm512_mask_fmsub_round_ps(a,mask,b,c,_MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC); } - __forceinline vfloat16 mask_msub_round_up (const vboolf16& mask,const vfloat16& a, const vfloat16& b, const vfloat16& c) { return _mm512_mask_fmsub_round_ps(a,mask,b,c,_MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC); } - - __forceinline vfloat16 mask_mul_round_down(const vboolf16& mask,const vfloat16& a, const vfloat16& b, const vfloat16& c) { return _mm512_mask_mul_round_ps(a,mask,b,c,_MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC); } - __forceinline vfloat16 mask_mul_round_up (const vboolf16& mask,const vfloat16& a, const vfloat16& b, const vfloat16& c) { return _mm512_mask_mul_round_ps(a,mask,b,c,_MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC); } - - __forceinline vfloat16 mask_sub_round_down(const vboolf16& mask,const vfloat16& a, const vfloat16& b, const vfloat16& c) { return _mm512_mask_sub_round_ps(a,mask,b,c,_MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC); } - __forceinline vfloat16 mask_sub_round_up (const vboolf16& mask,const vfloat16& a, const vfloat16& b, const vfloat16& c) { return _mm512_mask_sub_round_ps(a,mask,b,c,_MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC); } - - - //////////////////////////////////////////////////////////////////////////////// - /// Assignment Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vfloat16& operator +=(vfloat16& a, const vfloat16& b) { return a = a + b; } - __forceinline vfloat16& operator +=(vfloat16& a, float b) { return a = a + b; } - - __forceinline vfloat16& operator -=(vfloat16& a, const vfloat16& b) { return a = a - b; } - __forceinline vfloat16& operator -=(vfloat16& a, float b) { return a = a - b; } - - __forceinline vfloat16& operator *=(vfloat16& a, const vfloat16& b) { return a = a * b; } - __forceinline vfloat16& operator *=(vfloat16& a, float b) { return a = a * b; } - - __forceinline vfloat16& operator /=(vfloat16& a, const vfloat16& b) { return a = a / b; } - __forceinline vfloat16& operator /=(vfloat16& a, float b) { return a = a / b; } - - //////////////////////////////////////////////////////////////////////////////// - /// Comparison Operators + Select - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboolf16 operator ==(const vfloat16& a, const vfloat16& b) { return _mm512_cmp_ps_mask(a,b,_MM_CMPINT_EQ); } - __forceinline vboolf16 operator ==(const vfloat16& a, float b) { return a == vfloat16(b); } - __forceinline vboolf16 operator ==(float a, const vfloat16& b) { return vfloat16(a) == b; } - - __forceinline vboolf16 operator !=(const vfloat16& a, const vfloat16& b) { return _mm512_cmp_ps_mask(a,b,_MM_CMPINT_NE); } - __forceinline vboolf16 operator !=(const vfloat16& a, float b) { return a != vfloat16(b); } - __forceinline vboolf16 operator !=(float a, const vfloat16& b) { return vfloat16(a) != b; } - - __forceinline vboolf16 operator < (const vfloat16& a, const vfloat16& b) { return _mm512_cmp_ps_mask(a,b,_MM_CMPINT_LT); } - __forceinline vboolf16 operator < (const vfloat16& a, float b) { return a < vfloat16(b); } - __forceinline vboolf16 operator < (float a, const vfloat16& b) { return vfloat16(a) < b; } - - __forceinline vboolf16 operator >=(const vfloat16& a, const vfloat16& b) { return _mm512_cmp_ps_mask(a,b,_MM_CMPINT_GE); } - __forceinline vboolf16 operator >=(const vfloat16& a, float b) { return a >= vfloat16(b); } - __forceinline vboolf16 operator >=(float a, const vfloat16& b) { return vfloat16(a) >= b; } - - __forceinline vboolf16 operator > (const vfloat16& a, const vfloat16& b) { return _mm512_cmp_ps_mask(a,b,_MM_CMPINT_GT); } - __forceinline vboolf16 operator > (const vfloat16& a, float b) { return a > vfloat16(b); } - __forceinline vboolf16 operator > (float a, const vfloat16& b) { return vfloat16(a) > b; } - - __forceinline vboolf16 operator <=(const vfloat16& a, const vfloat16& b) { return _mm512_cmp_ps_mask(a,b,_MM_CMPINT_LE); } - __forceinline vboolf16 operator <=(const vfloat16& a, float b) { return a <= vfloat16(b); } - __forceinline vboolf16 operator <=(float a, const vfloat16& b) { return vfloat16(a) <= b; } - - __forceinline vboolf16 eq(const vfloat16& a, const vfloat16& b) { return _mm512_cmp_ps_mask(a,b,_MM_CMPINT_EQ); } - __forceinline vboolf16 ne(const vfloat16& a, const vfloat16& b) { return _mm512_cmp_ps_mask(a,b,_MM_CMPINT_NE); } - __forceinline vboolf16 lt(const vfloat16& a, const vfloat16& b) { return _mm512_cmp_ps_mask(a,b,_MM_CMPINT_LT); } - __forceinline vboolf16 ge(const vfloat16& a, const vfloat16& b) { return _mm512_cmp_ps_mask(a,b,_MM_CMPINT_GE); } - __forceinline vboolf16 gt(const vfloat16& a, const vfloat16& b) { return _mm512_cmp_ps_mask(a,b,_MM_CMPINT_GT); } - __forceinline vboolf16 le(const vfloat16& a, const vfloat16& b) { return _mm512_cmp_ps_mask(a,b,_MM_CMPINT_LE); } - - __forceinline vboolf16 eq(const vboolf16& mask, const vfloat16& a, const vfloat16& b) { return _mm512_mask_cmp_ps_mask(mask,a,b,_MM_CMPINT_EQ); } - __forceinline vboolf16 ne(const vboolf16& mask, const vfloat16& a, const vfloat16& b) { return _mm512_mask_cmp_ps_mask(mask,a,b,_MM_CMPINT_NE); } - __forceinline vboolf16 lt(const vboolf16& mask, const vfloat16& a, const vfloat16& b) { return _mm512_mask_cmp_ps_mask(mask,a,b,_MM_CMPINT_LT); } - __forceinline vboolf16 ge(const vboolf16& mask, const vfloat16& a, const vfloat16& b) { return _mm512_mask_cmp_ps_mask(mask,a,b,_MM_CMPINT_GE); } - __forceinline vboolf16 gt(const vboolf16& mask, const vfloat16& a, const vfloat16& b) { return _mm512_mask_cmp_ps_mask(mask,a,b,_MM_CMPINT_GT); } - __forceinline vboolf16 le(const vboolf16& mask, const vfloat16& a, const vfloat16& b) { return _mm512_mask_cmp_ps_mask(mask,a,b,_MM_CMPINT_LE); } - - __forceinline vfloat16 select(const vboolf16& s, const vfloat16& t, const vfloat16& f) { - return _mm512_mask_blend_ps(s, f, t); - } - - __forceinline vfloat16 lerp(const vfloat16& a, const vfloat16& b, const vfloat16& t) { - return madd(t,b-a,a); - } - - __forceinline void xchg(vboolf16 m, vfloat16& a, vfloat16& b) - { - vfloat16 c = a; - a = select(m,b,a); - b = select(m,c,b); - } - - //////////////////////////////////////////////////////////////////////////////// - /// Rounding Functions - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vfloat16 floor(const vfloat16& a) { - return _mm512_floor_ps(a); - } - __forceinline vfloat16 ceil (const vfloat16& a) { - return _mm512_ceil_ps(a); - } - __forceinline vfloat16 round (const vfloat16& a) { - return _mm512_roundscale_ps(a, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); - } - __forceinline vint16 floori (const vfloat16& a) { - return _mm512_cvt_roundps_epi32(a, _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC); - } - - //////////////////////////////////////////////////////////////////////////////// - /// Movement/Shifting/Shuffling Functions - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vfloat16 unpacklo(const vfloat16& a, const vfloat16& b) { return _mm512_unpacklo_ps(a, b); } - __forceinline vfloat16 unpackhi(const vfloat16& a, const vfloat16& b) { return _mm512_unpackhi_ps(a, b); } - - template<int i> - __forceinline vfloat16 shuffle(const vfloat16& v) { - return _mm512_permute_ps(v, _MM_SHUFFLE(i, i, i, i)); - } - - template<int i0, int i1, int i2, int i3> - __forceinline vfloat16 shuffle(const vfloat16& v) { - return _mm512_permute_ps(v, _MM_SHUFFLE(i3, i2, i1, i0)); - } - - template<int i> - __forceinline vfloat16 shuffle4(const vfloat16& v) { - return _mm512_shuffle_f32x4(v, v ,_MM_SHUFFLE(i, i, i, i)); - } - - template<int i0, int i1, int i2, int i3> - __forceinline vfloat16 shuffle4(const vfloat16& v) { - return _mm512_shuffle_f32x4(v, v, _MM_SHUFFLE(i3, i2, i1, i0)); - } - - __forceinline vfloat16 interleave_even(const vfloat16& a, const vfloat16& b) { - return _mm512_castsi512_ps(_mm512_mask_shuffle_epi32(_mm512_castps_si512(a), mm512_int2mask(0xaaaa), _mm512_castps_si512(b), (_MM_PERM_ENUM)0xb1)); - } - - __forceinline vfloat16 interleave_odd(const vfloat16& a, const vfloat16& b) { - return _mm512_castsi512_ps(_mm512_mask_shuffle_epi32(_mm512_castps_si512(b), mm512_int2mask(0x5555), _mm512_castps_si512(a), (_MM_PERM_ENUM)0xb1)); - } - - __forceinline vfloat16 interleave2_even(const vfloat16& a, const vfloat16& b) { - /* mask should be 8-bit but is 16-bit to reuse for interleave_even */ - return _mm512_castsi512_ps(_mm512_mask_permutex_epi64(_mm512_castps_si512(a), mm512_int2mask(0xaaaa), _mm512_castps_si512(b), (_MM_PERM_ENUM)0xb1)); - } - - __forceinline vfloat16 interleave2_odd(const vfloat16& a, const vfloat16& b) { - /* mask should be 8-bit but is 16-bit to reuse for interleave_odd */ - return _mm512_castsi512_ps(_mm512_mask_permutex_epi64(_mm512_castps_si512(b), mm512_int2mask(0x5555), _mm512_castps_si512(a), (_MM_PERM_ENUM)0xb1)); - } - - __forceinline vfloat16 interleave4_even(const vfloat16& a, const vfloat16& b) { - return _mm512_castsi512_ps(_mm512_mask_permutex_epi64(_mm512_castps_si512(a), mm512_int2mask(0xcc), _mm512_castps_si512(b), (_MM_PERM_ENUM)0x4e)); - } - - __forceinline vfloat16 interleave4_odd(const vfloat16& a, const vfloat16& b) { - return _mm512_castsi512_ps(_mm512_mask_permutex_epi64(_mm512_castps_si512(b), mm512_int2mask(0x33), _mm512_castps_si512(a), (_MM_PERM_ENUM)0x4e)); - } - - __forceinline vfloat16 permute(vfloat16 v, __m512i index) { - return _mm512_castsi512_ps(_mm512_permutexvar_epi32(index, _mm512_castps_si512(v))); - } - - __forceinline vfloat16 reverse(const vfloat16& v) { - return permute(v,_mm512_setr_epi32(15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0)); - } - - template<int i> - __forceinline vfloat16 align_shift_right(const vfloat16& a, const vfloat16& b) { - return _mm512_castsi512_ps(_mm512_alignr_epi32(_mm512_castps_si512(a),_mm512_castps_si512(b),i)); - }; - - template<int i> - __forceinline vfloat16 mask_align_shift_right(const vboolf16& mask, vfloat16& c, const vfloat16& a, const vfloat16& b) { - return _mm512_castsi512_ps(_mm512_mask_alignr_epi32(_mm512_castps_si512(c),mask,_mm512_castps_si512(a),_mm512_castps_si512(b),i)); - }; - - __forceinline vfloat16 shift_left_1(const vfloat16& a) { - vfloat16 z = zero; - return mask_align_shift_right<15>(0xfffe,z,a,a); - } - - __forceinline vfloat16 shift_right_1(const vfloat16& x) { - return align_shift_right<1>(zero,x); - } - - __forceinline float toScalar(const vfloat16& v) { return mm512_cvtss_f32(v); } - - - template<int i> __forceinline vfloat16 insert4(const vfloat16& a, const vfloat4& b) { return _mm512_insertf32x4(a, b, i); } - - template<int N, int i> - vfloat<N> extractN(const vfloat16& v); - - template<> __forceinline vfloat4 extractN<4,0>(const vfloat16& v) { return _mm512_castps512_ps128(v); } - template<> __forceinline vfloat4 extractN<4,1>(const vfloat16& v) { return _mm512_extractf32x4_ps(v, 1); } - template<> __forceinline vfloat4 extractN<4,2>(const vfloat16& v) { return _mm512_extractf32x4_ps(v, 2); } - template<> __forceinline vfloat4 extractN<4,3>(const vfloat16& v) { return _mm512_extractf32x4_ps(v, 3); } - - template<> __forceinline vfloat8 extractN<8,0>(const vfloat16& v) { return _mm512_castps512_ps256(v); } - template<> __forceinline vfloat8 extractN<8,1>(const vfloat16& v) { return _mm512_extractf32x8_ps(v, 1); } - - template<int i> __forceinline vfloat4 extract4 (const vfloat16& v) { return _mm512_extractf32x4_ps(v, i); } - template<> __forceinline vfloat4 extract4<0>(const vfloat16& v) { return _mm512_castps512_ps128(v); } - - template<int i> __forceinline vfloat8 extract8 (const vfloat16& v) { return _mm512_extractf32x8_ps(v, i); } - template<> __forceinline vfloat8 extract8<0>(const vfloat16& v) { return _mm512_castps512_ps256(v); } - - //////////////////////////////////////////////////////////////////////////////// - /// Transpose - //////////////////////////////////////////////////////////////////////////////// - - __forceinline void transpose(const vfloat16& r0, const vfloat16& r1, const vfloat16& r2, const vfloat16& r3, - vfloat16& c0, vfloat16& c1, vfloat16& c2, vfloat16& c3) - { -#if defined(__AVX512F__) && !defined(__AVX512VL__) // KNL - vfloat16 a0a1_c0c1 = interleave_even(r0, r1); - vfloat16 a2a3_c2c3 = interleave_even(r2, r3); - vfloat16 b0b1_d0d1 = interleave_odd (r0, r1); - vfloat16 b2b3_d2d3 = interleave_odd (r2, r3); - - c0 = interleave2_even(a0a1_c0c1, a2a3_c2c3); - c1 = interleave2_even(b0b1_d0d1, b2b3_d2d3); - c2 = interleave2_odd (a0a1_c0c1, a2a3_c2c3); - c3 = interleave2_odd (b0b1_d0d1, b2b3_d2d3); -#else - vfloat16 a0a2_b0b2 = unpacklo(r0, r2); - vfloat16 c0c2_d0d2 = unpackhi(r0, r2); - vfloat16 a1a3_b1b3 = unpacklo(r1, r3); - vfloat16 c1c3_d1d3 = unpackhi(r1, r3); - - c0 = unpacklo(a0a2_b0b2, a1a3_b1b3); - c1 = unpackhi(a0a2_b0b2, a1a3_b1b3); - c2 = unpacklo(c0c2_d0d2, c1c3_d1d3); - c3 = unpackhi(c0c2_d0d2, c1c3_d1d3); -#endif - } - - __forceinline void transpose(const vfloat4& r0, const vfloat4& r1, const vfloat4& r2, const vfloat4& r3, - const vfloat4& r4, const vfloat4& r5, const vfloat4& r6, const vfloat4& r7, - const vfloat4& r8, const vfloat4& r9, const vfloat4& r10, const vfloat4& r11, - const vfloat4& r12, const vfloat4& r13, const vfloat4& r14, const vfloat4& r15, - vfloat16& c0, vfloat16& c1, vfloat16& c2, vfloat16& c3) - { - return transpose(vfloat16(r0, r4, r8, r12), vfloat16(r1, r5, r9, r13), vfloat16(r2, r6, r10, r14), vfloat16(r3, r7, r11, r15), - c0, c1, c2, c3); - } - - __forceinline void transpose(const vfloat16& r0, const vfloat16& r1, const vfloat16& r2, const vfloat16& r3, - const vfloat16& r4, const vfloat16& r5, const vfloat16& r6, const vfloat16& r7, - vfloat16& c0, vfloat16& c1, vfloat16& c2, vfloat16& c3, - vfloat16& c4, vfloat16& c5, vfloat16& c6, vfloat16& c7) - { - vfloat16 a0a1a2a3_e0e1e2e3, b0b1b2b3_f0f1f2f3, c0c1c2c3_g0g1g2g3, d0d1d2d3_h0h1h2h3; - transpose(r0, r1, r2, r3, a0a1a2a3_e0e1e2e3, b0b1b2b3_f0f1f2f3, c0c1c2c3_g0g1g2g3, d0d1d2d3_h0h1h2h3); - - vfloat16 a4a5a6a7_e4e5e6e7, b4b5b6b7_f4f5f6f7, c4c5c6c7_g4g5g6g7, d4d5d6d7_h4h5h6h7; - transpose(r4, r5, r6, r7, a4a5a6a7_e4e5e6e7, b4b5b6b7_f4f5f6f7, c4c5c6c7_g4g5g6g7, d4d5d6d7_h4h5h6h7); - - c0 = interleave4_even(a0a1a2a3_e0e1e2e3, a4a5a6a7_e4e5e6e7); - c1 = interleave4_even(b0b1b2b3_f0f1f2f3, b4b5b6b7_f4f5f6f7); - c2 = interleave4_even(c0c1c2c3_g0g1g2g3, c4c5c6c7_g4g5g6g7); - c3 = interleave4_even(d0d1d2d3_h0h1h2h3, d4d5d6d7_h4h5h6h7); - c4 = interleave4_odd (a0a1a2a3_e0e1e2e3, a4a5a6a7_e4e5e6e7); - c5 = interleave4_odd (b0b1b2b3_f0f1f2f3, b4b5b6b7_f4f5f6f7); - c6 = interleave4_odd (c0c1c2c3_g0g1g2g3, c4c5c6c7_g4g5g6g7); - c7 = interleave4_odd (d0d1d2d3_h0h1h2h3, d4d5d6d7_h4h5h6h7); - } - - __forceinline void transpose(const vfloat8& r0, const vfloat8& r1, const vfloat8& r2, const vfloat8& r3, - const vfloat8& r4, const vfloat8& r5, const vfloat8& r6, const vfloat8& r7, - const vfloat8& r8, const vfloat8& r9, const vfloat8& r10, const vfloat8& r11, - const vfloat8& r12, const vfloat8& r13, const vfloat8& r14, const vfloat8& r15, - vfloat16& c0, vfloat16& c1, vfloat16& c2, vfloat16& c3, - vfloat16& c4, vfloat16& c5, vfloat16& c6, vfloat16& c7) - { - return transpose(vfloat16(r0, r8), vfloat16(r1, r9), vfloat16(r2, r10), vfloat16(r3, r11), - vfloat16(r4, r12), vfloat16(r5, r13), vfloat16(r6, r14), vfloat16(r7, r15), - c0, c1, c2, c3, c4, c5, c6, c7); - } - - //////////////////////////////////////////////////////////////////////////////// - /// Reductions - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vfloat16 vreduce_add2(vfloat16 x) { return x + shuffle<1,0,3,2>(x); } - __forceinline vfloat16 vreduce_add4(vfloat16 x) { x = vreduce_add2(x); return x + shuffle<2,3,0,1>(x); } - __forceinline vfloat16 vreduce_add8(vfloat16 x) { x = vreduce_add4(x); return x + shuffle4<1,0,3,2>(x); } - __forceinline vfloat16 vreduce_add (vfloat16 x) { x = vreduce_add8(x); return x + shuffle4<2,3,0,1>(x); } - - __forceinline vfloat16 vreduce_min2(vfloat16 x) { return min(x, shuffle<1,0,3,2>(x)); } - __forceinline vfloat16 vreduce_min4(vfloat16 x) { x = vreduce_min2(x); return min(x, shuffle<2,3,0,1>(x)); } - __forceinline vfloat16 vreduce_min8(vfloat16 x) { x = vreduce_min4(x); return min(x, shuffle4<1,0,3,2>(x)); } - __forceinline vfloat16 vreduce_min (vfloat16 x) { x = vreduce_min8(x); return min(x, shuffle4<2,3,0,1>(x)); } - - __forceinline vfloat16 vreduce_max2(vfloat16 x) { return max(x, shuffle<1,0,3,2>(x)); } - __forceinline vfloat16 vreduce_max4(vfloat16 x) { x = vreduce_max2(x); return max(x, shuffle<2,3,0,1>(x)); } - __forceinline vfloat16 vreduce_max8(vfloat16 x) { x = vreduce_max4(x); return max(x, shuffle4<1,0,3,2>(x)); } - __forceinline vfloat16 vreduce_max (vfloat16 x) { x = vreduce_max8(x); return max(x, shuffle4<2,3,0,1>(x)); } - - __forceinline float reduce_add(const vfloat16& v) { return toScalar(vreduce_add(v)); } - __forceinline float reduce_min(const vfloat16& v) { return toScalar(vreduce_min(v)); } - __forceinline float reduce_max(const vfloat16& v) { return toScalar(vreduce_max(v)); } - - __forceinline size_t select_min(const vfloat16& v) { - return bsf(_mm512_kmov(_mm512_cmp_epi32_mask(_mm512_castps_si512(v),_mm512_castps_si512(vreduce_min(v)),_MM_CMPINT_EQ))); - } - - __forceinline size_t select_max(const vfloat16& v) { - return bsf(_mm512_kmov(_mm512_cmp_epi32_mask(_mm512_castps_si512(v),_mm512_castps_si512(vreduce_max(v)),_MM_CMPINT_EQ))); - } - - __forceinline size_t select_min(const vboolf16& valid, const vfloat16& v) - { - const vfloat16 a = select(valid,v,vfloat16(pos_inf)); - const vbool16 valid_min = valid & (a == vreduce_min(a)); - return bsf(movemask(any(valid_min) ? valid_min : valid)); - } - - __forceinline size_t select_max(const vboolf16& valid, const vfloat16& v) - { - const vfloat16 a = select(valid,v,vfloat16(neg_inf)); - const vbool16 valid_max = valid & (a == vreduce_max(a)); - return bsf(movemask(any(valid_max) ? valid_max : valid)); - } - - __forceinline vfloat16 prefix_sum(const vfloat16& a) - { - const vfloat16 z(zero); - vfloat16 v = a; - v = v + align_shift_right<16-1>(v,z); - v = v + align_shift_right<16-2>(v,z); - v = v + align_shift_right<16-4>(v,z); - v = v + align_shift_right<16-8>(v,z); - return v; - } - - __forceinline vfloat16 reverse_prefix_sum(const vfloat16& a) - { - const vfloat16 z(zero); - vfloat16 v = a; - v = v + align_shift_right<1>(z,v); - v = v + align_shift_right<2>(z,v); - v = v + align_shift_right<4>(z,v); - v = v + align_shift_right<8>(z,v); - return v; - } - - __forceinline vfloat16 prefix_min(const vfloat16& a) - { - const vfloat16 z(pos_inf); - vfloat16 v = a; - v = min(v,align_shift_right<16-1>(v,z)); - v = min(v,align_shift_right<16-2>(v,z)); - v = min(v,align_shift_right<16-4>(v,z)); - v = min(v,align_shift_right<16-8>(v,z)); - return v; - } - - __forceinline vfloat16 prefix_max(const vfloat16& a) - { - const vfloat16 z(neg_inf); - vfloat16 v = a; - v = max(v,align_shift_right<16-1>(v,z)); - v = max(v,align_shift_right<16-2>(v,z)); - v = max(v,align_shift_right<16-4>(v,z)); - v = max(v,align_shift_right<16-8>(v,z)); - return v; - } - - - __forceinline vfloat16 reverse_prefix_min(const vfloat16& a) - { - const vfloat16 z(pos_inf); - vfloat16 v = a; - v = min(v,align_shift_right<1>(z,v)); - v = min(v,align_shift_right<2>(z,v)); - v = min(v,align_shift_right<4>(z,v)); - v = min(v,align_shift_right<8>(z,v)); - return v; - } - - __forceinline vfloat16 reverse_prefix_max(const vfloat16& a) - { - const vfloat16 z(neg_inf); - vfloat16 v = a; - v = max(v,align_shift_right<1>(z,v)); - v = max(v,align_shift_right<2>(z,v)); - v = max(v,align_shift_right<4>(z,v)); - v = max(v,align_shift_right<8>(z,v)); - return v; - } - - //////////////////////////////////////////////////////////////////////////////// - /// Memory load and store operations - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vfloat16 loadAOS4to16f(const float& x, const float& y, const float& z) - { - vfloat16 f = zero; - f = select(0x1111,vfloat16::broadcast(&x),f); - f = select(0x2222,vfloat16::broadcast(&y),f); - f = select(0x4444,vfloat16::broadcast(&z),f); - return f; - } - - __forceinline vfloat16 loadAOS4to16f(unsigned int index, - const vfloat16& x, - const vfloat16& y, - const vfloat16& z) - { - vfloat16 f = zero; - f = select(0x1111,vfloat16::broadcast((float*)&x + index),f); - f = select(0x2222,vfloat16::broadcast((float*)&y + index),f); - f = select(0x4444,vfloat16::broadcast((float*)&z + index),f); - return f; - } - - __forceinline vfloat16 loadAOS4to16f(unsigned int index, - const vfloat16& x, - const vfloat16& y, - const vfloat16& z, - const vfloat16& fill) - { - vfloat16 f = fill; - f = select(0x1111,vfloat16::broadcast((float*)&x + index),f); - f = select(0x2222,vfloat16::broadcast((float*)&y + index),f); - f = select(0x4444,vfloat16::broadcast((float*)&z + index),f); - return f; - } - - __forceinline vfloat16 rcp_safe(const vfloat16& a) { - return rcp(select(a != vfloat16(zero), a, vfloat16(min_rcp_input))); - } - - //////////////////////////////////////////////////////////////////////////////// - /// Output Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline embree_ostream operator <<(embree_ostream cout, const vfloat16& v) - { - cout << "<" << v[0]; - for (int i=1; i<16; i++) cout << ", " << v[i]; - cout << ">"; - return cout; - } -} diff --git a/thirdparty/embree-aarch64/common/simd/vfloat4_sse2.h b/thirdparty/embree-aarch64/common/simd/vfloat4_sse2.h deleted file mode 100644 index 5732c0fbc8..0000000000 --- a/thirdparty/embree-aarch64/common/simd/vfloat4_sse2.h +++ /dev/null @@ -1,925 +0,0 @@ -// Copyright 2009-2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 - -#pragma once - -namespace embree -{ - /* 4-wide SSE float type */ - template<> - struct vfloat<4> - { - ALIGNED_STRUCT_(16); - - typedef vboolf4 Bool; - typedef vint4 Int; - typedef vfloat4 Float; - - enum { size = 4 }; // number of SIMD elements - union { __m128 v; float f[4]; int i[4]; }; // data - - //////////////////////////////////////////////////////////////////////////////// - /// Constructors, Assignment & Cast Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vfloat() {} - __forceinline vfloat(const vfloat4& other) { v = other.v; } - __forceinline vfloat4& operator =(const vfloat4& other) { v = other.v; return *this; } - - __forceinline vfloat(__m128 a) : v(a) {} - __forceinline operator const __m128&() const { return v; } - __forceinline operator __m128&() { return v; } - - __forceinline vfloat(float a) : v(_mm_set1_ps(a)) {} - __forceinline vfloat(float a, float b, float c, float d) : v(_mm_set_ps(d, c, b, a)) {} - - __forceinline explicit vfloat(const vint4& a) : v(_mm_cvtepi32_ps(a)) {} -#if defined(__aarch64__) - __forceinline explicit vfloat(const vuint4& x) { - v = vcvtq_f32_u32(vreinterpretq_u32_s32(x.v)); - } -#else - __forceinline explicit vfloat(const vuint4& x) { - const __m128i a = _mm_and_si128(x,_mm_set1_epi32(0x7FFFFFFF)); - const __m128i b = _mm_and_si128(_mm_srai_epi32(x,31),_mm_set1_epi32(0x4F000000)); //0x4F000000 = 2^31 - const __m128 af = _mm_cvtepi32_ps(a); - const __m128 bf = _mm_castsi128_ps(b); - v = _mm_add_ps(af,bf); - } -#endif - //////////////////////////////////////////////////////////////////////////////// - /// Constants - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vfloat(ZeroTy) : v(_mm_setzero_ps()) {} - __forceinline vfloat(OneTy) : v(_mm_set1_ps(1.0f)) {} - __forceinline vfloat(PosInfTy) : v(_mm_set1_ps(pos_inf)) {} - __forceinline vfloat(NegInfTy) : v(_mm_set1_ps(neg_inf)) {} - __forceinline vfloat(StepTy) : v(_mm_set_ps(3.0f, 2.0f, 1.0f, 0.0f)) {} - __forceinline vfloat(NaNTy) : v(_mm_set1_ps(nan)) {} - __forceinline vfloat(UndefinedTy) : v(_mm_undefined_ps()) {} - - //////////////////////////////////////////////////////////////////////////////// - /// Loads and Stores - //////////////////////////////////////////////////////////////////////////////// - - static __forceinline vfloat4 load (const void* a) { return _mm_load_ps((float*)a); } - static __forceinline vfloat4 loadu(const void* a) { return _mm_loadu_ps((float*)a); } - - static __forceinline void store (void* ptr, const vfloat4& v) { _mm_store_ps((float*)ptr,v); } - static __forceinline void storeu(void* ptr, const vfloat4& v) { _mm_storeu_ps((float*)ptr,v); } - -#if defined(__AVX512VL__) - - static __forceinline vfloat4 compact(const vboolf4& mask, vfloat4 &v) { - return _mm_mask_compress_ps(v, mask, v); - } - static __forceinline vfloat4 compact(const vboolf4& mask, vfloat4 &a, const vfloat4& b) { - return _mm_mask_compress_ps(a, mask, b); - } - - static __forceinline vfloat4 load (const vboolf4& mask, const void* ptr) { return _mm_mask_load_ps (_mm_setzero_ps(),mask,(float*)ptr); } - static __forceinline vfloat4 loadu(const vboolf4& mask, const void* ptr) { return _mm_mask_loadu_ps(_mm_setzero_ps(),mask,(float*)ptr); } - - static __forceinline void store (const vboolf4& mask, void* ptr, const vfloat4& v) { _mm_mask_store_ps ((float*)ptr,mask,v); } - static __forceinline void storeu(const vboolf4& mask, void* ptr, const vfloat4& v) { _mm_mask_storeu_ps((float*)ptr,mask,v); } -#elif defined(__AVX__) - static __forceinline vfloat4 load (const vboolf4& mask, const void* ptr) { return _mm_maskload_ps((float*)ptr,mask); } - static __forceinline vfloat4 loadu(const vboolf4& mask, const void* ptr) { return _mm_maskload_ps((float*)ptr,mask); } - - static __forceinline void store (const vboolf4& mask, void* ptr, const vfloat4& v) { _mm_maskstore_ps((float*)ptr,(__m128i)mask,v); } - static __forceinline void storeu(const vboolf4& mask, void* ptr, const vfloat4& v) { _mm_maskstore_ps((float*)ptr,(__m128i)mask,v); } -#else - static __forceinline vfloat4 load (const vboolf4& mask, const void* ptr) { return _mm_and_ps(_mm_load_ps ((float*)ptr),mask); } - static __forceinline vfloat4 loadu(const vboolf4& mask, const void* ptr) { return _mm_and_ps(_mm_loadu_ps((float*)ptr),mask); } - - static __forceinline void store (const vboolf4& mask, void* ptr, const vfloat4& v) { store (ptr,select(mask,v,load (ptr))); } - static __forceinline void storeu(const vboolf4& mask, void* ptr, const vfloat4& v) { storeu(ptr,select(mask,v,loadu(ptr))); } -#endif - -#if defined(__AVX__) - static __forceinline vfloat4 broadcast(const void* a) { return _mm_broadcast_ss((float*)a); } -#else - static __forceinline vfloat4 broadcast(const void* a) { return _mm_set1_ps(*(float*)a); } -#endif - - static __forceinline vfloat4 load_nt (const float* ptr) { -#if defined (__SSE4_1__) - return _mm_castsi128_ps(_mm_stream_load_si128((__m128i*)ptr)); -#else - return _mm_load_ps(ptr); -#endif - } - -#if defined(__aarch64__) - static __forceinline vfloat4 load(const int8_t* ptr) { - return __m128(_mm_load4epi8_f32(((__m128i*)ptr))); - } -#elif defined(__SSE4_1__) - static __forceinline vfloat4 load(const int8_t* ptr) { - return _mm_cvtepi32_ps(_mm_cvtepi8_epi32(_mm_loadu_si128((__m128i*)ptr))); - } -#else - static __forceinline vfloat4 load(const int8_t* ptr) { - return vfloat4(ptr[0],ptr[1],ptr[2],ptr[3]); - } -#endif - -#if defined(__aarch64__) - static __forceinline vfloat4 load(const uint8_t* ptr) { - return __m128(_mm_load4epu8_f32(((__m128i*)ptr))); - } -#elif defined(__SSE4_1__) - static __forceinline vfloat4 load(const uint8_t* ptr) { - return _mm_cvtepi32_ps(_mm_cvtepu8_epi32(_mm_loadu_si128((__m128i*)ptr))); - } -#else - static __forceinline vfloat4 load(const uint8_t* ptr) { - //return _mm_cvtpu8_ps(*(__m64*)ptr); // don't enable, will use MMX instructions - return vfloat4(ptr[0],ptr[1],ptr[2],ptr[3]); - } -#endif - -#if defined(__aarch64__) - static __forceinline vfloat4 load(const short* ptr) { - return __m128(_mm_load4epi16_f32(((__m128i*)ptr))); - } -#elif defined(__SSE4_1__) - static __forceinline vfloat4 load(const short* ptr) { - return _mm_cvtepi32_ps(_mm_cvtepi16_epi32(_mm_loadu_si128((__m128i*)ptr))); - } -#else - static __forceinline vfloat4 load(const short* ptr) { - return vfloat4(ptr[0],ptr[1],ptr[2],ptr[3]); - } -#endif - - static __forceinline vfloat4 load(const unsigned short* ptr) { - return _mm_mul_ps(vfloat4(vint4::load(ptr)),vfloat4(1.0f/65535.0f)); - } - - static __forceinline void store_nt(void* ptr, const vfloat4& v) - { -#if defined (__SSE4_1__) -#if defined(__aarch64__) - _mm_stream_ps((float*)ptr,vreinterpretq_s32_f32(v.v)); -#else - _mm_stream_ps((float*)ptr,v); -#endif -#else - _mm_store_ps((float*)ptr,v); -#endif - } - - template<int scale = 4> - static __forceinline vfloat4 gather(const float* ptr, const vint4& index) { -#if defined(__AVX2__) && !defined(__aarch64__) - return _mm_i32gather_ps(ptr, index, scale); -#else - return vfloat4( - *(float*)(((int8_t*)ptr)+scale*index[0]), - *(float*)(((int8_t*)ptr)+scale*index[1]), - *(float*)(((int8_t*)ptr)+scale*index[2]), - *(float*)(((int8_t*)ptr)+scale*index[3])); -#endif - } - - template<int scale = 4> - static __forceinline vfloat4 gather(const vboolf4& mask, const float* ptr, const vint4& index) { - vfloat4 r = zero; -#if defined(__AVX512VL__) - return _mm_mmask_i32gather_ps(r, mask, index, ptr, scale); -#elif defined(__AVX2__) && !defined(__aarch64__) - return _mm_mask_i32gather_ps(r, ptr, index, mask, scale); -#else - if (likely(mask[0])) r[0] = *(float*)(((int8_t*)ptr)+scale*index[0]); - if (likely(mask[1])) r[1] = *(float*)(((int8_t*)ptr)+scale*index[1]); - if (likely(mask[2])) r[2] = *(float*)(((int8_t*)ptr)+scale*index[2]); - if (likely(mask[3])) r[3] = *(float*)(((int8_t*)ptr)+scale*index[3]); - return r; -#endif - } - - template<int scale = 4> - static __forceinline void scatter(void* ptr, const vint4& index, const vfloat4& v) - { -#if defined(__AVX512VL__) - _mm_i32scatter_ps((float*)ptr, index, v, scale); -#else - *(float*)(((int8_t*)ptr)+scale*index[0]) = v[0]; - *(float*)(((int8_t*)ptr)+scale*index[1]) = v[1]; - *(float*)(((int8_t*)ptr)+scale*index[2]) = v[2]; - *(float*)(((int8_t*)ptr)+scale*index[3]) = v[3]; -#endif - } - - template<int scale = 4> - static __forceinline void scatter(const vboolf4& mask, void* ptr, const vint4& index, const vfloat4& v) - { -#if defined(__AVX512VL__) - _mm_mask_i32scatter_ps((float*)ptr ,mask, index, v, scale); -#else - if (likely(mask[0])) *(float*)(((int8_t*)ptr)+scale*index[0]) = v[0]; - if (likely(mask[1])) *(float*)(((int8_t*)ptr)+scale*index[1]) = v[1]; - if (likely(mask[2])) *(float*)(((int8_t*)ptr)+scale*index[2]) = v[2]; - if (likely(mask[3])) *(float*)(((int8_t*)ptr)+scale*index[3]) = v[3]; -#endif - } - - static __forceinline void store(const vboolf4& mask, int8_t* ptr, const vint4& ofs, const vfloat4& v) { - scatter<1>(mask,ptr,ofs,v); - } - static __forceinline void store(const vboolf4& mask, float* ptr, const vint4& ofs, const vfloat4& v) { - scatter<4>(mask,ptr,ofs,v); - } - - //////////////////////////////////////////////////////////////////////////////// - /// Array Access - //////////////////////////////////////////////////////////////////////////////// - - __forceinline const float& operator [](size_t index) const { assert(index < 4); return f[index]; } - __forceinline float& operator [](size_t index) { assert(index < 4); return f[index]; } - - friend __forceinline vfloat4 select(const vboolf4& m, const vfloat4& t, const vfloat4& f) { -#if defined(__AVX512VL__) - return _mm_mask_blend_ps(m, f, t); -#elif defined(__SSE4_1__) || (defined(__aarch64__)) - return _mm_blendv_ps(f, t, m); -#else - return _mm_or_ps(_mm_and_ps(m, t), _mm_andnot_ps(m, f)); -#endif - } - }; - - - //////////////////////////////////////////////////////////////////////////////// - /// Unary Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vfloat4 asFloat(const vint4& a) { return _mm_castsi128_ps(a); } - __forceinline vint4 asInt (const vfloat4& a) { return _mm_castps_si128(a); } - __forceinline vuint4 asUInt (const vfloat4& a) { return _mm_castps_si128(a); } - - __forceinline vint4 toInt (const vfloat4& a) { return vint4(a); } - __forceinline vfloat4 toFloat(const vint4& a) { return vfloat4(a); } - - __forceinline vfloat4 operator +(const vfloat4& a) { return a; } -#if defined(__aarch64__) - __forceinline vfloat4 operator -(const vfloat4& a) { - return vnegq_f32(a); - } -#else - __forceinline vfloat4 operator -(const vfloat4& a) { return _mm_xor_ps(a, _mm_castsi128_ps(_mm_set1_epi32(0x80000000))); } -#endif - -#if defined(__aarch64__) - __forceinline vfloat4 abs(const vfloat4& a) { return _mm_abs_ps(a); } -#else - __forceinline vfloat4 abs(const vfloat4& a) { return _mm_and_ps(a, _mm_castsi128_ps(_mm_set1_epi32(0x7fffffff))); } -#endif - -#if defined(__AVX512VL__) - __forceinline vfloat4 sign(const vfloat4& a) { return _mm_mask_blend_ps(_mm_cmp_ps_mask(a, vfloat4(zero), _CMP_LT_OQ), vfloat4(one), -vfloat4(one)); } -#else - __forceinline vfloat4 sign(const vfloat4& a) { return blendv_ps(vfloat4(one), -vfloat4(one), _mm_cmplt_ps(a, vfloat4(zero))); } -#endif - -#if defined(__aarch64__) - __forceinline vfloat4 signmsk(const vfloat4& a) { return _mm_and_ps(a, vreinterpretq_f32_u32(v0x80000000)); } -#else - __forceinline vfloat4 signmsk(const vfloat4& a) { return _mm_and_ps(a,_mm_castsi128_ps(_mm_set1_epi32(0x80000000))); } -#endif - - __forceinline vfloat4 rcp(const vfloat4& a) - { -#if defined(__aarch64__) -#if defined(BUILD_IOS) - return vfloat4(vdivq_f32(vdupq_n_f32(1.0f),a.v)); -#else //BUILD_IOS - __m128 reciprocal = _mm_rcp_ps(a); - reciprocal = vmulq_f32(vrecpsq_f32(a, reciprocal), reciprocal); - reciprocal = vmulq_f32(vrecpsq_f32(a, reciprocal), reciprocal); - // +1 round since NEON's reciprocal estimate instruction has less accuracy than SSE2's rcp. - reciprocal = vmulq_f32(vrecpsq_f32(a, reciprocal), reciprocal); - return (const vfloat4)reciprocal; -#endif // BUILD_IOS -#else - -#if defined(__AVX512VL__) - const vfloat4 r = _mm_rcp14_ps(a); -#else - const vfloat4 r = _mm_rcp_ps(a); -#endif - -#if defined(__AVX2__) - return _mm_mul_ps(r,_mm_fnmadd_ps(r, a, vfloat4(2.0f))); -#else - return _mm_mul_ps(r,_mm_sub_ps(vfloat4(2.0f), _mm_mul_ps(r, a))); -#endif - -#endif //defined(__aarch64__) - } - __forceinline vfloat4 sqr (const vfloat4& a) { return _mm_mul_ps(a,a); } - __forceinline vfloat4 sqrt(const vfloat4& a) { return _mm_sqrt_ps(a); } - - __forceinline vfloat4 rsqrt(const vfloat4& a) - { -#if defined(__aarch64__) - vfloat4 r = _mm_rsqrt_ps(a); - r = vmulq_f32(r, vrsqrtsq_f32(vmulq_f32(a, r), r)); - r = vmulq_f32(r, vrsqrtsq_f32(vmulq_f32(a, r), r)); - r = vmulq_f32(r, vrsqrtsq_f32(vmulq_f32(a, r), r)); - return r; -#else - -#if defined(__AVX512VL__) - const vfloat4 r = _mm_rsqrt14_ps(a); -#else - const vfloat4 r = _mm_rsqrt_ps(a); -#endif - -#if defined(__AVX2__) - return _mm_fmadd_ps(_mm_set1_ps(1.5f), r, - _mm_mul_ps(_mm_mul_ps(_mm_mul_ps(a, _mm_set1_ps(-0.5f)), r), _mm_mul_ps(r, r))); -#else - return _mm_add_ps(_mm_mul_ps(_mm_set1_ps(1.5f), r), - _mm_mul_ps(_mm_mul_ps(_mm_mul_ps(a, _mm_set1_ps(-0.5f)), r), _mm_mul_ps(r, r))); -#endif - -#endif - } - - __forceinline vboolf4 isnan(const vfloat4& a) { -#if defined(__aarch64__) - const vfloat4 b = _mm_and_ps(a, vreinterpretq_f32_u32(v0x7fffffff)); -#else - const vfloat4 b = _mm_and_ps(a, _mm_castsi128_ps(_mm_set1_epi32(0x7fffffff))); -#endif -#if defined(__AVX512VL__) - return _mm_cmp_epi32_mask(_mm_castps_si128(b), _mm_set1_epi32(0x7f800000), _MM_CMPINT_GT); -#else - return _mm_castsi128_ps(_mm_cmpgt_epi32(_mm_castps_si128(b), _mm_set1_epi32(0x7f800000))); -#endif - } - - //////////////////////////////////////////////////////////////////////////////// - /// Binary Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vfloat4 operator +(const vfloat4& a, const vfloat4& b) { return _mm_add_ps(a, b); } - __forceinline vfloat4 operator +(const vfloat4& a, float b) { return a + vfloat4(b); } - __forceinline vfloat4 operator +(float a, const vfloat4& b) { return vfloat4(a) + b; } - - __forceinline vfloat4 operator -(const vfloat4& a, const vfloat4& b) { return _mm_sub_ps(a, b); } - __forceinline vfloat4 operator -(const vfloat4& a, float b) { return a - vfloat4(b); } - __forceinline vfloat4 operator -(float a, const vfloat4& b) { return vfloat4(a) - b; } - - __forceinline vfloat4 operator *(const vfloat4& a, const vfloat4& b) { return _mm_mul_ps(a, b); } - __forceinline vfloat4 operator *(const vfloat4& a, float b) { return a * vfloat4(b); } - __forceinline vfloat4 operator *(float a, const vfloat4& b) { return vfloat4(a) * b; } - - __forceinline vfloat4 operator /(const vfloat4& a, const vfloat4& b) { return _mm_div_ps(a,b); } - __forceinline vfloat4 operator /(const vfloat4& a, float b) { return a/vfloat4(b); } - __forceinline vfloat4 operator /(float a, const vfloat4& b) { return vfloat4(a)/b; } - - __forceinline vfloat4 operator &(const vfloat4& a, const vfloat4& b) { return _mm_and_ps(a,b); } - __forceinline vfloat4 operator |(const vfloat4& a, const vfloat4& b) { return _mm_or_ps(a,b); } - __forceinline vfloat4 operator ^(const vfloat4& a, const vfloat4& b) { return _mm_xor_ps(a,b); } - __forceinline vfloat4 operator ^(const vfloat4& a, const vint4& b) { return _mm_xor_ps(a,_mm_castsi128_ps(b)); } - - __forceinline vfloat4 min(const vfloat4& a, const vfloat4& b) { return _mm_min_ps(a,b); } - __forceinline vfloat4 min(const vfloat4& a, float b) { return _mm_min_ps(a,vfloat4(b)); } - __forceinline vfloat4 min(float a, const vfloat4& b) { return _mm_min_ps(vfloat4(a),b); } - - __forceinline vfloat4 max(const vfloat4& a, const vfloat4& b) { return _mm_max_ps(a,b); } - __forceinline vfloat4 max(const vfloat4& a, float b) { return _mm_max_ps(a,vfloat4(b)); } - __forceinline vfloat4 max(float a, const vfloat4& b) { return _mm_max_ps(vfloat4(a),b); } - -#if defined(__SSE4_1__) || defined(__aarch64__) - - __forceinline vfloat4 mini(const vfloat4& a, const vfloat4& b) { - const vint4 ai = _mm_castps_si128(a); - const vint4 bi = _mm_castps_si128(b); - const vint4 ci = _mm_min_epi32(ai,bi); - return _mm_castsi128_ps(ci); - } - - __forceinline vfloat4 maxi(const vfloat4& a, const vfloat4& b) { - const vint4 ai = _mm_castps_si128(a); - const vint4 bi = _mm_castps_si128(b); - const vint4 ci = _mm_max_epi32(ai,bi); - return _mm_castsi128_ps(ci); - } - - __forceinline vfloat4 minui(const vfloat4& a, const vfloat4& b) { - const vint4 ai = _mm_castps_si128(a); - const vint4 bi = _mm_castps_si128(b); - const vint4 ci = _mm_min_epu32(ai,bi); - return _mm_castsi128_ps(ci); - } - - __forceinline vfloat4 maxui(const vfloat4& a, const vfloat4& b) { - const vint4 ai = _mm_castps_si128(a); - const vint4 bi = _mm_castps_si128(b); - const vint4 ci = _mm_max_epu32(ai,bi); - return _mm_castsi128_ps(ci); - } -#else - __forceinline vfloat4 mini(const vfloat4& a, const vfloat4& b) { - return min(a,b); - } - - __forceinline vfloat4 maxi(const vfloat4& a, const vfloat4& b) { - return max(a,b); - } -#endif - - //////////////////////////////////////////////////////////////////////////////// - /// Ternary Operators - //////////////////////////////////////////////////////////////////////////////// - -#if defined(__AVX2__) - __forceinline vfloat4 madd (const vfloat4& a, const vfloat4& b, const vfloat4& c) { return _mm_fmadd_ps(a,b,c); } - __forceinline vfloat4 msub (const vfloat4& a, const vfloat4& b, const vfloat4& c) { return _mm_fmsub_ps(a,b,c); } - __forceinline vfloat4 nmadd(const vfloat4& a, const vfloat4& b, const vfloat4& c) { return _mm_fnmadd_ps(a,b,c); } - __forceinline vfloat4 nmsub(const vfloat4& a, const vfloat4& b, const vfloat4& c) { return _mm_fnmsub_ps(a,b,c); } -#else - -#if defined(__aarch64__) - __forceinline vfloat4 madd (const vfloat4& a, const vfloat4& b, const vfloat4& c) { - return _mm_madd_ps(a, b, c); //a*b+c; - } - __forceinline vfloat4 nmadd(const vfloat4& a, const vfloat4& b, const vfloat4& c) { - return _mm_msub_ps(a, b, c); //-a*b+c; - } - __forceinline vfloat4 nmsub(const vfloat4& a, const vfloat4& b, const vfloat4& c) { - return vnegq_f32(vfmaq_f32(c,a, b)); - } -#else - __forceinline vfloat4 madd (const vfloat4& a, const vfloat4& b, const vfloat4& c) { return a*b+c; } - __forceinline vfloat4 nmadd(const vfloat4& a, const vfloat4& b, const vfloat4& c) { return -a*b+c;} - __forceinline vfloat4 nmsub(const vfloat4& a, const vfloat4& b, const vfloat4& c) { return -a*b-c; } -#endif - __forceinline vfloat4 msub (const vfloat4& a, const vfloat4& b, const vfloat4& c) { return a*b-c; } - -#endif - - //////////////////////////////////////////////////////////////////////////////// - /// Assignment Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vfloat4& operator +=(vfloat4& a, const vfloat4& b) { return a = a + b; } - __forceinline vfloat4& operator +=(vfloat4& a, float b) { return a = a + b; } - - __forceinline vfloat4& operator -=(vfloat4& a, const vfloat4& b) { return a = a - b; } - __forceinline vfloat4& operator -=(vfloat4& a, float b) { return a = a - b; } - - __forceinline vfloat4& operator *=(vfloat4& a, const vfloat4& b) { return a = a * b; } - __forceinline vfloat4& operator *=(vfloat4& a, float b) { return a = a * b; } - - __forceinline vfloat4& operator /=(vfloat4& a, const vfloat4& b) { return a = a / b; } - __forceinline vfloat4& operator /=(vfloat4& a, float b) { return a = a / b; } - - //////////////////////////////////////////////////////////////////////////////// - /// Comparison Operators + Select - //////////////////////////////////////////////////////////////////////////////// - -#if defined(__AVX512VL__) - __forceinline vboolf4 operator ==(const vfloat4& a, const vfloat4& b) { return _mm_cmp_ps_mask(a, b, _MM_CMPINT_EQ); } - __forceinline vboolf4 operator !=(const vfloat4& a, const vfloat4& b) { return _mm_cmp_ps_mask(a, b, _MM_CMPINT_NE); } - __forceinline vboolf4 operator < (const vfloat4& a, const vfloat4& b) { return _mm_cmp_ps_mask(a, b, _MM_CMPINT_LT); } - __forceinline vboolf4 operator >=(const vfloat4& a, const vfloat4& b) { return _mm_cmp_ps_mask(a, b, _MM_CMPINT_GE); } - __forceinline vboolf4 operator > (const vfloat4& a, const vfloat4& b) { return _mm_cmp_ps_mask(a, b, _MM_CMPINT_GT); } - __forceinline vboolf4 operator <=(const vfloat4& a, const vfloat4& b) { return _mm_cmp_ps_mask(a, b, _MM_CMPINT_LE); } -#else - __forceinline vboolf4 operator ==(const vfloat4& a, const vfloat4& b) { return _mm_cmpeq_ps (a, b); } - __forceinline vboolf4 operator !=(const vfloat4& a, const vfloat4& b) { return _mm_cmpneq_ps(a, b); } - __forceinline vboolf4 operator < (const vfloat4& a, const vfloat4& b) { return _mm_cmplt_ps (a, b); } -#if defined(__aarch64__) - __forceinline vboolf4 operator >=(const vfloat4& a, const vfloat4& b) { return _mm_cmpge_ps (a, b); } - __forceinline vboolf4 operator > (const vfloat4& a, const vfloat4& b) { return _mm_cmpgt_ps (a, b); } -#else - __forceinline vboolf4 operator >=(const vfloat4& a, const vfloat4& b) { return _mm_cmpnlt_ps(a, b); } - __forceinline vboolf4 operator > (const vfloat4& a, const vfloat4& b) { return _mm_cmpnle_ps(a, b); } -#endif - __forceinline vboolf4 operator <=(const vfloat4& a, const vfloat4& b) { return _mm_cmple_ps (a, b); } -#endif - - __forceinline vboolf4 operator ==(const vfloat4& a, float b) { return a == vfloat4(b); } - __forceinline vboolf4 operator ==(float a, const vfloat4& b) { return vfloat4(a) == b; } - - __forceinline vboolf4 operator !=(const vfloat4& a, float b) { return a != vfloat4(b); } - __forceinline vboolf4 operator !=(float a, const vfloat4& b) { return vfloat4(a) != b; } - - __forceinline vboolf4 operator < (const vfloat4& a, float b) { return a < vfloat4(b); } - __forceinline vboolf4 operator < (float a, const vfloat4& b) { return vfloat4(a) < b; } - - __forceinline vboolf4 operator >=(const vfloat4& a, float b) { return a >= vfloat4(b); } - __forceinline vboolf4 operator >=(float a, const vfloat4& b) { return vfloat4(a) >= b; } - - __forceinline vboolf4 operator > (const vfloat4& a, float b) { return a > vfloat4(b); } - __forceinline vboolf4 operator > (float a, const vfloat4& b) { return vfloat4(a) > b; } - - __forceinline vboolf4 operator <=(const vfloat4& a, float b) { return a <= vfloat4(b); } - __forceinline vboolf4 operator <=(float a, const vfloat4& b) { return vfloat4(a) <= b; } - - __forceinline vboolf4 eq(const vfloat4& a, const vfloat4& b) { return a == b; } - __forceinline vboolf4 ne(const vfloat4& a, const vfloat4& b) { return a != b; } - __forceinline vboolf4 lt(const vfloat4& a, const vfloat4& b) { return a < b; } - __forceinline vboolf4 ge(const vfloat4& a, const vfloat4& b) { return a >= b; } - __forceinline vboolf4 gt(const vfloat4& a, const vfloat4& b) { return a > b; } - __forceinline vboolf4 le(const vfloat4& a, const vfloat4& b) { return a <= b; } - -#if defined(__AVX512VL__) - __forceinline vboolf4 eq(const vboolf4& mask, const vfloat4& a, const vfloat4& b) { return _mm_mask_cmp_ps_mask(mask, a, b, _MM_CMPINT_EQ); } - __forceinline vboolf4 ne(const vboolf4& mask, const vfloat4& a, const vfloat4& b) { return _mm_mask_cmp_ps_mask(mask, a, b, _MM_CMPINT_NE); } - __forceinline vboolf4 lt(const vboolf4& mask, const vfloat4& a, const vfloat4& b) { return _mm_mask_cmp_ps_mask(mask, a, b, _MM_CMPINT_LT); } - __forceinline vboolf4 ge(const vboolf4& mask, const vfloat4& a, const vfloat4& b) { return _mm_mask_cmp_ps_mask(mask, a, b, _MM_CMPINT_GE); } - __forceinline vboolf4 gt(const vboolf4& mask, const vfloat4& a, const vfloat4& b) { return _mm_mask_cmp_ps_mask(mask, a, b, _MM_CMPINT_GT); } - __forceinline vboolf4 le(const vboolf4& mask, const vfloat4& a, const vfloat4& b) { return _mm_mask_cmp_ps_mask(mask, a, b, _MM_CMPINT_LE); } -#else - __forceinline vboolf4 eq(const vboolf4& mask, const vfloat4& a, const vfloat4& b) { return mask & (a == b); } - __forceinline vboolf4 ne(const vboolf4& mask, const vfloat4& a, const vfloat4& b) { return mask & (a != b); } - __forceinline vboolf4 lt(const vboolf4& mask, const vfloat4& a, const vfloat4& b) { return mask & (a < b); } - __forceinline vboolf4 ge(const vboolf4& mask, const vfloat4& a, const vfloat4& b) { return mask & (a >= b); } - __forceinline vboolf4 gt(const vboolf4& mask, const vfloat4& a, const vfloat4& b) { return mask & (a > b); } - __forceinline vboolf4 le(const vboolf4& mask, const vfloat4& a, const vfloat4& b) { return mask & (a <= b); } -#endif - - template<int mask> - __forceinline vfloat4 select(const vfloat4& t, const vfloat4& f) - { -#if defined(__SSE4_1__) - return _mm_blend_ps(f, t, mask); -#else - return select(vboolf4(mask), t, f); -#endif - } - -#if defined(__aarch64__) - template<> __forceinline vfloat4 select<0>(const vfloat4& t, const vfloat4& f) { - return _mm_blendv_ps(f, t, vreinterpretq_f32_u32(vzero)); - } - template<> __forceinline vfloat4 select<1>(const vfloat4& t, const vfloat4& f) { - return _mm_blendv_ps(f, t, vreinterpretq_f32_u32(v000F)); - } - template<> __forceinline vfloat4 select<2>(const vfloat4& t, const vfloat4& f) { - return _mm_blendv_ps(f, t, vreinterpretq_f32_u32(v00F0)); - } - template<> __forceinline vfloat4 select<3>(const vfloat4& t, const vfloat4& f) { - return _mm_blendv_ps(f, t, vreinterpretq_f32_u32(v00FF)); - } - template<> __forceinline vfloat4 select<4>(const vfloat4& t, const vfloat4& f) { - return _mm_blendv_ps(f, t, vreinterpretq_f32_u32(v0F00)); - } - template<> __forceinline vfloat4 select<5>(const vfloat4& t, const vfloat4& f) { - return _mm_blendv_ps(f, t, vreinterpretq_f32_u32(v0F0F)); - } - template<> __forceinline vfloat4 select<6>(const vfloat4& t, const vfloat4& f) { - return _mm_blendv_ps(f, t, vreinterpretq_f32_u32(v0FF0)); - } - template<> __forceinline vfloat4 select<7>(const vfloat4& t, const vfloat4& f) { - return _mm_blendv_ps(f, t, vreinterpretq_f32_u32(v0FFF)); - } - template<> __forceinline vfloat4 select<8>(const vfloat4& t, const vfloat4& f) { - return _mm_blendv_ps(f, t, vreinterpretq_f32_u32(vF000)); - } - template<> __forceinline vfloat4 select<9>(const vfloat4& t, const vfloat4& f) { - return _mm_blendv_ps(f, t, vreinterpretq_f32_u32(vF00F)); - } - template<> __forceinline vfloat4 select<10>(const vfloat4& t, const vfloat4& f) { - return _mm_blendv_ps(f, t, vreinterpretq_f32_u32(vF0F0)); - } - template<> __forceinline vfloat4 select<11>(const vfloat4& t, const vfloat4& f) { - return _mm_blendv_ps(f, t, vreinterpretq_f32_u32(vF0FF)); - } - template<> __forceinline vfloat4 select<12>(const vfloat4& t, const vfloat4& f) { - return _mm_blendv_ps(f, t, vreinterpretq_f32_u32(vFF00)); - } - template<> __forceinline vfloat4 select<13>(const vfloat4& t, const vfloat4& f) { - return _mm_blendv_ps(f, t, vreinterpretq_f32_u32(vFF0F)); - } - template<> __forceinline vfloat4 select<14>(const vfloat4& t, const vfloat4& f) { - return _mm_blendv_ps(f, t, vreinterpretq_f32_u32(vFFF0)); - } - template<> __forceinline vfloat4 select<15>(const vfloat4& t, const vfloat4& f) { - return _mm_blendv_ps(f, t, vreinterpretq_f32_u32(vFFFF)); - } -#endif - - __forceinline vfloat4 lerp(const vfloat4& a, const vfloat4& b, const vfloat4& t) { - return madd(t,b-a,a); - } - - __forceinline bool isvalid(const vfloat4& v) { - return all((v > vfloat4(-FLT_LARGE)) & (v < vfloat4(+FLT_LARGE))); - } - - __forceinline bool is_finite(const vfloat4& a) { - return all((a >= vfloat4(-FLT_MAX)) & (a <= vfloat4(+FLT_MAX))); - } - - __forceinline bool is_finite(const vboolf4& valid, const vfloat4& a) { - return all(valid, (a >= vfloat4(-FLT_MAX)) & (a <= vfloat4(+FLT_MAX))); - } - - //////////////////////////////////////////////////////////////////////////////// - /// Rounding Functions - //////////////////////////////////////////////////////////////////////////////// - -#if defined(__aarch64__) - __forceinline vfloat4 floor(const vfloat4& a) { return vrndmq_f32(a.v); } // towards -inf - __forceinline vfloat4 ceil (const vfloat4& a) { return vrndpq_f32(a.v); } // toward +inf - __forceinline vfloat4 trunc(const vfloat4& a) { return vrndq_f32(a.v); } // towards 0 - __forceinline vfloat4 round(const vfloat4& a) { return vrndnq_f32(a.v); } // to nearest, ties to even. NOTE(LTE): arm clang uses vrndnq, old gcc uses vrndqn? -#elif defined (__SSE4_1__) - __forceinline vfloat4 floor(const vfloat4& a) { return _mm_round_ps(a, _MM_FROUND_TO_NEG_INF ); } - __forceinline vfloat4 ceil (const vfloat4& a) { return _mm_round_ps(a, _MM_FROUND_TO_POS_INF ); } - __forceinline vfloat4 trunc(const vfloat4& a) { return _mm_round_ps(a, _MM_FROUND_TO_ZERO ); } - __forceinline vfloat4 round(const vfloat4& a) { return _mm_round_ps(a, _MM_FROUND_TO_NEAREST_INT); } // (even) https://www.felixcloutier.com/x86/roundpd -#else - __forceinline vfloat4 floor(const vfloat4& a) { return vfloat4(floorf(a[0]),floorf(a[1]),floorf(a[2]),floorf(a[3])); } - __forceinline vfloat4 ceil (const vfloat4& a) { return vfloat4(ceilf (a[0]),ceilf (a[1]),ceilf (a[2]),ceilf (a[3])); } - __forceinline vfloat4 trunc(const vfloat4& a) { return vfloat4(truncf(a[0]),truncf(a[1]),truncf(a[2]),truncf(a[3])); } - __forceinline vfloat4 round(const vfloat4& a) { return vfloat4(roundf(a[0]),roundf(a[1]),roundf(a[2]),roundf(a[3])); } -#endif - __forceinline vfloat4 frac(const vfloat4& a) { return a-floor(a); } - - __forceinline vint4 floori(const vfloat4& a) { -#if defined(__aarch64__) - return vcvtq_s32_f32(floor(a)); -#elif defined(__SSE4_1__) - return vint4(floor(a)); -#else - return vint4(a-vfloat4(0.5f)); -#endif - } - - //////////////////////////////////////////////////////////////////////////////// - /// Movement/Shifting/Shuffling Functions - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vfloat4 unpacklo(const vfloat4& a, const vfloat4& b) { return _mm_unpacklo_ps(a, b); } - __forceinline vfloat4 unpackhi(const vfloat4& a, const vfloat4& b) { return _mm_unpackhi_ps(a, b); } - -#if defined(__aarch64__) - template<int i0, int i1, int i2, int i3> - __forceinline vfloat4 shuffle(const vfloat4& v) { - return vreinterpretq_f32_u8(vqtbl1q_u8( (uint8x16_t)v.v, _MN_SHUFFLE(i0, i1, i2, i3))); - } - template<int i0, int i1, int i2, int i3> - __forceinline vfloat4 shuffle(const vfloat4& a, const vfloat4& b) { - return vreinterpretq_f32_u8(vqtbl2q_u8( (uint8x16x2_t){(uint8x16_t)a.v, (uint8x16_t)b.v}, _MF_SHUFFLE(i0, i1, i2, i3))); - } -#else - template<int i0, int i1, int i2, int i3> - __forceinline vfloat4 shuffle(const vfloat4& v) { - return _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v), _MM_SHUFFLE(i3, i2, i1, i0))); - } - - template<int i0, int i1, int i2, int i3> - __forceinline vfloat4 shuffle(const vfloat4& a, const vfloat4& b) { - return _mm_shuffle_ps(a, b, _MM_SHUFFLE(i3, i2, i1, i0)); - } -#endif - -#if defined (__SSSE3__) - __forceinline vfloat4 shuffle8(const vfloat4& a, const vint4& shuf) { - return _mm_castsi128_ps(_mm_shuffle_epi8(_mm_castps_si128(a), shuf)); - } -#endif - -#if defined(__aarch64__) - template<> __forceinline vfloat4 shuffle<0, 0, 2, 2>(const vfloat4& v) { return __m128(vqtbl1q_u8( uint8x16_t(v.v), v0022 )); } - template<> __forceinline vfloat4 shuffle<1, 1, 3, 3>(const vfloat4& v) { return __m128(vqtbl1q_u8( uint8x16_t(v.v), v1133)); } - template<> __forceinline vfloat4 shuffle<0, 1, 0, 1>(const vfloat4& v) { return __m128(vqtbl1q_u8( uint8x16_t(v.v), v0101)); } -#elif defined(__SSE3__) - template<> __forceinline vfloat4 shuffle<0, 0, 2, 2>(const vfloat4& v) { return _mm_moveldup_ps(v); } - template<> __forceinline vfloat4 shuffle<1, 1, 3, 3>(const vfloat4& v) { return _mm_movehdup_ps(v); } - template<> __forceinline vfloat4 shuffle<0, 1, 0, 1>(const vfloat4& v) { return _mm_castpd_ps(_mm_movedup_pd(_mm_castps_pd(v))); } -#endif - - template<int i> - __forceinline vfloat4 shuffle(const vfloat4& v) { - return shuffle<i,i,i,i>(v); - } - -#if defined(__aarch64__) - template<int i> __forceinline float extract(const vfloat4& a); - template<> __forceinline float extract<0>(const vfloat4& b) { - return b[0]; - } - template<> __forceinline float extract<1>(const vfloat4& b) { - return b[1]; - } - template<> __forceinline float extract<2>(const vfloat4& b) { - return b[2]; - } - template<> __forceinline float extract<3>(const vfloat4& b) { - return b[3]; - } -#elif defined (__SSE4_1__) && !defined(__GNUC__) - template<int i> __forceinline float extract(const vfloat4& a) { return _mm_cvtss_f32(_mm_extract_ps(a,i)); } - template<> __forceinline float extract<0>(const vfloat4& a) { return _mm_cvtss_f32(a); } -#else - template<int i> __forceinline float extract(const vfloat4& a) { return _mm_cvtss_f32(shuffle<i,i,i,i>(a)); } - template<> __forceinline float extract<0>(const vfloat4& a) { return _mm_cvtss_f32(a); } -#endif - - -#if defined(__aarch64__) - template<int dst> __forceinline vfloat4 insert(const vfloat4& a, float b); - template<> __forceinline vfloat4 insert<0>(const vfloat4& a, float b) - { - vfloat4 c = a; - c[0] = b; - return c; - } - template<> __forceinline vfloat4 insert<1>(const vfloat4& a, float b) - { - vfloat4 c = a; - c[1] = b; - return c; - } - template<> __forceinline vfloat4 insert<2>(const vfloat4& a, float b) - { - vfloat4 c = a; - c[2] = b; - return c; - } - template<> __forceinline vfloat4 insert<3>(const vfloat4& a, float b) - { - vfloat4 c = a; - c[3] = b; - return c; - } -#elif defined (__SSE4_1__) - template<int dst, int src, int clr> __forceinline vfloat4 insert(const vfloat4& a, const vfloat4& b) { return _mm_insert_ps(a, b, (dst << 4) | (src << 6) | clr); } - template<int dst, int src> __forceinline vfloat4 insert(const vfloat4& a, const vfloat4& b) { return insert<dst, src, 0>(a, b); } - template<int dst> __forceinline vfloat4 insert(const vfloat4& a, const float b) { return insert<dst, 0>(a, _mm_set_ss(b)); } -#else - template<int dst, int src> __forceinline vfloat4 insert(const vfloat4& a, const vfloat4& b) { vfloat4 c = a; c[dst&3] = b[src&3]; return c; } - template<int dst> __forceinline vfloat4 insert(const vfloat4& a, float b) { vfloat4 c = a; c[dst&3] = b; return c; } -#endif - -#if defined(__aarch64__) - __forceinline float toScalar(const vfloat4& v) { - return v[0]; - } -#else - __forceinline float toScalar(const vfloat4& v) { return _mm_cvtss_f32(v); } -#endif - __forceinline vfloat4 broadcast4f(const vfloat4& a, size_t k) { - return vfloat4::broadcast(&a[k]); - } - - __forceinline vfloat4 shift_right_1(const vfloat4& x) { - return _mm_castsi128_ps(_mm_srli_si128(_mm_castps_si128(x), 4)); - } - -#if defined (__AVX2__) - __forceinline vfloat4 permute(const vfloat4 &a, const __m128i &index) { - return _mm_permutevar_ps(a,index); - } - - __forceinline vfloat4 broadcast1f(const void* a) { return _mm_broadcast_ss((float*)a); } - -#endif - -#if defined(__AVX512VL__) - template<int i> - __forceinline vfloat4 align_shift_right(const vfloat4& a, const vfloat4& b) { - return _mm_castsi128_ps(_mm_alignr_epi32(_mm_castps_si128(a), _mm_castps_si128(b), i)); - } -#endif - - - //////////////////////////////////////////////////////////////////////////////// - /// Sorting Network - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vfloat4 sort_ascending(const vfloat4& v) - { - const vfloat4 a0 = v; - const vfloat4 b0 = shuffle<1,0,3,2>(a0); - const vfloat4 c0 = min(a0,b0); - const vfloat4 d0 = max(a0,b0); - const vfloat4 a1 = select<0x5 /* 0b0101 */>(c0,d0); - const vfloat4 b1 = shuffle<2,3,0,1>(a1); - const vfloat4 c1 = min(a1,b1); - const vfloat4 d1 = max(a1,b1); - const vfloat4 a2 = select<0x3 /* 0b0011 */>(c1,d1); - const vfloat4 b2 = shuffle<0,2,1,3>(a2); - const vfloat4 c2 = min(a2,b2); - const vfloat4 d2 = max(a2,b2); - const vfloat4 a3 = select<0x2 /* 0b0010 */>(c2,d2); - return a3; - } - - __forceinline vfloat4 sort_descending(const vfloat4& v) - { - const vfloat4 a0 = v; - const vfloat4 b0 = shuffle<1,0,3,2>(a0); - const vfloat4 c0 = max(a0,b0); - const vfloat4 d0 = min(a0,b0); - const vfloat4 a1 = select<0x5 /* 0b0101 */>(c0,d0); - const vfloat4 b1 = shuffle<2,3,0,1>(a1); - const vfloat4 c1 = max(a1,b1); - const vfloat4 d1 = min(a1,b1); - const vfloat4 a2 = select<0x3 /* 0b0011 */>(c1,d1); - const vfloat4 b2 = shuffle<0,2,1,3>(a2); - const vfloat4 c2 = max(a2,b2); - const vfloat4 d2 = min(a2,b2); - const vfloat4 a3 = select<0x2 /* 0b0010 */>(c2,d2); - return a3; - } - - //////////////////////////////////////////////////////////////////////////////// - /// Transpose - //////////////////////////////////////////////////////////////////////////////// - - __forceinline void transpose(const vfloat4& r0, const vfloat4& r1, const vfloat4& r2, const vfloat4& r3, vfloat4& c0, vfloat4& c1, vfloat4& c2, vfloat4& c3) - { - vfloat4 l02 = unpacklo(r0,r2); - vfloat4 h02 = unpackhi(r0,r2); - vfloat4 l13 = unpacklo(r1,r3); - vfloat4 h13 = unpackhi(r1,r3); - c0 = unpacklo(l02,l13); - c1 = unpackhi(l02,l13); - c2 = unpacklo(h02,h13); - c3 = unpackhi(h02,h13); - } - - __forceinline void transpose(const vfloat4& r0, const vfloat4& r1, const vfloat4& r2, const vfloat4& r3, vfloat4& c0, vfloat4& c1, vfloat4& c2) - { - vfloat4 l02 = unpacklo(r0,r2); - vfloat4 h02 = unpackhi(r0,r2); - vfloat4 l13 = unpacklo(r1,r3); - vfloat4 h13 = unpackhi(r1,r3); - c0 = unpacklo(l02,l13); - c1 = unpackhi(l02,l13); - c2 = unpacklo(h02,h13); - } - - //////////////////////////////////////////////////////////////////////////////// - /// Reductions - //////////////////////////////////////////////////////////////////////////////// -#if defined(__aarch64__) - __forceinline vfloat4 vreduce_min(const vfloat4& v) { float h = vminvq_f32(v); return vdupq_n_f32(h); } - __forceinline vfloat4 vreduce_max(const vfloat4& v) { float h = vmaxvq_f32(v); return vdupq_n_f32(h); } - __forceinline vfloat4 vreduce_add(const vfloat4& v) { float h = vaddvq_f32(v); return vdupq_n_f32(h); } -#else - __forceinline vfloat4 vreduce_min(const vfloat4& v) { vfloat4 h = min(shuffle<1,0,3,2>(v),v); return min(shuffle<2,3,0,1>(h),h); } - __forceinline vfloat4 vreduce_max(const vfloat4& v) { vfloat4 h = max(shuffle<1,0,3,2>(v),v); return max(shuffle<2,3,0,1>(h),h); } - __forceinline vfloat4 vreduce_add(const vfloat4& v) { vfloat4 h = shuffle<1,0,3,2>(v) + v ; return shuffle<2,3,0,1>(h) + h ; } -#endif - -#if defined(__aarch64__) - __forceinline float reduce_min(const vfloat4& v) { return vminvq_f32(v); } - __forceinline float reduce_max(const vfloat4& v) { return vmaxvq_f32(v); } - __forceinline float reduce_add(const vfloat4& v) { return vaddvq_f32(v); } -#else - __forceinline float reduce_min(const vfloat4& v) { return _mm_cvtss_f32(vreduce_min(v)); } - __forceinline float reduce_max(const vfloat4& v) { return _mm_cvtss_f32(vreduce_max(v)); } - __forceinline float reduce_add(const vfloat4& v) { return _mm_cvtss_f32(vreduce_add(v)); } -#endif - - __forceinline size_t select_min(const vboolf4& valid, const vfloat4& v) - { - const vfloat4 a = select(valid,v,vfloat4(pos_inf)); - const vbool4 valid_min = valid & (a == vreduce_min(a)); - return bsf(movemask(any(valid_min) ? valid_min : valid)); - } - __forceinline size_t select_max(const vboolf4& valid, const vfloat4& v) - { - const vfloat4 a = select(valid,v,vfloat4(neg_inf)); - const vbool4 valid_max = valid & (a == vreduce_max(a)); - return bsf(movemask(any(valid_max) ? valid_max : valid)); - } - - //////////////////////////////////////////////////////////////////////////////// - /// Euclidian Space Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline float dot(const vfloat4& a, const vfloat4& b) { - return reduce_add(a*b); - } - - __forceinline vfloat4 cross(const vfloat4& a, const vfloat4& b) - { - const vfloat4 a0 = a; - const vfloat4 b0 = shuffle<1,2,0,3>(b); - const vfloat4 a1 = shuffle<1,2,0,3>(a); - const vfloat4 b1 = b; - return shuffle<1,2,0,3>(prod_diff(a0,b0,a1,b1)); - } - - //////////////////////////////////////////////////////////////////////////////// - /// Output Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline embree_ostream operator <<(embree_ostream cout, const vfloat4& a) { - return cout << "<" << a[0] << ", " << a[1] << ", " << a[2] << ", " << a[3] << ">"; - } - -} diff --git a/thirdparty/embree-aarch64/common/simd/vfloat8_avx.h b/thirdparty/embree-aarch64/common/simd/vfloat8_avx.h deleted file mode 100644 index 3c7e4a8cdc..0000000000 --- a/thirdparty/embree-aarch64/common/simd/vfloat8_avx.h +++ /dev/null @@ -1,847 +0,0 @@ -// Copyright 2009-2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 - -#pragma once - -namespace embree -{ - /* 8-wide AVX float type */ - template<> - struct vfloat<8> - { - ALIGNED_STRUCT_(32); - - typedef vboolf8 Bool; - typedef vint8 Int; - typedef vfloat8 Float; - - enum { size = 8 }; // number of SIMD elements - union { __m256 v; float f[8]; int i[8]; }; // data - - //////////////////////////////////////////////////////////////////////////////// - /// Constructors, Assignment & Cast Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vfloat() {} - __forceinline vfloat(const vfloat8& other) { v = other.v; } - __forceinline vfloat8& operator =(const vfloat8& other) { v = other.v; return *this; } - - __forceinline vfloat(__m256 a) : v(a) {} - __forceinline operator const __m256&() const { return v; } - __forceinline operator __m256&() { return v; } - - __forceinline explicit vfloat(const vfloat4& a) : v(_mm256_insertf128_ps(_mm256_castps128_ps256(a),a,1)) {} - __forceinline vfloat(const vfloat4& a, const vfloat4& b) : v(_mm256_insertf128_ps(_mm256_castps128_ps256(a),b,1)) {} - - __forceinline explicit vfloat(const int8_t* a) : v(_mm256_loadu_ps((const float*)a)) {} - __forceinline vfloat(float a) : v(_mm256_set1_ps(a)) {} - __forceinline vfloat(float a, float b) : v(_mm256_set_ps(b, a, b, a, b, a, b, a)) {} - __forceinline vfloat(float a, float b, float c, float d) : v(_mm256_set_ps(d, c, b, a, d, c, b, a)) {} - __forceinline vfloat(float a, float b, float c, float d, float e, float f, float g, float h) : v(_mm256_set_ps(h, g, f, e, d, c, b, a)) {} - - __forceinline explicit vfloat(__m256i a) : v(_mm256_cvtepi32_ps(a)) {} - - //////////////////////////////////////////////////////////////////////////////// - /// Constants - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vfloat(ZeroTy) : v(_mm256_setzero_ps()) {} - __forceinline vfloat(OneTy) : v(_mm256_set1_ps(1.0f)) {} - __forceinline vfloat(PosInfTy) : v(_mm256_set1_ps(pos_inf)) {} - __forceinline vfloat(NegInfTy) : v(_mm256_set1_ps(neg_inf)) {} - __forceinline vfloat(StepTy) : v(_mm256_set_ps(7.0f, 6.0f, 5.0f, 4.0f, 3.0f, 2.0f, 1.0f, 0.0f)) {} - __forceinline vfloat(NaNTy) : v(_mm256_set1_ps(nan)) {} - __forceinline vfloat(UndefinedTy) : v(_mm256_undefined_ps()) {} - - //////////////////////////////////////////////////////////////////////////////// - /// Loads and Stores - //////////////////////////////////////////////////////////////////////////////// - - static __forceinline vfloat8 broadcast(const void* a) { - return _mm256_broadcast_ss((float*)a); - } - - static __forceinline vfloat8 broadcast2(const float* a, const float* b) { -#if defined(__INTEL_COMPILER) - const vfloat8 v0 = _mm256_broadcast_ss(a); - const vfloat8 v1 = _mm256_broadcast_ss(b); - return _mm256_blend_ps(v1, v0, 0xf); -#else - return _mm256_set_ps(*b,*b,*b,*b,*a,*a,*a,*a); -#endif - } - - static __forceinline vfloat8 broadcast4f(const vfloat4* ptr) { - return _mm256_broadcast_ps((__m128*)ptr); - } - - static __forceinline vfloat8 load(const int8_t* ptr) { -#if defined(__AVX2__) - return _mm256_cvtepi32_ps(_mm256_cvtepi8_epi32(_mm_loadu_si128((__m128i*)ptr))); -#else - return vfloat8(vfloat4::load(ptr),vfloat4::load(ptr+4)); -#endif - } - - static __forceinline vfloat8 load(const uint8_t* ptr) { -#if defined(__AVX2__) - return _mm256_cvtepi32_ps(_mm256_cvtepu8_epi32(_mm_loadu_si128((__m128i*)ptr))); -#else - return vfloat8(vfloat4::load(ptr),vfloat4::load(ptr+4)); -#endif - } - - static __forceinline vfloat8 load(const short* ptr) { -#if defined(__AVX2__) - return _mm256_cvtepi32_ps(_mm256_cvtepi16_epi32(_mm_loadu_si128((__m128i*)ptr))); -#else - return vfloat8(vfloat4::load(ptr),vfloat4::load(ptr+4)); -#endif - } - - static __forceinline vfloat8 load (const void* ptr) { return _mm256_load_ps((float*)ptr); } - static __forceinline vfloat8 loadu(const void* ptr) { return _mm256_loadu_ps((float*)ptr); } - - static __forceinline void store (void* ptr, const vfloat8& v) { return _mm256_store_ps((float*)ptr,v); } - static __forceinline void storeu(void* ptr, const vfloat8& v) { return _mm256_storeu_ps((float*)ptr,v); } - -#if defined(__AVX512VL__) - - static __forceinline vfloat8 compact(const vboolf8& mask, vfloat8 &v) { - return _mm256_mask_compress_ps(v, mask, v); - } - static __forceinline vfloat8 compact(const vboolf8& mask, vfloat8 &a, const vfloat8& b) { - return _mm256_mask_compress_ps(a, mask, b); - } - - static __forceinline vfloat8 load (const vboolf8& mask, const void* ptr) { return _mm256_mask_load_ps (_mm256_setzero_ps(),mask,(float*)ptr); } - static __forceinline vfloat8 loadu(const vboolf8& mask, const void* ptr) { return _mm256_mask_loadu_ps(_mm256_setzero_ps(),mask,(float*)ptr); } - - static __forceinline void store (const vboolf8& mask, void* ptr, const vfloat8& v) { _mm256_mask_store_ps ((float*)ptr,mask,v); } - static __forceinline void storeu(const vboolf8& mask, void* ptr, const vfloat8& v) { _mm256_mask_storeu_ps((float*)ptr,mask,v); } -#elif defined(__aarch64__) - static __forceinline vfloat8 load (const vboolf8& mask, const void* ptr) { return _mm256_maskload_ps((float*)ptr,(__m256i)mask.v); } - static __forceinline vfloat8 loadu(const vboolf8& mask, const void* ptr) { return _mm256_maskload_ps((float*)ptr,(__m256i)mask.v); } - - static __forceinline void store (const vboolf8& mask, void* ptr, const vfloat8& v) { _mm256_maskstore_ps((float*)ptr,(__m256i)mask.v,v); } - static __forceinline void storeu(const vboolf8& mask, void* ptr, const vfloat8& v) { _mm256_maskstore_ps((float*)ptr,(__m256i)mask.v,v); } -#else - static __forceinline vfloat8 load (const vboolf8& mask, const void* ptr) { return _mm256_maskload_ps((float*)ptr,(__m256i)mask); } - static __forceinline vfloat8 loadu(const vboolf8& mask, const void* ptr) { return _mm256_maskload_ps((float*)ptr,(__m256i)mask); } - - static __forceinline void store (const vboolf8& mask, void* ptr, const vfloat8& v) { _mm256_maskstore_ps((float*)ptr,(__m256i)mask,v); } - static __forceinline void storeu(const vboolf8& mask, void* ptr, const vfloat8& v) { _mm256_maskstore_ps((float*)ptr,(__m256i)mask,v); } -#endif - -#if defined(__AVX2__) - static __forceinline vfloat8 load_nt(void* ptr) { - return _mm256_castsi256_ps(_mm256_stream_load_si256((__m256i*)ptr)); - } -#endif - - static __forceinline void store_nt(void* ptr, const vfloat8& v) { - _mm256_stream_ps((float*)ptr,v); - } - - template<int scale = 4> - static __forceinline vfloat8 gather(const float* ptr, const vint8& index) { -#if defined(__AVX2__) && !defined(__aarch64__) - return _mm256_i32gather_ps(ptr, index ,scale); -#else - return vfloat8( - *(float*)(((int8_t*)ptr)+scale*index[0]), - *(float*)(((int8_t*)ptr)+scale*index[1]), - *(float*)(((int8_t*)ptr)+scale*index[2]), - *(float*)(((int8_t*)ptr)+scale*index[3]), - *(float*)(((int8_t*)ptr)+scale*index[4]), - *(float*)(((int8_t*)ptr)+scale*index[5]), - *(float*)(((int8_t*)ptr)+scale*index[6]), - *(float*)(((int8_t*)ptr)+scale*index[7])); -#endif - } - - template<int scale = 4> - static __forceinline vfloat8 gather(const vboolf8& mask, const float* ptr, const vint8& index) { - vfloat8 r = zero; -#if defined(__AVX512VL__) - return _mm256_mmask_i32gather_ps(r, mask, index, ptr, scale); -#elif defined(__AVX2__) && !defined(__aarch64__) - return _mm256_mask_i32gather_ps(r, ptr, index, mask, scale); -#else - if (likely(mask[0])) r[0] = *(float*)(((int8_t*)ptr)+scale*index[0]); - if (likely(mask[1])) r[1] = *(float*)(((int8_t*)ptr)+scale*index[1]); - if (likely(mask[2])) r[2] = *(float*)(((int8_t*)ptr)+scale*index[2]); - if (likely(mask[3])) r[3] = *(float*)(((int8_t*)ptr)+scale*index[3]); - if (likely(mask[4])) r[4] = *(float*)(((int8_t*)ptr)+scale*index[4]); - if (likely(mask[5])) r[5] = *(float*)(((int8_t*)ptr)+scale*index[5]); - if (likely(mask[6])) r[6] = *(float*)(((int8_t*)ptr)+scale*index[6]); - if (likely(mask[7])) r[7] = *(float*)(((int8_t*)ptr)+scale*index[7]); - return r; - #endif - } - - template<int scale = 4> - static __forceinline void scatter(void* ptr, const vint8& ofs, const vfloat8& v) - { -#if defined(__AVX512VL__) - _mm256_i32scatter_ps((float*)ptr, ofs, v, scale); -#else - *(float*)(((int8_t*)ptr)+scale*ofs[0]) = v[0]; - *(float*)(((int8_t*)ptr)+scale*ofs[1]) = v[1]; - *(float*)(((int8_t*)ptr)+scale*ofs[2]) = v[2]; - *(float*)(((int8_t*)ptr)+scale*ofs[3]) = v[3]; - *(float*)(((int8_t*)ptr)+scale*ofs[4]) = v[4]; - *(float*)(((int8_t*)ptr)+scale*ofs[5]) = v[5]; - *(float*)(((int8_t*)ptr)+scale*ofs[6]) = v[6]; - *(float*)(((int8_t*)ptr)+scale*ofs[7]) = v[7]; -#endif - } - - template<int scale = 4> - static __forceinline void scatter(const vboolf8& mask, void* ptr, const vint8& ofs, const vfloat8& v) - { -#if defined(__AVX512VL__) - _mm256_mask_i32scatter_ps((float*)ptr, mask, ofs, v, scale); -#else - if (likely(mask[0])) *(float*)(((int8_t*)ptr)+scale*ofs[0]) = v[0]; - if (likely(mask[1])) *(float*)(((int8_t*)ptr)+scale*ofs[1]) = v[1]; - if (likely(mask[2])) *(float*)(((int8_t*)ptr)+scale*ofs[2]) = v[2]; - if (likely(mask[3])) *(float*)(((int8_t*)ptr)+scale*ofs[3]) = v[3]; - if (likely(mask[4])) *(float*)(((int8_t*)ptr)+scale*ofs[4]) = v[4]; - if (likely(mask[5])) *(float*)(((int8_t*)ptr)+scale*ofs[5]) = v[5]; - if (likely(mask[6])) *(float*)(((int8_t*)ptr)+scale*ofs[6]) = v[6]; - if (likely(mask[7])) *(float*)(((int8_t*)ptr)+scale*ofs[7]) = v[7]; -#endif - } - - static __forceinline void store(const vboolf8& mask, int8_t* ptr, const vint8& ofs, const vfloat8& v) { - scatter<1>(mask,ptr,ofs,v); - } - static __forceinline void store(const vboolf8& mask, float* ptr, const vint8& ofs, const vfloat8& v) { - scatter<4>(mask,ptr,ofs,v); - } - - //////////////////////////////////////////////////////////////////////////////// - /// Array Access - //////////////////////////////////////////////////////////////////////////////// - - __forceinline const float& operator [](size_t index) const { assert(index < 8); return f[index]; } - __forceinline float& operator [](size_t index) { assert(index < 8); return f[index]; } - }; - - - //////////////////////////////////////////////////////////////////////////////// - /// Unary Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vfloat8 asFloat(const vint8& a) { return _mm256_castsi256_ps(a); } - __forceinline vint8 asInt (const vfloat8& a) { return _mm256_castps_si256(a); } - - __forceinline vint8 toInt (const vfloat8& a) { return vint8(a); } - __forceinline vfloat8 toFloat(const vint8& a) { return vfloat8(a); } - - __forceinline vfloat8 operator +(const vfloat8& a) { return a; } -#if !defined(__aarch64__) - __forceinline vfloat8 operator -(const vfloat8& a) { - const __m256 mask = _mm256_castsi256_ps(_mm256_set1_epi32(0x80000000)); - return _mm256_xor_ps(a, mask); - } -#else - __forceinline vfloat8 operator -(const vfloat8& a) { - __m256 res; - res.lo = vnegq_f32(a.v.lo); - res.hi = vnegq_f32(a.v.hi); - return res; -} -#endif - -#if !defined(__aarch64__) -__forceinline vfloat8 abs(const vfloat8& a) { - const __m256 mask = _mm256_castsi256_ps(_mm256_set1_epi32(0x7fffffff)); - return _mm256_and_ps(a, mask); -} -#else -__forceinline vfloat8 abs(const vfloat8& a) { - __m256 res; - res.lo = vabsq_f32(a.v.lo); - res.hi = vabsq_f32(a.v.hi); - return res; -} -#endif - -#if !defined(__aarch64__) - __forceinline vfloat8 sign (const vfloat8& a) { return _mm256_blendv_ps(vfloat8(one), -vfloat8(one), _mm256_cmp_ps(a, vfloat8(zero), _CMP_NGE_UQ)); } -#else - __forceinline vfloat8 sign (const vfloat8& a) { return _mm256_blendv_ps(vfloat8(one), -vfloat8(one), _mm256_cmplt_ps(a, vfloat8(zero))); } -#endif - __forceinline vfloat8 signmsk(const vfloat8& a) { return _mm256_and_ps(a,_mm256_castsi256_ps(_mm256_set1_epi32(0x80000000))); } - - - static __forceinline vfloat8 rcp(const vfloat8& a) - { -#if defined(BUILD_IOS) && defined(__aarch64__) - // ios devices are faster doing full divide, no need for NR fixup - vfloat8 ret; - const float32x4_t one = vdupq_n_f32(1.0f); - ret.v.lo = vdivq_f32(one, a.v.lo); - ret.v.hi = vdivq_f32(one, a.v.hi); - return ret; -#endif - -#if defined(__AVX512VL__) - const vfloat8 r = _mm256_rcp14_ps(a); -#else - const vfloat8 r = _mm256_rcp_ps(a); -#endif - -#if defined(__AVX2__) //&& !defined(aarch64) - return _mm256_mul_ps(r, _mm256_fnmadd_ps(r, a, vfloat8(2.0f))); -#else - return _mm256_mul_ps(r, _mm256_sub_ps(vfloat8(2.0f), _mm256_mul_ps(r, a))); -#endif - } - __forceinline vfloat8 sqr (const vfloat8& a) { return _mm256_mul_ps(a,a); } - __forceinline vfloat8 sqrt(const vfloat8& a) { return _mm256_sqrt_ps(a); } - - static __forceinline vfloat8 rsqrt(const vfloat8& a) - { -#if defined(__AVX512VL__) - const vfloat8 r = _mm256_rsqrt14_ps(a); -#else - const vfloat8 r = _mm256_rsqrt_ps(a); -#endif - -#if defined(__AVX2__) - return _mm256_fmadd_ps(_mm256_set1_ps(1.5f), r, - _mm256_mul_ps(_mm256_mul_ps(_mm256_mul_ps(a, _mm256_set1_ps(-0.5f)), r), _mm256_mul_ps(r, r))); -#else - return _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(1.5f), r), - _mm256_mul_ps(_mm256_mul_ps(_mm256_mul_ps(a, _mm256_set1_ps(-0.5f)), r), _mm256_mul_ps(r, r))); -#endif - } - - //////////////////////////////////////////////////////////////////////////////// - /// Binary Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vfloat8 operator +(const vfloat8& a, const vfloat8& b) { return _mm256_add_ps(a, b); } - __forceinline vfloat8 operator +(const vfloat8& a, float b) { return a + vfloat8(b); } - __forceinline vfloat8 operator +(float a, const vfloat8& b) { return vfloat8(a) + b; } - - __forceinline vfloat8 operator -(const vfloat8& a, const vfloat8& b) { return _mm256_sub_ps(a, b); } - __forceinline vfloat8 operator -(const vfloat8& a, float b) { return a - vfloat8(b); } - __forceinline vfloat8 operator -(float a, const vfloat8& b) { return vfloat8(a) - b; } - - __forceinline vfloat8 operator *(const vfloat8& a, const vfloat8& b) { return _mm256_mul_ps(a, b); } - __forceinline vfloat8 operator *(const vfloat8& a, float b) { return a * vfloat8(b); } - __forceinline vfloat8 operator *(float a, const vfloat8& b) { return vfloat8(a) * b; } - - __forceinline vfloat8 operator /(const vfloat8& a, const vfloat8& b) { return _mm256_div_ps(a, b); } - __forceinline vfloat8 operator /(const vfloat8& a, float b) { return a / vfloat8(b); } - __forceinline vfloat8 operator /(float a, const vfloat8& b) { return vfloat8(a) / b; } - - __forceinline vfloat8 operator &(const vfloat8& a, const vfloat8& b) { return _mm256_and_ps(a,b); } - __forceinline vfloat8 operator |(const vfloat8& a, const vfloat8& b) { return _mm256_or_ps(a,b); } - __forceinline vfloat8 operator ^(const vfloat8& a, const vfloat8& b) { return _mm256_xor_ps(a,b); } - __forceinline vfloat8 operator ^(const vfloat8& a, const vint8& b) { return _mm256_xor_ps(a,_mm256_castsi256_ps(b)); } - - __forceinline vfloat8 min(const vfloat8& a, const vfloat8& b) { return _mm256_min_ps(a, b); } - __forceinline vfloat8 min(const vfloat8& a, float b) { return _mm256_min_ps(a, vfloat8(b)); } - __forceinline vfloat8 min(float a, const vfloat8& b) { return _mm256_min_ps(vfloat8(a), b); } - - __forceinline vfloat8 max(const vfloat8& a, const vfloat8& b) { return _mm256_max_ps(a, b); } - __forceinline vfloat8 max(const vfloat8& a, float b) { return _mm256_max_ps(a, vfloat8(b)); } - __forceinline vfloat8 max(float a, const vfloat8& b) { return _mm256_max_ps(vfloat8(a), b); } - - /* need "static __forceinline for MSVC, otherwise we'll link the wrong version in debug mode */ -#if defined(__AVX2__) - - static __forceinline vfloat8 mini(const vfloat8& a, const vfloat8& b) { - const vint8 ai = _mm256_castps_si256(a); - const vint8 bi = _mm256_castps_si256(b); - const vint8 ci = _mm256_min_epi32(ai,bi); - return _mm256_castsi256_ps(ci); - } - - static __forceinline vfloat8 maxi(const vfloat8& a, const vfloat8& b) { - const vint8 ai = _mm256_castps_si256(a); - const vint8 bi = _mm256_castps_si256(b); - const vint8 ci = _mm256_max_epi32(ai,bi); - return _mm256_castsi256_ps(ci); - } - - static __forceinline vfloat8 minui(const vfloat8& a, const vfloat8& b) { - const vint8 ai = _mm256_castps_si256(a); - const vint8 bi = _mm256_castps_si256(b); - const vint8 ci = _mm256_min_epu32(ai,bi); - return _mm256_castsi256_ps(ci); - } - - static __forceinline vfloat8 maxui(const vfloat8& a, const vfloat8& b) { - const vint8 ai = _mm256_castps_si256(a); - const vint8 bi = _mm256_castps_si256(b); - const vint8 ci = _mm256_max_epu32(ai,bi); - return _mm256_castsi256_ps(ci); - } - -#else - - static __forceinline vfloat8 mini(const vfloat8& a, const vfloat8& b) { - return asFloat(min(asInt(a),asInt(b))); - } - - static __forceinline vfloat8 maxi(const vfloat8& a, const vfloat8& b) { - return asFloat(max(asInt(a),asInt(b))); - } - -#endif - - //////////////////////////////////////////////////////////////////////////////// - /// Ternary Operators - //////////////////////////////////////////////////////////////////////////////// - -#if defined(__AVX2__) - static __forceinline vfloat8 madd (const vfloat8& a, const vfloat8& b, const vfloat8& c) { return _mm256_fmadd_ps(a,b,c); } - static __forceinline vfloat8 msub (const vfloat8& a, const vfloat8& b, const vfloat8& c) { return _mm256_fmsub_ps(a,b,c); } - static __forceinline vfloat8 nmadd (const vfloat8& a, const vfloat8& b, const vfloat8& c) { return _mm256_fnmadd_ps(a,b,c); } - static __forceinline vfloat8 nmsub (const vfloat8& a, const vfloat8& b, const vfloat8& c) { return _mm256_fnmsub_ps(a,b,c); } -#else - static __forceinline vfloat8 madd (const vfloat8& a, const vfloat8& b, const vfloat8& c) { return a*b+c; } - static __forceinline vfloat8 msub (const vfloat8& a, const vfloat8& b, const vfloat8& c) { return a*b-c; } - static __forceinline vfloat8 nmadd (const vfloat8& a, const vfloat8& b, const vfloat8& c) { return -a*b+c;} - static __forceinline vfloat8 nmsub (const vfloat8& a, const vfloat8& b, const vfloat8& c) { return -a*b-c; } -#endif - - //////////////////////////////////////////////////////////////////////////////// - /// Assignment Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vfloat8& operator +=(vfloat8& a, const vfloat8& b) { return a = a + b; } - __forceinline vfloat8& operator +=(vfloat8& a, float b) { return a = a + b; } - - __forceinline vfloat8& operator -=(vfloat8& a, const vfloat8& b) { return a = a - b; } - __forceinline vfloat8& operator -=(vfloat8& a, float b) { return a = a - b; } - - __forceinline vfloat8& operator *=(vfloat8& a, const vfloat8& b) { return a = a * b; } - __forceinline vfloat8& operator *=(vfloat8& a, float b) { return a = a * b; } - - __forceinline vfloat8& operator /=(vfloat8& a, const vfloat8& b) { return a = a / b; } - __forceinline vfloat8& operator /=(vfloat8& a, float b) { return a = a / b; } - - //////////////////////////////////////////////////////////////////////////////// - /// Comparison Operators + Select - //////////////////////////////////////////////////////////////////////////////// - -#if defined(__AVX512VL__) - static __forceinline vboolf8 operator ==(const vfloat8& a, const vfloat8& b) { return _mm256_cmp_ps_mask(a, b, _MM_CMPINT_EQ); } - static __forceinline vboolf8 operator !=(const vfloat8& a, const vfloat8& b) { return _mm256_cmp_ps_mask(a, b, _MM_CMPINT_NE); } - static __forceinline vboolf8 operator < (const vfloat8& a, const vfloat8& b) { return _mm256_cmp_ps_mask(a, b, _MM_CMPINT_LT); } - static __forceinline vboolf8 operator >=(const vfloat8& a, const vfloat8& b) { return _mm256_cmp_ps_mask(a, b, _MM_CMPINT_GE); } - static __forceinline vboolf8 operator > (const vfloat8& a, const vfloat8& b) { return _mm256_cmp_ps_mask(a, b, _MM_CMPINT_GT); } - static __forceinline vboolf8 operator <=(const vfloat8& a, const vfloat8& b) { return _mm256_cmp_ps_mask(a, b, _MM_CMPINT_LE); } - - static __forceinline vfloat8 select(const vboolf8& m, const vfloat8& t, const vfloat8& f) { - return _mm256_mask_blend_ps(m, f, t); - } -#elif !defined(__aarch64__) - __forceinline vboolf8 operator ==(const vfloat8& a, const vfloat8& b) { return _mm256_cmp_ps(a, b, _CMP_EQ_OQ); } - __forceinline vboolf8 operator !=(const vfloat8& a, const vfloat8& b) { return _mm256_cmp_ps(a, b, _CMP_NEQ_UQ); } - __forceinline vboolf8 operator < (const vfloat8& a, const vfloat8& b) { return _mm256_cmp_ps(a, b, _CMP_LT_OS); } - __forceinline vboolf8 operator >=(const vfloat8& a, const vfloat8& b) { return _mm256_cmp_ps(a, b, _CMP_NLT_US); } - __forceinline vboolf8 operator > (const vfloat8& a, const vfloat8& b) { return _mm256_cmp_ps(a, b, _CMP_NLE_US); } - __forceinline vboolf8 operator <=(const vfloat8& a, const vfloat8& b) { return _mm256_cmp_ps(a, b, _CMP_LE_OS); } - - __forceinline vfloat8 select(const vboolf8& m, const vfloat8& t, const vfloat8& f) { - return _mm256_blendv_ps(f, t, m); - } -#else - __forceinline vboolf8 operator ==(const vfloat8& a, const vfloat8& b) { return _mm256_cmpeq_ps(a, b); } - __forceinline vboolf8 operator !=(const vfloat8& a, const vfloat8& b) { return _mm256_cmpneq_ps(a, b); } - __forceinline vboolf8 operator < (const vfloat8& a, const vfloat8& b) { return _mm256_cmplt_ps(a, b); } - __forceinline vboolf8 operator >=(const vfloat8& a, const vfloat8& b) { return _mm256_cmpge_ps(a, b); } - __forceinline vboolf8 operator > (const vfloat8& a, const vfloat8& b) { return _mm256_cmpgt_ps(a, b); } - __forceinline vboolf8 operator <=(const vfloat8& a, const vfloat8& b) { return _mm256_cmple_ps(a, b); } - - __forceinline vfloat8 select(const vboolf8& m, const vfloat8& t, const vfloat8& f) { - return _mm256_blendv_ps(f, t, m); - } - -#endif - - template<int mask> - __forceinline vfloat8 select(const vfloat8& t, const vfloat8& f) { - return _mm256_blend_ps(f, t, mask); - } - - __forceinline vboolf8 operator ==(const vfloat8& a, const float& b) { return a == vfloat8(b); } - __forceinline vboolf8 operator ==(const float& a, const vfloat8& b) { return vfloat8(a) == b; } - - __forceinline vboolf8 operator !=(const vfloat8& a, const float& b) { return a != vfloat8(b); } - __forceinline vboolf8 operator !=(const float& a, const vfloat8& b) { return vfloat8(a) != b; } - - __forceinline vboolf8 operator < (const vfloat8& a, const float& b) { return a < vfloat8(b); } - __forceinline vboolf8 operator < (const float& a, const vfloat8& b) { return vfloat8(a) < b; } - - __forceinline vboolf8 operator >=(const vfloat8& a, const float& b) { return a >= vfloat8(b); } - __forceinline vboolf8 operator >=(const float& a, const vfloat8& b) { return vfloat8(a) >= b; } - - __forceinline vboolf8 operator > (const vfloat8& a, const float& b) { return a > vfloat8(b); } - __forceinline vboolf8 operator > (const float& a, const vfloat8& b) { return vfloat8(a) > b; } - - __forceinline vboolf8 operator <=(const vfloat8& a, const float& b) { return a <= vfloat8(b); } - __forceinline vboolf8 operator <=(const float& a, const vfloat8& b) { return vfloat8(a) <= b; } - - __forceinline vboolf8 eq(const vfloat8& a, const vfloat8& b) { return a == b; } - __forceinline vboolf8 ne(const vfloat8& a, const vfloat8& b) { return a != b; } - __forceinline vboolf8 lt(const vfloat8& a, const vfloat8& b) { return a < b; } - __forceinline vboolf8 ge(const vfloat8& a, const vfloat8& b) { return a >= b; } - __forceinline vboolf8 gt(const vfloat8& a, const vfloat8& b) { return a > b; } - __forceinline vboolf8 le(const vfloat8& a, const vfloat8& b) { return a <= b; } - -#if defined(__AVX512VL__) - static __forceinline vboolf8 eq(const vboolf8& mask, const vfloat8& a, const vfloat8& b) { return _mm256_mask_cmp_ps_mask(mask, a, b, _MM_CMPINT_EQ); } - static __forceinline vboolf8 ne(const vboolf8& mask, const vfloat8& a, const vfloat8& b) { return _mm256_mask_cmp_ps_mask(mask, a, b, _MM_CMPINT_NE); } - static __forceinline vboolf8 lt(const vboolf8& mask, const vfloat8& a, const vfloat8& b) { return _mm256_mask_cmp_ps_mask(mask, a, b, _MM_CMPINT_LT); } - static __forceinline vboolf8 ge(const vboolf8& mask, const vfloat8& a, const vfloat8& b) { return _mm256_mask_cmp_ps_mask(mask, a, b, _MM_CMPINT_GE); } - static __forceinline vboolf8 gt(const vboolf8& mask, const vfloat8& a, const vfloat8& b) { return _mm256_mask_cmp_ps_mask(mask, a, b, _MM_CMPINT_GT); } - static __forceinline vboolf8 le(const vboolf8& mask, const vfloat8& a, const vfloat8& b) { return _mm256_mask_cmp_ps_mask(mask, a, b, _MM_CMPINT_LE); } -#else - static __forceinline vboolf8 eq(const vboolf8& mask, const vfloat8& a, const vfloat8& b) { return mask & (a == b); } - static __forceinline vboolf8 ne(const vboolf8& mask, const vfloat8& a, const vfloat8& b) { return mask & (a != b); } - static __forceinline vboolf8 lt(const vboolf8& mask, const vfloat8& a, const vfloat8& b) { return mask & (a < b); } - static __forceinline vboolf8 ge(const vboolf8& mask, const vfloat8& a, const vfloat8& b) { return mask & (a >= b); } - static __forceinline vboolf8 gt(const vboolf8& mask, const vfloat8& a, const vfloat8& b) { return mask & (a > b); } - static __forceinline vboolf8 le(const vboolf8& mask, const vfloat8& a, const vfloat8& b) { return mask & (a <= b); } -#endif - - __forceinline vfloat8 lerp(const vfloat8& a, const vfloat8& b, const vfloat8& t) { - return madd(t,b-a,a); - } - - __forceinline bool isvalid (const vfloat8& v) { - return all((v > vfloat8(-FLT_LARGE)) & (v < vfloat8(+FLT_LARGE))); - } - - __forceinline bool is_finite (const vfloat8& a) { - return all((a >= vfloat8(-FLT_MAX)) & (a <= vfloat8(+FLT_MAX))); - } - - __forceinline bool is_finite (const vboolf8& valid, const vfloat8& a) { - return all(valid, (a >= vfloat8(-FLT_MAX)) & (a <= vfloat8(+FLT_MAX))); - } - - //////////////////////////////////////////////////////////////////////////////// - /// Rounding Functions - //////////////////////////////////////////////////////////////////////////////// - -#if !defined(__aarch64__) - __forceinline vfloat8 floor(const vfloat8& a) { return _mm256_round_ps(a, _MM_FROUND_TO_NEG_INF ); } - __forceinline vfloat8 ceil (const vfloat8& a) { return _mm256_round_ps(a, _MM_FROUND_TO_POS_INF ); } - __forceinline vfloat8 trunc(const vfloat8& a) { return _mm256_round_ps(a, _MM_FROUND_TO_ZERO ); } - __forceinline vfloat8 round(const vfloat8& a) { return _mm256_round_ps(a, _MM_FROUND_TO_NEAREST_INT); } -#else - __forceinline vfloat8 floor(const vfloat8& a) { return _mm256_floor_ps(a); } - __forceinline vfloat8 ceil (const vfloat8& a) { return _mm256_ceil_ps(a); } -#endif - - - __forceinline vfloat8 frac (const vfloat8& a) { return a-floor(a); } - - //////////////////////////////////////////////////////////////////////////////// - /// Movement/Shifting/Shuffling Functions - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vfloat8 unpacklo(const vfloat8& a, const vfloat8& b) { return _mm256_unpacklo_ps(a, b); } - __forceinline vfloat8 unpackhi(const vfloat8& a, const vfloat8& b) { return _mm256_unpackhi_ps(a, b); } - - template<int i> - __forceinline vfloat8 shuffle(const vfloat8& v) { - return _mm256_permute_ps(v, _MM_SHUFFLE(i, i, i, i)); - } - - template<int i0, int i1> - __forceinline vfloat8 shuffle4(const vfloat8& v) { - return _mm256_permute2f128_ps(v, v, (i1 << 4) | (i0 << 0)); - } - - template<int i0, int i1> - __forceinline vfloat8 shuffle4(const vfloat8& a, const vfloat8& b) { - return _mm256_permute2f128_ps(a, b, (i1 << 4) | (i0 << 0)); - } - - template<int i0, int i1, int i2, int i3> - __forceinline vfloat8 shuffle(const vfloat8& v) { - return _mm256_permute_ps(v, _MM_SHUFFLE(i3, i2, i1, i0)); - } - - template<int i0, int i1, int i2, int i3> - __forceinline vfloat8 shuffle(const vfloat8& a, const vfloat8& b) { - return _mm256_shuffle_ps(a, b, _MM_SHUFFLE(i3, i2, i1, i0)); - } - -#if !defined(__aarch64__) - template<> __forceinline vfloat8 shuffle<0, 0, 2, 2>(const vfloat8& v) { return _mm256_moveldup_ps(v); } - template<> __forceinline vfloat8 shuffle<1, 1, 3, 3>(const vfloat8& v) { return _mm256_movehdup_ps(v); } - template<> __forceinline vfloat8 shuffle<0, 1, 0, 1>(const vfloat8& v) { return _mm256_castpd_ps(_mm256_movedup_pd(_mm256_castps_pd(v))); } -#endif - - __forceinline vfloat8 broadcast(const float* ptr) { return _mm256_broadcast_ss(ptr); } - template<size_t i> __forceinline vfloat8 insert4(const vfloat8& a, const vfloat4& b) { return _mm256_insertf128_ps(a, b, i); } - template<size_t i> __forceinline vfloat4 extract4 (const vfloat8& a) { return _mm256_extractf128_ps(a, i); } - template<> __forceinline vfloat4 extract4<0>(const vfloat8& a) { return _mm256_castps256_ps128(a); } - - __forceinline float toScalar(const vfloat8& v) { return _mm_cvtss_f32(_mm256_castps256_ps128(v)); } - - __forceinline vfloat8 assign(const vfloat4& a) { return _mm256_castps128_ps256(a); } - -#if defined (__AVX2__) && !defined(__aarch64__) - __forceinline vfloat8 permute(const vfloat8& a, const __m256i& index) { - return _mm256_permutevar8x32_ps(a, index); - } -#endif - -#if defined(__AVX512VL__) - template<int i> - static __forceinline vfloat8 align_shift_right(const vfloat8& a, const vfloat8& b) { - return _mm256_castsi256_ps(_mm256_alignr_epi32(_mm256_castps_si256(a), _mm256_castps_si256(b), i)); - } -#endif - -#if defined (__AVX_I__) - template<const int mode> - static __forceinline vint4 convert_to_hf16(const vfloat8& a) { - return _mm256_cvtps_ph(a, mode); - } - - static __forceinline vfloat8 convert_from_hf16(const vint4& a) { - return _mm256_cvtph_ps(a); - } -#endif - - __forceinline vfloat4 broadcast4f(const vfloat8& a, const size_t k) { - return vfloat4::broadcast(&a[k]); - } - - __forceinline vfloat8 broadcast8f(const vfloat8& a, const size_t k) { - return vfloat8::broadcast(&a[k]); - } - -#if defined(__AVX512VL__) - static __forceinline vfloat8 shift_right_1(const vfloat8& x) { - return align_shift_right<1>(zero,x); - } -#else - static __forceinline vfloat8 shift_right_1(const vfloat8& x) { - const vfloat8 t0 = shuffle<1,2,3,0>(x); - const vfloat8 t1 = shuffle4<1,0>(t0); - return _mm256_blend_ps(t0,t1,0x88); - } -#endif - - __forceinline vint8 floori(const vfloat8& a) { - return vint8(floor(a)); - } - - //////////////////////////////////////////////////////////////////////////////// - /// Transpose - //////////////////////////////////////////////////////////////////////////////// - - __forceinline void transpose(const vfloat8& r0, const vfloat8& r1, const vfloat8& r2, const vfloat8& r3, vfloat8& c0, vfloat8& c1, vfloat8& c2, vfloat8& c3) - { - vfloat8 l02 = unpacklo(r0,r2); - vfloat8 h02 = unpackhi(r0,r2); - vfloat8 l13 = unpacklo(r1,r3); - vfloat8 h13 = unpackhi(r1,r3); - c0 = unpacklo(l02,l13); - c1 = unpackhi(l02,l13); - c2 = unpacklo(h02,h13); - c3 = unpackhi(h02,h13); - } - - __forceinline void transpose(const vfloat8& r0, const vfloat8& r1, const vfloat8& r2, const vfloat8& r3, vfloat8& c0, vfloat8& c1, vfloat8& c2) - { - vfloat8 l02 = unpacklo(r0,r2); - vfloat8 h02 = unpackhi(r0,r2); - vfloat8 l13 = unpacklo(r1,r3); - vfloat8 h13 = unpackhi(r1,r3); - c0 = unpacklo(l02,l13); - c1 = unpackhi(l02,l13); - c2 = unpacklo(h02,h13); - } - - __forceinline void transpose(const vfloat8& r0, const vfloat8& r1, const vfloat8& r2, const vfloat8& r3, const vfloat8& r4, const vfloat8& r5, const vfloat8& r6, const vfloat8& r7, - vfloat8& c0, vfloat8& c1, vfloat8& c2, vfloat8& c3, vfloat8& c4, vfloat8& c5, vfloat8& c6, vfloat8& c7) - { - vfloat8 h0,h1,h2,h3; transpose(r0,r1,r2,r3,h0,h1,h2,h3); - vfloat8 h4,h5,h6,h7; transpose(r4,r5,r6,r7,h4,h5,h6,h7); - c0 = shuffle4<0,2>(h0,h4); - c1 = shuffle4<0,2>(h1,h5); - c2 = shuffle4<0,2>(h2,h6); - c3 = shuffle4<0,2>(h3,h7); - c4 = shuffle4<1,3>(h0,h4); - c5 = shuffle4<1,3>(h1,h5); - c6 = shuffle4<1,3>(h2,h6); - c7 = shuffle4<1,3>(h3,h7); - } - - __forceinline void transpose(const vfloat4& r0, const vfloat4& r1, const vfloat4& r2, const vfloat4& r3, const vfloat4& r4, const vfloat4& r5, const vfloat4& r6, const vfloat4& r7, - vfloat8& c0, vfloat8& c1, vfloat8& c2, vfloat8& c3) - { - transpose(vfloat8(r0,r4), vfloat8(r1,r5), vfloat8(r2,r6), vfloat8(r3,r7), c0, c1, c2, c3); - } - - __forceinline void transpose(const vfloat4& r0, const vfloat4& r1, const vfloat4& r2, const vfloat4& r3, const vfloat4& r4, const vfloat4& r5, const vfloat4& r6, const vfloat4& r7, - vfloat8& c0, vfloat8& c1, vfloat8& c2) - { - transpose(vfloat8(r0,r4), vfloat8(r1,r5), vfloat8(r2,r6), vfloat8(r3,r7), c0, c1, c2); - } - - //////////////////////////////////////////////////////////////////////////////// - /// Reductions - //////////////////////////////////////////////////////////////////////////////// -#if !defined(__aarch64__) - __forceinline vfloat8 vreduce_min2(const vfloat8& v) { return min(v,shuffle<1,0,3,2>(v)); } - __forceinline vfloat8 vreduce_min4(const vfloat8& v) { vfloat8 v1 = vreduce_min2(v); return min(v1,shuffle<2,3,0,1>(v1)); } - __forceinline vfloat8 vreduce_min (const vfloat8& v) { vfloat8 v1 = vreduce_min4(v); return min(v1,shuffle4<1,0>(v1)); } - - __forceinline vfloat8 vreduce_max2(const vfloat8& v) { return max(v,shuffle<1,0,3,2>(v)); } - __forceinline vfloat8 vreduce_max4(const vfloat8& v) { vfloat8 v1 = vreduce_max2(v); return max(v1,shuffle<2,3,0,1>(v1)); } - __forceinline vfloat8 vreduce_max (const vfloat8& v) { vfloat8 v1 = vreduce_max4(v); return max(v1,shuffle4<1,0>(v1)); } - - __forceinline vfloat8 vreduce_add2(const vfloat8& v) { return v + shuffle<1,0,3,2>(v); } - __forceinline vfloat8 vreduce_add4(const vfloat8& v) { vfloat8 v1 = vreduce_add2(v); return v1 + shuffle<2,3,0,1>(v1); } - __forceinline vfloat8 vreduce_add (const vfloat8& v) { vfloat8 v1 = vreduce_add4(v); return v1 + shuffle4<1,0>(v1); } - - __forceinline float reduce_min(const vfloat8& v) { return toScalar(vreduce_min(v)); } - __forceinline float reduce_max(const vfloat8& v) { return toScalar(vreduce_max(v)); } - __forceinline float reduce_add(const vfloat8& v) { return toScalar(vreduce_add(v)); } -#else - __forceinline float reduce_min(const vfloat8& v) { return vminvq_f32(_mm_min_ps(v.v.lo,v.v.hi)); } - __forceinline float reduce_max(const vfloat8& v) { return vmaxvq_f32(_mm_max_ps(v.v.lo,v.v.hi)); } - __forceinline vfloat8 vreduce_min(const vfloat8& v) { return vfloat8(reduce_min(v)); } - __forceinline vfloat8 vreduce_max(const vfloat8& v) { return vfloat8(reduce_max(v)); } - __forceinline float reduce_add(const vfloat8& v) { return vaddvq_f32(_mm_add_ps(v.v.lo,v.v.hi)); } - -#endif - __forceinline size_t select_min(const vboolf8& valid, const vfloat8& v) - { - const vfloat8 a = select(valid,v,vfloat8(pos_inf)); - const vbool8 valid_min = valid & (a == vreduce_min(a)); - return bsf(movemask(any(valid_min) ? valid_min : valid)); - } - - __forceinline size_t select_max(const vboolf8& valid, const vfloat8& v) - { - const vfloat8 a = select(valid,v,vfloat8(neg_inf)); - const vbool8 valid_max = valid & (a == vreduce_max(a)); - return bsf(movemask(any(valid_max) ? valid_max : valid)); - } - - - //////////////////////////////////////////////////////////////////////////////// - /// Euclidian Space Operators (pairs of Vec3fa's) - //////////////////////////////////////////////////////////////////////////////// - - //__forceinline vfloat8 dot(const vfloat8& a, const vfloat8& b) { - // return vreduce_add4(a*b); - //} - - __forceinline vfloat8 dot(const vfloat8& a, const vfloat8& b) { - return _mm256_dp_ps(a,b,0x7F); - } - - __forceinline vfloat8 cross(const vfloat8& a, const vfloat8& b) - { - const vfloat8 a0 = a; - const vfloat8 b0 = shuffle<1,2,0,3>(b); - const vfloat8 a1 = shuffle<1,2,0,3>(a); - const vfloat8 b1 = b; - return shuffle<1,2,0,3>(msub(a0,b0,a1*b1)); - } - - //__forceinline float sqr_length (const vfloat<8>& a) { return dot(a,a); } - //__forceinline float rcp_length (const vfloat<8>& a) { return rsqrt(dot(a,a)); } - //__forceinline float rcp_length2(const vfloat<8>& a) { return rcp(dot(a,a)); } - //__forceinline float length (const vfloat<8>& a) { return sqrt(dot(a,a)); } - __forceinline vfloat<8> normalize(const vfloat<8>& a) { return a*rsqrt(dot(a,a)); } - //__forceinline float distance(const vfloat<8>& a, const vfloat<8>& b) { return length(a-b); } - //__forceinline float halfArea(const vfloat<8>& d) { return madd(d.x,(d.y+d.z),d.y*d.z); } - //__forceinline float area (const vfloat<8>& d) { return 2.0f*halfArea(d); } - //__forceinline vfloat<8> reflect(const vfloat<8>& V, const vfloat<8>& N) { return 2.0f*dot(V,N)*N-V; } - - //__forceinline vfloat<8> normalize_safe(const vfloat<8>& a) { - // const float d = dot(a,a); if (unlikely(d == 0.0f)) return a; else return a*rsqrt(d); - //} - - //////////////////////////////////////////////////////////////////////////////// - /// In Register Sorting - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vfloat8 sort_ascending(const vfloat8& v) - { - const vfloat8 a0 = v; - const vfloat8 b0 = shuffle<1,0,3,2>(a0); - const vfloat8 c0 = min(a0,b0); - const vfloat8 d0 = max(a0,b0); - const vfloat8 a1 = select<0x99 /* 0b10011001 */>(c0,d0); - const vfloat8 b1 = shuffle<2,3,0,1>(a1); - const vfloat8 c1 = min(a1,b1); - const vfloat8 d1 = max(a1,b1); - const vfloat8 a2 = select<0xc3 /* 0b11000011 */>(c1,d1); - const vfloat8 b2 = shuffle<1,0,3,2>(a2); - const vfloat8 c2 = min(a2,b2); - const vfloat8 d2 = max(a2,b2); - const vfloat8 a3 = select<0xa5 /* 0b10100101 */>(c2,d2); - const vfloat8 b3 = shuffle4<1,0>(a3); - const vfloat8 c3 = min(a3,b3); - const vfloat8 d3 = max(a3,b3); - const vfloat8 a4 = select<0xf /* 0b00001111 */>(c3,d3); - const vfloat8 b4 = shuffle<2,3,0,1>(a4); - const vfloat8 c4 = min(a4,b4); - const vfloat8 d4 = max(a4,b4); - const vfloat8 a5 = select<0x33 /* 0b00110011 */>(c4,d4); - const vfloat8 b5 = shuffle<1,0,3,2>(a5); - const vfloat8 c5 = min(a5,b5); - const vfloat8 d5 = max(a5,b5); - const vfloat8 a6 = select<0x55 /* 0b01010101 */>(c5,d5); - return a6; - } - - __forceinline vfloat8 sort_descending(const vfloat8& v) - { - const vfloat8 a0 = v; - const vfloat8 b0 = shuffle<1,0,3,2>(a0); - const vfloat8 c0 = max(a0,b0); - const vfloat8 d0 = min(a0,b0); - const vfloat8 a1 = select<0x99 /* 0b10011001 */>(c0,d0); - const vfloat8 b1 = shuffle<2,3,0,1>(a1); - const vfloat8 c1 = max(a1,b1); - const vfloat8 d1 = min(a1,b1); - const vfloat8 a2 = select<0xc3 /* 0b11000011 */>(c1,d1); - const vfloat8 b2 = shuffle<1,0,3,2>(a2); - const vfloat8 c2 = max(a2,b2); - const vfloat8 d2 = min(a2,b2); - const vfloat8 a3 = select<0xa5 /* 0b10100101 */>(c2,d2); - const vfloat8 b3 = shuffle4<1,0>(a3); - const vfloat8 c3 = max(a3,b3); - const vfloat8 d3 = min(a3,b3); - const vfloat8 a4 = select<0xf /* 0b00001111 */>(c3,d3); - const vfloat8 b4 = shuffle<2,3,0,1>(a4); - const vfloat8 c4 = max(a4,b4); - const vfloat8 d4 = min(a4,b4); - const vfloat8 a5 = select<0x33 /* 0b00110011 */>(c4,d4); - const vfloat8 b5 = shuffle<1,0,3,2>(a5); - const vfloat8 c5 = max(a5,b5); - const vfloat8 d5 = min(a5,b5); - const vfloat8 a6 = select<0x55 /* 0b01010101 */>(c5,d5); - return a6; - } - - //////////////////////////////////////////////////////////////////////////////// - /// Output Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline embree_ostream operator <<(embree_ostream cout, const vfloat8& a) { - return cout << "<" << a[0] << ", " << a[1] << ", " << a[2] << ", " << a[3] << ", " << a[4] << ", " << a[5] << ", " << a[6] << ", " << a[7] << ">"; - } -} diff --git a/thirdparty/embree-aarch64/common/simd/vint16_avx512.h b/thirdparty/embree-aarch64/common/simd/vint16_avx512.h deleted file mode 100644 index 3249bc2b45..0000000000 --- a/thirdparty/embree-aarch64/common/simd/vint16_avx512.h +++ /dev/null @@ -1,490 +0,0 @@ -// Copyright 2009-2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 - -#pragma once - -namespace embree -{ - /* 16-wide AVX-512 integer type */ - template<> - struct vint<16> - { - ALIGNED_STRUCT_(64); - - typedef vboolf16 Bool; - typedef vint16 Int; - typedef vfloat16 Float; - - enum { size = 16 }; // number of SIMD elements - union { // data - __m512i v; - int i[16]; - }; - - //////////////////////////////////////////////////////////////////////////////// - /// Constructors, Assignment & Cast Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vint() {} - __forceinline vint(const vint16& t) { v = t.v; } - __forceinline vint16& operator =(const vint16& f) { v = f.v; return *this; } - - __forceinline vint(const __m512i& t) { v = t; } - __forceinline operator __m512i() const { return v; } - __forceinline operator __m256i() const { return _mm512_castsi512_si256(v); } - - __forceinline vint(int i) { - v = _mm512_set1_epi32(i); - } - - __forceinline vint(int a, int b, int c, int d) { - v = _mm512_set4_epi32(d,c,b,a); - } - - __forceinline vint(int a0 , int a1 , int a2 , int a3, - int a4 , int a5 , int a6 , int a7, - int a8 , int a9 , int a10, int a11, - int a12, int a13, int a14, int a15) - { - v = _mm512_set_epi32(a15,a14,a13,a12,a11,a10,a9,a8,a7,a6,a5,a4,a3,a2,a1,a0); - } - - __forceinline vint(const vint4& i) { - v = _mm512_broadcast_i32x4(i); - } - - __forceinline vint(const vint4& a, const vint4& b, const vint4& c, const vint4& d) { - v = _mm512_castsi128_si512(a); - v = _mm512_inserti32x4(v, b, 1); - v = _mm512_inserti32x4(v, c, 2); - v = _mm512_inserti32x4(v, d, 3); - } - - __forceinline vint(const vint8& i) { - v = _mm512_castps_si512(_mm512_castpd_ps(_mm512_broadcast_f64x4(_mm256_castsi256_pd(i)))); - } - - __forceinline vint(const vint8& a, const vint8& b) { - v = _mm512_castsi256_si512(a); - v = _mm512_inserti64x4(v, b, 1); - } - - __forceinline explicit vint(const __m512& f) { - v = _mm512_cvtps_epi32(f); - } - - //////////////////////////////////////////////////////////////////////////////// - /// Constants - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vint(ZeroTy) : v(_mm512_setzero_epi32()) {} - __forceinline vint(OneTy) : v(_mm512_set1_epi32(1)) {} - __forceinline vint(PosInfTy) : v(_mm512_set1_epi32(pos_inf)) {} - __forceinline vint(NegInfTy) : v(_mm512_set1_epi32(neg_inf)) {} - __forceinline vint(StepTy) : v(_mm512_set_epi32(15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0)) {} - __forceinline vint(ReverseStepTy) : v(_mm512_setr_epi32(15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0)) {} - - //////////////////////////////////////////////////////////////////////////////// - /// Loads and Stores - //////////////////////////////////////////////////////////////////////////////// - - static __forceinline vint16 load (const void* addr) { return _mm512_load_si512((int*)addr); } - - static __forceinline vint16 load(const uint8_t* ptr) { return _mm512_cvtepu8_epi32(_mm_load_si128((__m128i*)ptr)); } - static __forceinline vint16 load(const unsigned short* ptr) { return _mm512_cvtepu16_epi32(_mm256_load_si256((__m256i*)ptr)); } - - static __forceinline vint16 loadu(const uint8_t* ptr) { return _mm512_cvtepu8_epi32(_mm_loadu_si128((__m128i*)ptr)); } - static __forceinline vint16 loadu(const unsigned short* ptr) { return _mm512_cvtepu16_epi32(_mm256_loadu_si256((__m256i*)ptr)); } - - static __forceinline vint16 loadu(const void* addr) { return _mm512_loadu_si512(addr); } - - static __forceinline vint16 load (const vboolf16& mask, const void* addr) { return _mm512_mask_load_epi32 (_mm512_setzero_epi32(),mask,addr); } - static __forceinline vint16 loadu(const vboolf16& mask, const void* addr) { return _mm512_mask_loadu_epi32(_mm512_setzero_epi32(),mask,addr); } - - static __forceinline void store (void* ptr, const vint16& v) { _mm512_store_si512 (ptr,v); } - static __forceinline void storeu(void* ptr, const vint16& v) { _mm512_storeu_si512(ptr,v); } - - static __forceinline void store (const vboolf16& mask, void* addr, const vint16& v2) { _mm512_mask_store_epi32(addr,mask,v2); } - static __forceinline void storeu(const vboolf16& mask, void* ptr, const vint16& f) { _mm512_mask_storeu_epi32((int*)ptr,mask,f); } - - static __forceinline void store_nt(void* __restrict__ ptr, const vint16& a) { _mm512_stream_si512((__m512i*)ptr,a); } - - /* pass by value to avoid compiler generating inefficient code */ - static __forceinline void storeu_compact(const vboolf16 mask, void* addr, vint16 reg) { - _mm512_mask_compressstoreu_epi32(addr,mask,reg); - } - - static __forceinline void storeu_compact_single(const vboolf16 mask, void* addr, vint16 reg) { - //_mm512_mask_compressstoreu_epi32(addr,mask,reg); - *(float*)addr = mm512_cvtss_f32(_mm512_mask_compress_ps(_mm512_castsi512_ps(reg),mask,_mm512_castsi512_ps(reg))); - } - - static __forceinline vint16 compact64bit(const vboolf16& mask, vint16 &v) { - return _mm512_mask_compress_epi64(v,mask,v); - } - - static __forceinline vint16 compact(const vboolf16& mask, vint16 &v) { - return _mm512_mask_compress_epi32(v,mask,v); - } - - static __forceinline vint16 compact(const vboolf16& mask, const vint16 &a, vint16 &b) { - return _mm512_mask_compress_epi32(a,mask,b); - } - - static __forceinline vint16 expand(const vboolf16& mask, const vint16& a, vint16& b) { - return _mm512_mask_expand_epi32(b,mask,a); - } - - template<int scale = 4> - static __forceinline vint16 gather(const int* ptr, const vint16& index) { - return _mm512_i32gather_epi32(index,ptr,scale); - } - - template<int scale = 4> - static __forceinline vint16 gather(const vboolf16& mask, const int* ptr, const vint16& index) { - return _mm512_mask_i32gather_epi32(_mm512_undefined_epi32(),mask,index,ptr,scale); - } - - template<int scale = 4> - static __forceinline vint16 gather(const vboolf16& mask, vint16& dest, const int* ptr, const vint16& index) { - return _mm512_mask_i32gather_epi32(dest,mask,index,ptr,scale); - } - - template<int scale = 4> - static __forceinline void scatter(int* ptr, const vint16& index, const vint16& v) { - _mm512_i32scatter_epi32((int*)ptr,index,v,scale); - } - - template<int scale = 4> - static __forceinline void scatter(const vboolf16& mask, int* ptr, const vint16& index, const vint16& v) { - _mm512_mask_i32scatter_epi32((int*)ptr,mask,index,v,scale); - } - - static __forceinline vint16 broadcast64bit(size_t v) { - return _mm512_set1_epi64(v); - } - - //////////////////////////////////////////////////////////////////////////////// - /// Array Access - //////////////////////////////////////////////////////////////////////////////// - - __forceinline int& operator [](size_t index) { assert(index < 16); return i[index]; } - __forceinline const int& operator [](size_t index) const { assert(index < 16); return i[index]; } - - __forceinline unsigned int uint (size_t index) const { assert(index < 16); return ((unsigned int*)i)[index]; } - __forceinline size_t& uint64_t(size_t index) const { assert(index < 8); return ((size_t*)i)[index]; } - }; - - //////////////////////////////////////////////////////////////////////////////// - /// Unary Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboolf16 asBool(const vint16& a) { return _mm512_movepi32_mask(a); } - - __forceinline vint16 operator +(const vint16& a) { return a; } - __forceinline vint16 operator -(const vint16& a) { return _mm512_sub_epi32(_mm512_setzero_epi32(), a); } - - //////////////////////////////////////////////////////////////////////////////// - /// Binary Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vint16 operator +(const vint16& a, const vint16& b) { return _mm512_add_epi32(a, b); } - __forceinline vint16 operator +(const vint16& a, int b) { return a + vint16(b); } - __forceinline vint16 operator +(int a, const vint16& b) { return vint16(a) + b; } - - __forceinline vint16 operator -(const vint16& a, const vint16& b) { return _mm512_sub_epi32(a, b); } - __forceinline vint16 operator -(const vint16& a, int b) { return a - vint16(b); } - __forceinline vint16 operator -(int a, const vint16& b) { return vint16(a) - b; } - - __forceinline vint16 operator *(const vint16& a, const vint16& b) { return _mm512_mullo_epi32(a, b); } - __forceinline vint16 operator *(const vint16& a, int b) { return a * vint16(b); } - __forceinline vint16 operator *(int a, const vint16& b) { return vint16(a) * b; } - - __forceinline vint16 operator &(const vint16& a, const vint16& b) { return _mm512_and_epi32(a, b); } - __forceinline vint16 operator &(const vint16& a, int b) { return a & vint16(b); } - __forceinline vint16 operator &(int a, const vint16& b) { return vint16(a) & b; } - - __forceinline vint16 operator |(const vint16& a, const vint16& b) { return _mm512_or_epi32(a, b); } - __forceinline vint16 operator |(const vint16& a, int b) { return a | vint16(b); } - __forceinline vint16 operator |(int a, const vint16& b) { return vint16(a) | b; } - - __forceinline vint16 operator ^(const vint16& a, const vint16& b) { return _mm512_xor_epi32(a, b); } - __forceinline vint16 operator ^(const vint16& a, int b) { return a ^ vint16(b); } - __forceinline vint16 operator ^(int a, const vint16& b) { return vint16(a) ^ b; } - - __forceinline vint16 operator <<(const vint16& a, int n) { return _mm512_slli_epi32(a, n); } - __forceinline vint16 operator >>(const vint16& a, int n) { return _mm512_srai_epi32(a, n); } - - __forceinline vint16 operator <<(const vint16& a, const vint16& n) { return _mm512_sllv_epi32(a, n); } - __forceinline vint16 operator >>(const vint16& a, const vint16& n) { return _mm512_srav_epi32(a, n); } - - __forceinline vint16 sll (const vint16& a, int b) { return _mm512_slli_epi32(a, b); } - __forceinline vint16 sra (const vint16& a, int b) { return _mm512_srai_epi32(a, b); } - __forceinline vint16 srl (const vint16& a, int b) { return _mm512_srli_epi32(a, b); } - - __forceinline vint16 min(const vint16& a, const vint16& b) { return _mm512_min_epi32(a, b); } - __forceinline vint16 min(const vint16& a, int b) { return min(a,vint16(b)); } - __forceinline vint16 min(int a, const vint16& b) { return min(vint16(a),b); } - - __forceinline vint16 max(const vint16& a, const vint16& b) { return _mm512_max_epi32(a, b); } - __forceinline vint16 max(const vint16& a, int b) { return max(a,vint16(b)); } - __forceinline vint16 max(int a, const vint16& b) { return max(vint16(a),b); } - - __forceinline vint16 umin(const vint16& a, const vint16& b) { return _mm512_min_epu32(a, b); } - __forceinline vint16 umax(const vint16& a, const vint16& b) { return _mm512_max_epu32(a, b); } - - __forceinline vint16 mask_add(const vboolf16& mask, vint16& c, const vint16& a, const vint16& b) { return _mm512_mask_add_epi32(c,mask,a,b); } - __forceinline vint16 mask_sub(const vboolf16& mask, vint16& c, const vint16& a, const vint16& b) { return _mm512_mask_sub_epi32(c,mask,a,b); } - - __forceinline vint16 mask_and(const vboolf16& m, vint16& c, const vint16& a, const vint16& b) { return _mm512_mask_and_epi32(c,m,a,b); } - __forceinline vint16 mask_or (const vboolf16& m, vint16& c, const vint16& a, const vint16& b) { return _mm512_mask_or_epi32(c,m,a,b); } - - //////////////////////////////////////////////////////////////////////////////// - /// Assignment Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vint16& operator +=(vint16& a, const vint16& b) { return a = a + b; } - __forceinline vint16& operator +=(vint16& a, int b) { return a = a + b; } - - __forceinline vint16& operator -=(vint16& a, const vint16& b) { return a = a - b; } - __forceinline vint16& operator -=(vint16& a, int b) { return a = a - b; } - - __forceinline vint16& operator *=(vint16& a, const vint16& b) { return a = a * b; } - __forceinline vint16& operator *=(vint16& a, int b) { return a = a * b; } - - __forceinline vint16& operator &=(vint16& a, const vint16& b) { return a = a & b; } - __forceinline vint16& operator &=(vint16& a, int b) { return a = a & b; } - - __forceinline vint16& operator |=(vint16& a, const vint16& b) { return a = a | b; } - __forceinline vint16& operator |=(vint16& a, int b) { return a = a | b; } - - __forceinline vint16& operator <<=(vint16& a, int b) { return a = a << b; } - __forceinline vint16& operator >>=(vint16& a, int b) { return a = a >> b; } - - - //////////////////////////////////////////////////////////////////////////////// - /// Comparison Operators + Select - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboolf16 operator ==(const vint16& a, const vint16& b) { return _mm512_cmp_epi32_mask(a,b,_MM_CMPINT_EQ); } - __forceinline vboolf16 operator ==(const vint16& a, int b) { return a == vint16(b); } - __forceinline vboolf16 operator ==(int a, const vint16& b) { return vint16(a) == b; } - - __forceinline vboolf16 operator !=(const vint16& a, const vint16& b) { return _mm512_cmp_epi32_mask(a,b,_MM_CMPINT_NE); } - __forceinline vboolf16 operator !=(const vint16& a, int b) { return a != vint16(b); } - __forceinline vboolf16 operator !=(int a, const vint16& b) { return vint16(a) != b; } - - __forceinline vboolf16 operator < (const vint16& a, const vint16& b) { return _mm512_cmp_epi32_mask(a,b,_MM_CMPINT_LT); } - __forceinline vboolf16 operator < (const vint16& a, int b) { return a < vint16(b); } - __forceinline vboolf16 operator < (int a, const vint16& b) { return vint16(a) < b; } - - __forceinline vboolf16 operator >=(const vint16& a, const vint16& b) { return _mm512_cmp_epi32_mask(a,b,_MM_CMPINT_GE); } - __forceinline vboolf16 operator >=(const vint16& a, int b) { return a >= vint16(b); } - __forceinline vboolf16 operator >=(int a, const vint16& b) { return vint16(a) >= b; } - - __forceinline vboolf16 operator > (const vint16& a, const vint16& b) { return _mm512_cmp_epi32_mask(a,b,_MM_CMPINT_GT); } - __forceinline vboolf16 operator > (const vint16& a, int b) { return a > vint16(b); } - __forceinline vboolf16 operator > (int a, const vint16& b) { return vint16(a) > b; } - - __forceinline vboolf16 operator <=(const vint16& a, const vint16& b) { return _mm512_cmp_epi32_mask(a,b,_MM_CMPINT_LE); } - __forceinline vboolf16 operator <=(const vint16& a, int b) { return a <= vint16(b); } - __forceinline vboolf16 operator <=(int a, const vint16& b) { return vint16(a) <= b; } - - __forceinline vboolf16 eq(const vint16& a, const vint16& b) { return _mm512_cmp_epi32_mask(a,b,_MM_CMPINT_EQ); } - __forceinline vboolf16 ne(const vint16& a, const vint16& b) { return _mm512_cmp_epi32_mask(a,b,_MM_CMPINT_NE); } - __forceinline vboolf16 lt(const vint16& a, const vint16& b) { return _mm512_cmp_epi32_mask(a,b,_MM_CMPINT_LT); } - __forceinline vboolf16 ge(const vint16& a, const vint16& b) { return _mm512_cmp_epi32_mask(a,b,_MM_CMPINT_GE); } - __forceinline vboolf16 gt(const vint16& a, const vint16& b) { return _mm512_cmp_epi32_mask(a,b,_MM_CMPINT_GT); } - __forceinline vboolf16 le(const vint16& a, const vint16& b) { return _mm512_cmp_epi32_mask(a,b,_MM_CMPINT_LE); } - __forceinline vboolf16 uint_le(const vint16& a, const vint16& b) { return _mm512_cmp_epu32_mask(a,b,_MM_CMPINT_LE); } - __forceinline vboolf16 uint_gt(const vint16& a, const vint16& b) { return _mm512_cmp_epu32_mask(a,b,_MM_CMPINT_GT); } - - __forceinline vboolf16 eq(const vboolf16 mask, const vint16& a, const vint16& b) { return _mm512_mask_cmp_epi32_mask(mask,a,b,_MM_CMPINT_EQ); } - __forceinline vboolf16 ne(const vboolf16 mask, const vint16& a, const vint16& b) { return _mm512_mask_cmp_epi32_mask(mask,a,b,_MM_CMPINT_NE); } - __forceinline vboolf16 lt(const vboolf16 mask, const vint16& a, const vint16& b) { return _mm512_mask_cmp_epi32_mask(mask,a,b,_MM_CMPINT_LT); } - __forceinline vboolf16 ge(const vboolf16 mask, const vint16& a, const vint16& b) { return _mm512_mask_cmp_epi32_mask(mask,a,b,_MM_CMPINT_GE); } - __forceinline vboolf16 gt(const vboolf16 mask, const vint16& a, const vint16& b) { return _mm512_mask_cmp_epi32_mask(mask,a,b,_MM_CMPINT_GT); } - __forceinline vboolf16 le(const vboolf16 mask, const vint16& a, const vint16& b) { return _mm512_mask_cmp_epi32_mask(mask,a,b,_MM_CMPINT_LE); } - __forceinline vboolf16 uint_le(const vboolf16 mask, const vint16& a, const vint16& b) { return _mm512_mask_cmp_epu32_mask(mask,a,b,_MM_CMPINT_LE); } - __forceinline vboolf16 uint_gt(const vboolf16 mask, const vint16& a, const vint16& b) { return _mm512_mask_cmp_epu32_mask(mask,a,b,_MM_CMPINT_GT); } - - - __forceinline vint16 select(const vboolf16& m, const vint16& t, const vint16& f) { - return _mm512_mask_or_epi32(f,m,t,t); - } - - __forceinline void xchg(const vboolf16& m, vint16& a, vint16& b) { - const vint16 c = a; a = select(m,b,a); b = select(m,c,b); - } - - __forceinline vboolf16 test(const vboolf16& m, const vint16& a, const vint16& b) { - return _mm512_mask_test_epi32_mask(m,a,b); - } - - __forceinline vboolf16 test(const vint16& a, const vint16& b) { - return _mm512_test_epi32_mask(a,b); - } - - //////////////////////////////////////////////////////////////////////////////// - // Movement/Shifting/Shuffling Functions - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vint16 unpacklo(const vint16& a, const vint16& b) { return _mm512_unpacklo_epi32(a, b); } - __forceinline vint16 unpackhi(const vint16& a, const vint16& b) { return _mm512_unpackhi_epi32(a, b); } - - template<int i> - __forceinline vint16 shuffle(const vint16& v) { - return _mm512_castps_si512(_mm512_permute_ps(_mm512_castsi512_ps(v), _MM_SHUFFLE(i, i, i, i))); - } - - template<int i0, int i1, int i2, int i3> - __forceinline vint16 shuffle(const vint16& v) { - return _mm512_castps_si512(_mm512_permute_ps(_mm512_castsi512_ps(v), _MM_SHUFFLE(i3, i2, i1, i0))); - } - - template<int i> - __forceinline vint16 shuffle4(const vint16& v) { - return _mm512_castps_si512(_mm512_shuffle_f32x4(_mm512_castsi512_ps(v), _mm512_castsi512_ps(v), _MM_SHUFFLE(i, i, i, i))); - } - - template<int i0, int i1, int i2, int i3> - __forceinline vint16 shuffle4(const vint16& v) { - return _mm512_castps_si512(_mm512_shuffle_f32x4(_mm512_castsi512_ps(v), _mm512_castsi512_ps(v), _MM_SHUFFLE(i3, i2, i1, i0))); - } - - template<int i> - __forceinline vint16 align_shift_right(const vint16& a, const vint16& b) { - return _mm512_alignr_epi32(a, b, i); - }; - - __forceinline int toScalar(const vint16& v) { - return _mm_cvtsi128_si32(_mm512_castsi512_si128(v)); - } - - template<int i> __forceinline vint16 insert4(const vint16& a, const vint4& b) { return _mm512_inserti32x4(a, b, i); } - - __forceinline size_t extract64bit(const vint16& v) { - return _mm_cvtsi128_si64(_mm512_castsi512_si128(v)); - } - - template<int N, int i> - vint<N> extractN(const vint16& v); - - template<> __forceinline vint4 extractN<4,0>(const vint16& v) { return _mm512_castsi512_si128(v); } - template<> __forceinline vint4 extractN<4,1>(const vint16& v) { return _mm512_extracti32x4_epi32(v, 1); } - template<> __forceinline vint4 extractN<4,2>(const vint16& v) { return _mm512_extracti32x4_epi32(v, 2); } - template<> __forceinline vint4 extractN<4,3>(const vint16& v) { return _mm512_extracti32x4_epi32(v, 3); } - - template<> __forceinline vint8 extractN<8,0>(const vint16& v) { return _mm512_castsi512_si256(v); } - template<> __forceinline vint8 extractN<8,1>(const vint16& v) { return _mm512_extracti32x8_epi32(v, 1); } - - template<int i> __forceinline vint4 extract4 (const vint16& v) { return _mm512_extracti32x4_epi32(v, i); } - template<> __forceinline vint4 extract4<0>(const vint16& v) { return _mm512_castsi512_si128(v); } - - template<int i> __forceinline vint8 extract8 (const vint16& v) { return _mm512_extracti32x8_epi32(v, i); } - template<> __forceinline vint8 extract8<0>(const vint16& v) { return _mm512_castsi512_si256(v); } - - //////////////////////////////////////////////////////////////////////////////// - /// Reductions - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vint16 vreduce_min2(vint16 x) { return min(x, shuffle<1,0,3,2>(x)); } - __forceinline vint16 vreduce_min4(vint16 x) { x = vreduce_min2(x); return min(x, shuffle<2,3,0,1>(x)); } - __forceinline vint16 vreduce_min8(vint16 x) { x = vreduce_min4(x); return min(x, shuffle4<1,0,3,2>(x)); } - __forceinline vint16 vreduce_min (vint16 x) { x = vreduce_min8(x); return min(x, shuffle4<2,3,0,1>(x)); } - - __forceinline vint16 vreduce_max2(vint16 x) { return max(x, shuffle<1,0,3,2>(x)); } - __forceinline vint16 vreduce_max4(vint16 x) { x = vreduce_max2(x); return max(x, shuffle<2,3,0,1>(x)); } - __forceinline vint16 vreduce_max8(vint16 x) { x = vreduce_max4(x); return max(x, shuffle4<1,0,3,2>(x)); } - __forceinline vint16 vreduce_max (vint16 x) { x = vreduce_max8(x); return max(x, shuffle4<2,3,0,1>(x)); } - - __forceinline vint16 vreduce_and2(vint16 x) { return x & shuffle<1,0,3,2>(x); } - __forceinline vint16 vreduce_and4(vint16 x) { x = vreduce_and2(x); return x & shuffle<2,3,0,1>(x); } - __forceinline vint16 vreduce_and8(vint16 x) { x = vreduce_and4(x); return x & shuffle4<1,0,3,2>(x); } - __forceinline vint16 vreduce_and (vint16 x) { x = vreduce_and8(x); return x & shuffle4<2,3,0,1>(x); } - - __forceinline vint16 vreduce_or2(vint16 x) { return x | shuffle<1,0,3,2>(x); } - __forceinline vint16 vreduce_or4(vint16 x) { x = vreduce_or2(x); return x | shuffle<2,3,0,1>(x); } - __forceinline vint16 vreduce_or8(vint16 x) { x = vreduce_or4(x); return x | shuffle4<1,0,3,2>(x); } - __forceinline vint16 vreduce_or (vint16 x) { x = vreduce_or8(x); return x | shuffle4<2,3,0,1>(x); } - - __forceinline vint16 vreduce_add2(vint16 x) { return x + shuffle<1,0,3,2>(x); } - __forceinline vint16 vreduce_add4(vint16 x) { x = vreduce_add2(x); return x + shuffle<2,3,0,1>(x); } - __forceinline vint16 vreduce_add8(vint16 x) { x = vreduce_add4(x); return x + shuffle4<1,0,3,2>(x); } - __forceinline vint16 vreduce_add (vint16 x) { x = vreduce_add8(x); return x + shuffle4<2,3,0,1>(x); } - - __forceinline int reduce_min(const vint16& v) { return toScalar(vreduce_min(v)); } - __forceinline int reduce_max(const vint16& v) { return toScalar(vreduce_max(v)); } - __forceinline int reduce_and(const vint16& v) { return toScalar(vreduce_and(v)); } - __forceinline int reduce_or (const vint16& v) { return toScalar(vreduce_or (v)); } - __forceinline int reduce_add(const vint16& v) { return toScalar(vreduce_add(v)); } - - //////////////////////////////////////////////////////////////////////////////// - /// Memory load and store operations - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vint16 conflict(const vint16& index) - { - return _mm512_conflict_epi32(index); - } - - __forceinline vint16 conflict(const vboolf16& mask, vint16& dest, const vint16& index) - { - return _mm512_mask_conflict_epi32(dest,mask,index); - } - - __forceinline vint16 convert_uint32_t(const __m512& f) { - return _mm512_cvtps_epu32(f); - } - - __forceinline vint16 permute(vint16 v, vint16 index) { - return _mm512_permutexvar_epi32(index,v); - } - - __forceinline vint16 reverse(const vint16 &a) { - return permute(a,vint16(reverse_step)); - } - - __forceinline vint16 prefix_sum(const vint16& a) - { - const vint16 z(zero); - vint16 v = a; - v = v + align_shift_right<16-1>(v,z); - v = v + align_shift_right<16-2>(v,z); - v = v + align_shift_right<16-4>(v,z); - v = v + align_shift_right<16-8>(v,z); - return v; - } - - __forceinline vint16 reverse_prefix_sum(const vint16& a) - { - const vint16 z(zero); - vint16 v = a; - v = v + align_shift_right<1>(z,v); - v = v + align_shift_right<2>(z,v); - v = v + align_shift_right<4>(z,v); - v = v + align_shift_right<8>(z,v); - return v; - } - - /* this should use a vbool8 and a vint8_64...*/ - template<int scale = 1, int hint = _MM_HINT_T0> - __forceinline void gather_prefetch64(const void* base_addr, const vbool16& mask, const vint16& offset) - { -#if defined(__AVX512PF__) - _mm512_mask_prefetch_i64gather_pd(offset, mask, base_addr, scale, hint); -#endif - } - - //////////////////////////////////////////////////////////////////////////////// - /// Output Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline embree_ostream operator <<(embree_ostream cout, const vint16& v) - { - cout << "<" << v[0]; - for (int i=1; i<16; i++) cout << ", " << v[i]; - cout << ">"; - return cout; - } -} diff --git a/thirdparty/embree-aarch64/common/simd/vint4_sse2.h b/thirdparty/embree-aarch64/common/simd/vint4_sse2.h deleted file mode 100644 index 96f105a7c5..0000000000 --- a/thirdparty/embree-aarch64/common/simd/vint4_sse2.h +++ /dev/null @@ -1,681 +0,0 @@ -// Copyright 2009-2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 - -#pragma once - -#include "../math/math.h" - -namespace embree -{ - /* 4-wide SSE integer type */ - template<> - struct vint<4> - { - ALIGNED_STRUCT_(16); - - typedef vboolf4 Bool; - typedef vint4 Int; - typedef vfloat4 Float; - - enum { size = 4 }; // number of SIMD elements - union { __m128i v; int i[4]; }; // data - - //////////////////////////////////////////////////////////////////////////////// - /// Constructors, Assignment & Cast Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vint() {} - __forceinline vint(const vint4& a) { v = a.v; } - __forceinline vint4& operator =(const vint4& a) { v = a.v; return *this; } - - __forceinline vint(__m128i a) : v(a) {} - __forceinline operator const __m128i&() const { return v; } - __forceinline operator __m128i&() { return v; } - - __forceinline vint(int a) : v(_mm_set1_epi32(a)) {} - __forceinline vint(int a, int b, int c, int d) : v(_mm_set_epi32(d, c, b, a)) {} - - __forceinline explicit vint(__m128 a) : v(_mm_cvtps_epi32(a)) {} -#if defined(__AVX512VL__) - __forceinline explicit vint(const vboolf4& a) : v(_mm_movm_epi32(a)) {} -#else - __forceinline explicit vint(const vboolf4& a) : v(_mm_castps_si128((__m128)a)) {} -#endif - - __forceinline vint(long long a, long long b) : v(_mm_set_epi64x(b,a)) {} - - //////////////////////////////////////////////////////////////////////////////// - /// Constants - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vint(ZeroTy) : v(_mm_setzero_si128()) {} - __forceinline vint(OneTy) : v(_mm_set_epi32(1, 1, 1, 1)) {} - __forceinline vint(PosInfTy) : v(_mm_set_epi32(pos_inf, pos_inf, pos_inf, pos_inf)) {} - __forceinline vint(NegInfTy) : v(_mm_set_epi32(neg_inf, neg_inf, neg_inf, neg_inf)) {} - __forceinline vint(StepTy) : v(_mm_set_epi32(3, 2, 1, 0)) {} - __forceinline vint(ReverseStepTy) : v(_mm_set_epi32(0, 1, 2, 3)) {} - - __forceinline vint(TrueTy) { v = _mm_cmpeq_epi32(v,v); } - __forceinline vint(UndefinedTy) : v(_mm_castps_si128(_mm_undefined_ps())) {} - - - //////////////////////////////////////////////////////////////////////////////// - /// Loads and Stores - //////////////////////////////////////////////////////////////////////////////// - - static __forceinline vint4 load (const void* a) { return _mm_load_si128((__m128i*)a); } - static __forceinline vint4 loadu(const void* a) { return _mm_loadu_si128((__m128i*)a); } - - static __forceinline void store (void* ptr, const vint4& v) { _mm_store_si128((__m128i*)ptr,v); } - static __forceinline void storeu(void* ptr, const vint4& v) { _mm_storeu_si128((__m128i*)ptr,v); } - -#if defined(__AVX512VL__) - - static __forceinline vint4 compact(const vboolf4& mask, vint4 &v) { - return _mm_mask_compress_epi32(v, mask, v); - } - static __forceinline vint4 compact(const vboolf4& mask, vint4 &a, const vint4& b) { - return _mm_mask_compress_epi32(a, mask, b); - } - - static __forceinline vint4 load (const vboolf4& mask, const void* ptr) { return _mm_mask_load_epi32 (_mm_setzero_si128(),mask,ptr); } - static __forceinline vint4 loadu(const vboolf4& mask, const void* ptr) { return _mm_mask_loadu_epi32(_mm_setzero_si128(),mask,ptr); } - - static __forceinline void store (const vboolf4& mask, void* ptr, const vint4& v) { _mm_mask_store_epi32 (ptr,mask,v); } - static __forceinline void storeu(const vboolf4& mask, void* ptr, const vint4& v) { _mm_mask_storeu_epi32(ptr,mask,v); } -#elif defined(__AVX__) - static __forceinline vint4 load (const vbool4& mask, const void* a) { return _mm_castps_si128(_mm_maskload_ps((float*)a,mask)); } - static __forceinline vint4 loadu(const vbool4& mask, const void* a) { return _mm_castps_si128(_mm_maskload_ps((float*)a,mask)); } - - static __forceinline void store (const vboolf4& mask, void* ptr, const vint4& i) { _mm_maskstore_ps((float*)ptr,(__m128i)mask,_mm_castsi128_ps(i)); } - static __forceinline void storeu(const vboolf4& mask, void* ptr, const vint4& i) { _mm_maskstore_ps((float*)ptr,(__m128i)mask,_mm_castsi128_ps(i)); } -#else - static __forceinline vint4 load (const vbool4& mask, const void* a) { return _mm_and_si128(_mm_load_si128 ((__m128i*)a),mask); } - static __forceinline vint4 loadu(const vbool4& mask, const void* a) { return _mm_and_si128(_mm_loadu_si128((__m128i*)a),mask); } - - static __forceinline void store (const vboolf4& mask, void* ptr, const vint4& i) { store (ptr,select(mask,i,load (ptr))); } - static __forceinline void storeu(const vboolf4& mask, void* ptr, const vint4& i) { storeu(ptr,select(mask,i,loadu(ptr))); } -#endif - - -#if defined(__aarch64__) - static __forceinline vint4 load(const uint8_t* ptr) { - return _mm_load4epu8_epi32(((__m128i*)ptr)); - } - static __forceinline vint4 loadu(const uint8_t* ptr) { - return _mm_load4epu8_epi32(((__m128i*)ptr)); - } -#elif defined(__SSE4_1__) - static __forceinline vint4 load(const uint8_t* ptr) { - return _mm_cvtepu8_epi32(_mm_loadl_epi64((__m128i*)ptr)); - } - - static __forceinline vint4 loadu(const uint8_t* ptr) { - return _mm_cvtepu8_epi32(_mm_loadl_epi64((__m128i*)ptr)); - } -#else - - static __forceinline vint4 load(const uint8_t* ptr) { - return vint4(ptr[0],ptr[1],ptr[2],ptr[3]); - } - - static __forceinline vint4 loadu(const uint8_t* ptr) { - return vint4(ptr[0],ptr[1],ptr[2],ptr[3]); - } - -#endif - - static __forceinline vint4 load(const unsigned short* ptr) { -#if defined(__aarch64__) - return __m128i(vmovl_u16(vld1_u16(ptr))); -#elif defined (__SSE4_1__) - return _mm_cvtepu16_epi32(_mm_loadu_si128((__m128i*)ptr)); -#else - return vint4(ptr[0],ptr[1],ptr[2],ptr[3]); -#endif - } - - static __forceinline void store(uint8_t* ptr, const vint4& v) { -#if defined(__aarch64__) - int32x4_t x = v; - uint16x4_t y = vqmovn_u32(uint32x4_t(x)); - uint8x8_t z = vqmovn_u16(vcombine_u16(y, y)); - vst1_lane_u32((uint32_t *)ptr,uint32x2_t(z), 0); -#elif defined(__SSE4_1__) - __m128i x = v; - x = _mm_packus_epi32(x, x); - x = _mm_packus_epi16(x, x); - *(int*)ptr = _mm_cvtsi128_si32(x); -#else - for (size_t i=0;i<4;i++) - ptr[i] = (uint8_t)v[i]; -#endif - } - - static __forceinline void store(unsigned short* ptr, const vint4& v) { -#if defined(__aarch64__) - uint32x4_t x = uint32x4_t(v.v); - uint16x4_t y = vqmovn_u32(x); - vst1_u16(ptr, y); -#else - for (size_t i=0;i<4;i++) - ptr[i] = (unsigned short)v[i]; -#endif - } - - static __forceinline vint4 load_nt(void* ptr) { -#if defined(__aarch64__) || defined(__SSE4_1__) - return _mm_stream_load_si128((__m128i*)ptr); -#else - return _mm_load_si128((__m128i*)ptr); -#endif - } - - static __forceinline void store_nt(void* ptr, const vint4& v) { -#if !defined(__aarch64__) && defined(__SSE4_1__) - _mm_stream_ps((float*)ptr, _mm_castsi128_ps(v)); -#else - _mm_store_si128((__m128i*)ptr,v); -#endif - } - - template<int scale = 4> - static __forceinline vint4 gather(const int* ptr, const vint4& index) { -#if defined(__AVX2__) && !defined(__aarch64__) - return _mm_i32gather_epi32(ptr, index, scale); -#else - return vint4( - *(int*)(((int8_t*)ptr)+scale*index[0]), - *(int*)(((int8_t*)ptr)+scale*index[1]), - *(int*)(((int8_t*)ptr)+scale*index[2]), - *(int*)(((int8_t*)ptr)+scale*index[3])); -#endif - } - - template<int scale = 4> - static __forceinline vint4 gather(const vboolf4& mask, const int* ptr, const vint4& index) { - vint4 r = zero; -#if defined(__AVX512VL__) - return _mm_mmask_i32gather_epi32(r, mask, index, ptr, scale); -#elif defined(__AVX2__) && !defined(__aarch64__) - return _mm_mask_i32gather_epi32(r, ptr, index, mask, scale); -#else - if (likely(mask[0])) r[0] = *(int*)(((int8_t*)ptr)+scale*index[0]); - if (likely(mask[1])) r[1] = *(int*)(((int8_t*)ptr)+scale*index[1]); - if (likely(mask[2])) r[2] = *(int*)(((int8_t*)ptr)+scale*index[2]); - if (likely(mask[3])) r[3] = *(int*)(((int8_t*)ptr)+scale*index[3]); - return r; -#endif - } - - template<int scale = 4> - static __forceinline void scatter(void* ptr, const vint4& index, const vint4& v) - { -#if defined(__AVX512VL__) - _mm_i32scatter_epi32((int*)ptr, index, v, scale); -#else - *(int*)(((int8_t*)ptr)+scale*index[0]) = v[0]; - *(int*)(((int8_t*)ptr)+scale*index[1]) = v[1]; - *(int*)(((int8_t*)ptr)+scale*index[2]) = v[2]; - *(int*)(((int8_t*)ptr)+scale*index[3]) = v[3]; -#endif - } - - template<int scale = 4> - static __forceinline void scatter(const vboolf4& mask, void* ptr, const vint4& index, const vint4& v) - { -#if defined(__AVX512VL__) - _mm_mask_i32scatter_epi32((int*)ptr, mask, index, v, scale); -#else - if (likely(mask[0])) *(int*)(((int8_t*)ptr)+scale*index[0]) = v[0]; - if (likely(mask[1])) *(int*)(((int8_t*)ptr)+scale*index[1]) = v[1]; - if (likely(mask[2])) *(int*)(((int8_t*)ptr)+scale*index[2]) = v[2]; - if (likely(mask[3])) *(int*)(((int8_t*)ptr)+scale*index[3]) = v[3]; -#endif - } - -#if defined(__x86_64__) || defined(__aarch64__) - static __forceinline vint4 broadcast64(long long a) { return _mm_set1_epi64x(a); } -#endif - - //////////////////////////////////////////////////////////////////////////////// - /// Array Access - //////////////////////////////////////////////////////////////////////////////// - - __forceinline const int& operator [](size_t index) const { assert(index < 4); return i[index]; } - __forceinline int& operator [](size_t index) { assert(index < 4); return i[index]; } - - friend __forceinline vint4 select(const vboolf4& m, const vint4& t, const vint4& f) { -#if defined(__AVX512VL__) - return _mm_mask_blend_epi32(m, (__m128i)f, (__m128i)t); -#elif defined(__aarch64__) - return _mm_castps_si128(_mm_blendv_ps((__m128)f.v,(__m128) t.v, (__m128)m.v)); -#elif defined(__SSE4_1__) - return _mm_castps_si128(_mm_blendv_ps(_mm_castsi128_ps(f), _mm_castsi128_ps(t), m)); -#else - return _mm_or_si128(_mm_and_si128(m, t), _mm_andnot_si128(m, f)); -#endif - } - }; - - //////////////////////////////////////////////////////////////////////////////// - /// Unary Operators - //////////////////////////////////////////////////////////////////////////////// - -#if defined(__AVX512VL__) - __forceinline vboolf4 asBool(const vint4& a) { return _mm_movepi32_mask(a); } -#else - __forceinline vboolf4 asBool(const vint4& a) { return _mm_castsi128_ps(a); } -#endif - - __forceinline vint4 operator +(const vint4& a) { return a; } - __forceinline vint4 operator -(const vint4& a) { return _mm_sub_epi32(_mm_setzero_si128(), a); } -#if defined(__aarch64__) - __forceinline vint4 abs(const vint4& a) { return vabsq_s32(a.v); } -#elif defined(__SSSE3__) - __forceinline vint4 abs(const vint4& a) { return _mm_abs_epi32(a); } -#endif - - //////////////////////////////////////////////////////////////////////////////// - /// Binary Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vint4 operator +(const vint4& a, const vint4& b) { return _mm_add_epi32(a, b); } - __forceinline vint4 operator +(const vint4& a, int b) { return a + vint4(b); } - __forceinline vint4 operator +(int a, const vint4& b) { return vint4(a) + b; } - - __forceinline vint4 operator -(const vint4& a, const vint4& b) { return _mm_sub_epi32(a, b); } - __forceinline vint4 operator -(const vint4& a, int b) { return a - vint4(b); } - __forceinline vint4 operator -(int a, const vint4& b) { return vint4(a) - b; } - -#if (defined(__aarch64__)) || defined(__SSE4_1__) - __forceinline vint4 operator *(const vint4& a, const vint4& b) { return _mm_mullo_epi32(a, b); } -#else - __forceinline vint4 operator *(const vint4& a, const vint4& b) { return vint4(a[0]*b[0],a[1]*b[1],a[2]*b[2],a[3]*b[3]); } -#endif - __forceinline vint4 operator *(const vint4& a, int b) { return a * vint4(b); } - __forceinline vint4 operator *(int a, const vint4& b) { return vint4(a) * b; } - - __forceinline vint4 operator &(const vint4& a, const vint4& b) { return _mm_and_si128(a, b); } - __forceinline vint4 operator &(const vint4& a, int b) { return a & vint4(b); } - __forceinline vint4 operator &(int a, const vint4& b) { return vint4(a) & b; } - - __forceinline vint4 operator |(const vint4& a, const vint4& b) { return _mm_or_si128(a, b); } - __forceinline vint4 operator |(const vint4& a, int b) { return a | vint4(b); } - __forceinline vint4 operator |(int a, const vint4& b) { return vint4(a) | b; } - - __forceinline vint4 operator ^(const vint4& a, const vint4& b) { return _mm_xor_si128(a, b); } - __forceinline vint4 operator ^(const vint4& a, int b) { return a ^ vint4(b); } - __forceinline vint4 operator ^(int a, const vint4& b) { return vint4(a) ^ b; } - - __forceinline vint4 operator <<(const vint4& a, const int n) { return _mm_slli_epi32(a, n); } - __forceinline vint4 operator >>(const vint4& a, const int n) { return _mm_srai_epi32(a, n); } - - __forceinline vint4 sll (const vint4& a, int b) { return _mm_slli_epi32(a, b); } - __forceinline vint4 sra (const vint4& a, int b) { return _mm_srai_epi32(a, b); } - __forceinline vint4 srl (const vint4& a, int b) { return _mm_srli_epi32(a, b); } - - //////////////////////////////////////////////////////////////////////////////// - /// Assignment Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vint4& operator +=(vint4& a, const vint4& b) { return a = a + b; } - __forceinline vint4& operator +=(vint4& a, int b) { return a = a + b; } - - __forceinline vint4& operator -=(vint4& a, const vint4& b) { return a = a - b; } - __forceinline vint4& operator -=(vint4& a, int b) { return a = a - b; } - -#if (defined(__aarch64__)) || defined(__SSE4_1__) - __forceinline vint4& operator *=(vint4& a, const vint4& b) { return a = a * b; } - __forceinline vint4& operator *=(vint4& a, int b) { return a = a * b; } -#endif - - __forceinline vint4& operator &=(vint4& a, const vint4& b) { return a = a & b; } - __forceinline vint4& operator &=(vint4& a, int b) { return a = a & b; } - - __forceinline vint4& operator |=(vint4& a, const vint4& b) { return a = a | b; } - __forceinline vint4& operator |=(vint4& a, int b) { return a = a | b; } - - __forceinline vint4& operator <<=(vint4& a, int b) { return a = a << b; } - __forceinline vint4& operator >>=(vint4& a, int b) { return a = a >> b; } - - //////////////////////////////////////////////////////////////////////////////// - /// Comparison Operators + Select - //////////////////////////////////////////////////////////////////////////////// - -#if defined(__AVX512VL__) - __forceinline vboolf4 operator ==(const vint4& a, const vint4& b) { return _mm_cmp_epi32_mask(a,b,_MM_CMPINT_EQ); } - __forceinline vboolf4 operator !=(const vint4& a, const vint4& b) { return _mm_cmp_epi32_mask(a,b,_MM_CMPINT_NE); } - __forceinline vboolf4 operator < (const vint4& a, const vint4& b) { return _mm_cmp_epi32_mask(a,b,_MM_CMPINT_LT); } - __forceinline vboolf4 operator >=(const vint4& a, const vint4& b) { return _mm_cmp_epi32_mask(a,b,_MM_CMPINT_GE); } - __forceinline vboolf4 operator > (const vint4& a, const vint4& b) { return _mm_cmp_epi32_mask(a,b,_MM_CMPINT_GT); } - __forceinline vboolf4 operator <=(const vint4& a, const vint4& b) { return _mm_cmp_epi32_mask(a,b,_MM_CMPINT_LE); } -#else - __forceinline vboolf4 operator ==(const vint4& a, const vint4& b) { return _mm_castsi128_ps(_mm_cmpeq_epi32(a, b)); } - __forceinline vboolf4 operator !=(const vint4& a, const vint4& b) { return !(a == b); } - __forceinline vboolf4 operator < (const vint4& a, const vint4& b) { return _mm_castsi128_ps(_mm_cmplt_epi32(a, b)); } - __forceinline vboolf4 operator >=(const vint4& a, const vint4& b) { return !(a < b); } - __forceinline vboolf4 operator > (const vint4& a, const vint4& b) { return _mm_castsi128_ps(_mm_cmpgt_epi32(a, b)); } - __forceinline vboolf4 operator <=(const vint4& a, const vint4& b) { return !(a > b); } -#endif - - __forceinline vboolf4 operator ==(const vint4& a, int b) { return a == vint4(b); } - __forceinline vboolf4 operator ==(int a, const vint4& b) { return vint4(a) == b; } - - __forceinline vboolf4 operator !=(const vint4& a, int b) { return a != vint4(b); } - __forceinline vboolf4 operator !=(int a, const vint4& b) { return vint4(a) != b; } - - __forceinline vboolf4 operator < (const vint4& a, int b) { return a < vint4(b); } - __forceinline vboolf4 operator < (int a, const vint4& b) { return vint4(a) < b; } - - __forceinline vboolf4 operator >=(const vint4& a, int b) { return a >= vint4(b); } - __forceinline vboolf4 operator >=(int a, const vint4& b) { return vint4(a) >= b; } - - __forceinline vboolf4 operator > (const vint4& a, int b) { return a > vint4(b); } - __forceinline vboolf4 operator > (int a, const vint4& b) { return vint4(a) > b; } - - __forceinline vboolf4 operator <=(const vint4& a, int b) { return a <= vint4(b); } - __forceinline vboolf4 operator <=(int a, const vint4& b) { return vint4(a) <= b; } - - __forceinline vboolf4 eq(const vint4& a, const vint4& b) { return a == b; } - __forceinline vboolf4 ne(const vint4& a, const vint4& b) { return a != b; } - __forceinline vboolf4 lt(const vint4& a, const vint4& b) { return a < b; } - __forceinline vboolf4 ge(const vint4& a, const vint4& b) { return a >= b; } - __forceinline vboolf4 gt(const vint4& a, const vint4& b) { return a > b; } - __forceinline vboolf4 le(const vint4& a, const vint4& b) { return a <= b; } - -#if defined(__AVX512VL__) - __forceinline vboolf4 eq(const vboolf4& mask, const vint4& a, const vint4& b) { return _mm_mask_cmp_epi32_mask(mask, a, b, _MM_CMPINT_EQ); } - __forceinline vboolf4 ne(const vboolf4& mask, const vint4& a, const vint4& b) { return _mm_mask_cmp_epi32_mask(mask, a, b, _MM_CMPINT_NE); } - __forceinline vboolf4 lt(const vboolf4& mask, const vint4& a, const vint4& b) { return _mm_mask_cmp_epi32_mask(mask, a, b, _MM_CMPINT_LT); } - __forceinline vboolf4 ge(const vboolf4& mask, const vint4& a, const vint4& b) { return _mm_mask_cmp_epi32_mask(mask, a, b, _MM_CMPINT_GE); } - __forceinline vboolf4 gt(const vboolf4& mask, const vint4& a, const vint4& b) { return _mm_mask_cmp_epi32_mask(mask, a, b, _MM_CMPINT_GT); } - __forceinline vboolf4 le(const vboolf4& mask, const vint4& a, const vint4& b) { return _mm_mask_cmp_epi32_mask(mask, a, b, _MM_CMPINT_LE); } -#else - __forceinline vboolf4 eq(const vboolf4& mask, const vint4& a, const vint4& b) { return mask & (a == b); } - __forceinline vboolf4 ne(const vboolf4& mask, const vint4& a, const vint4& b) { return mask & (a != b); } - __forceinline vboolf4 lt(const vboolf4& mask, const vint4& a, const vint4& b) { return mask & (a < b); } - __forceinline vboolf4 ge(const vboolf4& mask, const vint4& a, const vint4& b) { return mask & (a >= b); } - __forceinline vboolf4 gt(const vboolf4& mask, const vint4& a, const vint4& b) { return mask & (a > b); } - __forceinline vboolf4 le(const vboolf4& mask, const vint4& a, const vint4& b) { return mask & (a <= b); } -#endif - - template<int mask> - __forceinline vint4 select(const vint4& t, const vint4& f) { -#if defined(__SSE4_1__) - return _mm_castps_si128(_mm_blend_ps(_mm_castsi128_ps(f), _mm_castsi128_ps(t), mask)); -#else - return select(vboolf4(mask), t, f); -#endif - } - - -#if defined(__aarch64__) || defined(__SSE4_1__) - __forceinline vint4 min(const vint4& a, const vint4& b) { return _mm_min_epi32(a, b); } - __forceinline vint4 max(const vint4& a, const vint4& b) { return _mm_max_epi32(a, b); } - - __forceinline vint4 umin(const vint4& a, const vint4& b) { return _mm_min_epu32(a, b); } - __forceinline vint4 umax(const vint4& a, const vint4& b) { return _mm_max_epu32(a, b); } - -#else - __forceinline vint4 min(const vint4& a, const vint4& b) { return select(a < b,a,b); } - __forceinline vint4 max(const vint4& a, const vint4& b) { return select(a < b,b,a); } -#endif - - __forceinline vint4 min(const vint4& a, int b) { return min(a,vint4(b)); } - __forceinline vint4 min(int a, const vint4& b) { return min(vint4(a),b); } - __forceinline vint4 max(const vint4& a, int b) { return max(a,vint4(b)); } - __forceinline vint4 max(int a, const vint4& b) { return max(vint4(a),b); } - - //////////////////////////////////////////////////////////////////////////////// - // Movement/Shifting/Shuffling Functions - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vint4 unpacklo(const vint4& a, const vint4& b) { return _mm_castps_si128(_mm_unpacklo_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b))); } - __forceinline vint4 unpackhi(const vint4& a, const vint4& b) { return _mm_castps_si128(_mm_unpackhi_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b))); } - -#if defined(__aarch64__) - template<int i0, int i1, int i2, int i3> - __forceinline vint4 shuffle(const vint4& v) { - return vreinterpretq_s32_u8(vqtbl1q_u8( (uint8x16_t)v.v, _MN_SHUFFLE(i0, i1, i2, i3))); - } - template<int i0, int i1, int i2, int i3> - __forceinline vint4 shuffle(const vint4& a, const vint4& b) { - return vreinterpretq_s32_u8(vqtbl2q_u8( (uint8x16x2_t){(uint8x16_t)a.v, (uint8x16_t)b.v}, _MF_SHUFFLE(i0, i1, i2, i3))); - } -#else - template<int i0, int i1, int i2, int i3> - __forceinline vint4 shuffle(const vint4& v) { - return _mm_shuffle_epi32(v, _MM_SHUFFLE(i3, i2, i1, i0)); - } - template<int i0, int i1, int i2, int i3> - __forceinline vint4 shuffle(const vint4& a, const vint4& b) { - return _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b), _MM_SHUFFLE(i3, i2, i1, i0))); - } -#endif -#if defined(__SSE3__) - template<> __forceinline vint4 shuffle<0, 0, 2, 2>(const vint4& v) { return _mm_castps_si128(_mm_moveldup_ps(_mm_castsi128_ps(v))); } - template<> __forceinline vint4 shuffle<1, 1, 3, 3>(const vint4& v) { return _mm_castps_si128(_mm_movehdup_ps(_mm_castsi128_ps(v))); } - template<> __forceinline vint4 shuffle<0, 1, 0, 1>(const vint4& v) { return _mm_castpd_si128(_mm_movedup_pd (_mm_castsi128_pd(v))); } -#endif - - template<int i> - __forceinline vint4 shuffle(const vint4& v) { - return shuffle<i,i,i,i>(v); - } - -#if defined(__aarch64__) - template<int src> __forceinline int extract(const vint4& b); - template<int dst> __forceinline vint4 insert(const vint4& a, const int b); -#elif defined(__SSE4_1__) - template<int src> __forceinline int extract(const vint4& b) { return _mm_extract_epi32(b, src); } - template<int dst> __forceinline vint4 insert(const vint4& a, const int b) { return _mm_insert_epi32(a, b, dst); } -#else - template<int src> __forceinline int extract(const vint4& b) { return b[src&3]; } - template<int dst> __forceinline vint4 insert(const vint4& a, int b) { vint4 c = a; c[dst&3] = b; return c; } -#endif - -#if defined(__aarch64__) - template<> __forceinline int extract<0>(const vint4& b) { - return b.v[0]; - } - template<> __forceinline int extract<1>(const vint4& b) { - return b.v[1]; - } - template<> __forceinline int extract<2>(const vint4& b) { - return b.v[2]; - } - template<> __forceinline int extract<3>(const vint4& b) { - return b.v[3]; - } - template<> __forceinline vint4 insert<0>(const vint4& a, int b) - { - vint4 c = a; - c[0] = b; - return c; - } - template<> __forceinline vint4 insert<1>(const vint4& a, int b) - { - vint4 c = a; - c[1] = b; - return c; - } - template<> __forceinline vint4 insert<2>(const vint4& a, int b) - { - vint4 c = a; - c[2] = b; - return c; - } - template<> __forceinline vint4 insert<3>(const vint4& a, int b) - { - vint4 c = a; - c[3] = b; - return c; - } - - __forceinline int toScalar(const vint4& v) { - return v[0]; - } - - __forceinline size_t toSizeT(const vint4& v) { - uint64x2_t x = uint64x2_t(v.v); - return x[0]; - } -#else - template<> __forceinline int extract<0>(const vint4& b) { return _mm_cvtsi128_si32(b); } - - __forceinline int toScalar(const vint4& v) { return _mm_cvtsi128_si32(v); } - - __forceinline size_t toSizeT(const vint4& v) { -#if defined(__WIN32__) && !defined(__X86_64__) // win32 workaround - return toScalar(v); -#elif defined(__ARM_NEON) - // FIXME(LTE): Do we need a swap(i.e. use lane 1)? - return vgetq_lane_u64(*(reinterpret_cast<const uint64x2_t *>(&v)), 0); -#else - return _mm_cvtsi128_si64(v); -#endif - } -#endif - -#if defined(__AVX512VL__) - - __forceinline vint4 permute(const vint4 &a, const vint4 &index) { - return _mm_castps_si128(_mm_permutevar_ps(_mm_castsi128_ps(a),index)); - } - - template<int i> - __forceinline vint4 align_shift_right(const vint4& a, const vint4& b) { - return _mm_alignr_epi32(a, b, i); - } -#endif - - //////////////////////////////////////////////////////////////////////////////// - /// Reductions - //////////////////////////////////////////////////////////////////////////////// - -#if defined(__aarch64__) || defined(__SSE4_1__) - -#if defined(__aarch64__) - __forceinline vint4 vreduce_min(const vint4& v) { int h = vminvq_s32(v); return vdupq_n_s32(h); } - __forceinline vint4 vreduce_max(const vint4& v) { int h = vmaxvq_s32(v); return vdupq_n_s32(h); } - __forceinline vint4 vreduce_add(const vint4& v) { int h = vaddvq_s32(v); return vdupq_n_s32(h); } - - __forceinline int reduce_min(const vint4& v) { return vminvq_s32(v); } - __forceinline int reduce_max(const vint4& v) { return vmaxvq_s32(v); } - __forceinline int reduce_add(const vint4& v) { return vaddvq_s32(v); } -#else - __forceinline vint4 vreduce_min(const vint4& v) { vint4 h = min(shuffle<1,0,3,2>(v),v); return min(shuffle<2,3,0,1>(h),h); } - __forceinline vint4 vreduce_max(const vint4& v) { vint4 h = max(shuffle<1,0,3,2>(v),v); return max(shuffle<2,3,0,1>(h),h); } - __forceinline vint4 vreduce_add(const vint4& v) { vint4 h = shuffle<1,0,3,2>(v) + v ; return shuffle<2,3,0,1>(h) + h ; } - - __forceinline int reduce_min(const vint4& v) { return toScalar(vreduce_min(v)); } - __forceinline int reduce_max(const vint4& v) { return toScalar(vreduce_max(v)); } - __forceinline int reduce_add(const vint4& v) { return toScalar(vreduce_add(v)); } -#endif - - __forceinline size_t select_min(const vint4& v) { return bsf(movemask(v == vreduce_min(v))); } - __forceinline size_t select_max(const vint4& v) { return bsf(movemask(v == vreduce_max(v))); } - - __forceinline size_t select_min(const vboolf4& valid, const vint4& v) { const vint4 a = select(valid,v,vint4(pos_inf)); return bsf(movemask(valid & (a == vreduce_min(a)))); } - __forceinline size_t select_max(const vboolf4& valid, const vint4& v) { const vint4 a = select(valid,v,vint4(neg_inf)); return bsf(movemask(valid & (a == vreduce_max(a)))); } - -#else - - __forceinline int reduce_min(const vint4& v) { return min(v[0],v[1],v[2],v[3]); } - __forceinline int reduce_max(const vint4& v) { return max(v[0],v[1],v[2],v[3]); } - __forceinline int reduce_add(const vint4& v) { return v[0]+v[1]+v[2]+v[3]; } - -#endif - - //////////////////////////////////////////////////////////////////////////////// - /// Sorting networks - //////////////////////////////////////////////////////////////////////////////// - -#if (defined(__aarch64__)) || defined(__SSE4_1__) - - __forceinline vint4 usort_ascending(const vint4& v) - { - const vint4 a0 = v; - const vint4 b0 = shuffle<1,0,3,2>(a0); - const vint4 c0 = umin(a0,b0); - const vint4 d0 = umax(a0,b0); - const vint4 a1 = select<0x5 /* 0b0101 */>(c0,d0); - const vint4 b1 = shuffle<2,3,0,1>(a1); - const vint4 c1 = umin(a1,b1); - const vint4 d1 = umax(a1,b1); - const vint4 a2 = select<0x3 /* 0b0011 */>(c1,d1); - const vint4 b2 = shuffle<0,2,1,3>(a2); - const vint4 c2 = umin(a2,b2); - const vint4 d2 = umax(a2,b2); - const vint4 a3 = select<0x2 /* 0b0010 */>(c2,d2); - return a3; - } - - __forceinline vint4 usort_descending(const vint4& v) - { - const vint4 a0 = v; - const vint4 b0 = shuffle<1,0,3,2>(a0); - const vint4 c0 = umax(a0,b0); - const vint4 d0 = umin(a0,b0); - const vint4 a1 = select<0x5 /* 0b0101 */>(c0,d0); - const vint4 b1 = shuffle<2,3,0,1>(a1); - const vint4 c1 = umax(a1,b1); - const vint4 d1 = umin(a1,b1); - const vint4 a2 = select<0x3 /* 0b0011 */>(c1,d1); - const vint4 b2 = shuffle<0,2,1,3>(a2); - const vint4 c2 = umax(a2,b2); - const vint4 d2 = umin(a2,b2); - const vint4 a3 = select<0x2 /* 0b0010 */>(c2,d2); - return a3; - } - -#else - - __forceinline vint4 usort_ascending(const vint4& v) - { - const vint4 a0 = v-vint4(0x80000000); - const vint4 b0 = shuffle<1,0,3,2>(a0); - const vint4 c0 = min(a0,b0); - const vint4 d0 = max(a0,b0); - const vint4 a1 = select<0x5 /* 0b0101 */>(c0,d0); - const vint4 b1 = shuffle<2,3,0,1>(a1); - const vint4 c1 = min(a1,b1); - const vint4 d1 = max(a1,b1); - const vint4 a2 = select<0x3 /* 0b0011 */>(c1,d1); - const vint4 b2 = shuffle<0,2,1,3>(a2); - const vint4 c2 = min(a2,b2); - const vint4 d2 = max(a2,b2); - const vint4 a3 = select<0x2 /* 0b0010 */>(c2,d2); - return a3+vint4(0x80000000); - } - - __forceinline vint4 usort_descending(const vint4& v) - { - const vint4 a0 = v-vint4(0x80000000); - const vint4 b0 = shuffle<1,0,3,2>(a0); - const vint4 c0 = max(a0,b0); - const vint4 d0 = min(a0,b0); - const vint4 a1 = select<0x5 /* 0b0101 */>(c0,d0); - const vint4 b1 = shuffle<2,3,0,1>(a1); - const vint4 c1 = max(a1,b1); - const vint4 d1 = min(a1,b1); - const vint4 a2 = select<0x3 /* 0b0011 */>(c1,d1); - const vint4 b2 = shuffle<0,2,1,3>(a2); - const vint4 c2 = max(a2,b2); - const vint4 d2 = min(a2,b2); - const vint4 a3 = select<0x2 /* 0b0010 */>(c2,d2); - return a3+vint4(0x80000000); - } - -#endif - - //////////////////////////////////////////////////////////////////////////////// - /// Output Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline embree_ostream operator <<(embree_ostream cout, const vint4& a) { - return cout << "<" << a[0] << ", " << a[1] << ", " << a[2] << ", " << a[3] << ">"; - } -} - diff --git a/thirdparty/embree-aarch64/common/simd/vint8_avx.h b/thirdparty/embree-aarch64/common/simd/vint8_avx.h deleted file mode 100644 index 25a771284d..0000000000 --- a/thirdparty/embree-aarch64/common/simd/vint8_avx.h +++ /dev/null @@ -1,464 +0,0 @@ -// Copyright 2009-2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 - -#pragma once - -namespace embree -{ - /* 8-wide AVX integer type */ - template<> - struct vint<8> - { - ALIGNED_STRUCT_(32); - - typedef vboolf8 Bool; - typedef vint8 Int; - typedef vfloat8 Float; - - enum { size = 8 }; // number of SIMD elements - union { // data - __m256i v; - struct { __m128i vl,vh; }; - int i[8]; - }; - - //////////////////////////////////////////////////////////////////////////////// - /// Constructors, Assignment & Cast Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vint() {} - __forceinline vint(const vint8& a) { v = a.v; } - __forceinline vint8& operator =(const vint8& a) { v = a.v; return *this; } - - __forceinline vint(__m256i a) : v(a) {} - __forceinline operator const __m256i&() const { return v; } - __forceinline operator __m256i&() { return v; } - - __forceinline explicit vint(const vint4& a) : v(_mm256_insertf128_si256(_mm256_castsi128_si256(a),a,1)) {} - __forceinline vint(const vint4& a, const vint4& b) : v(_mm256_insertf128_si256(_mm256_castsi128_si256(a),b,1)) {} - __forceinline vint(const __m128i& a, const __m128i& b) : vl(a), vh(b) {} - - __forceinline explicit vint(const int* a) : v(_mm256_castps_si256(_mm256_loadu_ps((const float*)a))) {} - __forceinline vint(int a) : v(_mm256_set1_epi32(a)) {} - __forceinline vint(int a, int b) : v(_mm256_set_epi32(b, a, b, a, b, a, b, a)) {} - __forceinline vint(int a, int b, int c, int d) : v(_mm256_set_epi32(d, c, b, a, d, c, b, a)) {} - __forceinline vint(int a, int b, int c, int d, int e, int f, int g, int vh) : v(_mm256_set_epi32(vh, g, f, e, d, c, b, a)) {} - - __forceinline explicit vint(__m256 a) : v(_mm256_cvtps_epi32(a)) {} - - //////////////////////////////////////////////////////////////////////////////// - /// Constants - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vint(ZeroTy) : v(_mm256_setzero_si256()) {} - __forceinline vint(OneTy) : v(_mm256_set_epi32(1,1,1,1,1,1,1,1)) {} - __forceinline vint(PosInfTy) : v(_mm256_set_epi32(pos_inf,pos_inf,pos_inf,pos_inf,pos_inf,pos_inf,pos_inf,pos_inf)) {} - __forceinline vint(NegInfTy) : v(_mm256_set_epi32(neg_inf,neg_inf,neg_inf,neg_inf,neg_inf,neg_inf,neg_inf,neg_inf)) {} - __forceinline vint(StepTy) : v(_mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0)) {} - __forceinline vint(ReverseStepTy) : v(_mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7)) {} - __forceinline vint(UndefinedTy) : v(_mm256_undefined_si256()) {} - - //////////////////////////////////////////////////////////////////////////////// - /// Loads and Stores - //////////////////////////////////////////////////////////////////////////////// - - static __forceinline vint8 load (const void* a) { return _mm256_castps_si256(_mm256_load_ps((float*)a)); } - static __forceinline vint8 loadu(const void* a) { return _mm256_castps_si256(_mm256_loadu_ps((float*)a)); } - - static __forceinline vint8 load (const vboolf8& mask, const void* a) { return _mm256_castps_si256(_mm256_maskload_ps((float*)a,mask)); } - static __forceinline vint8 loadu(const vboolf8& mask, const void* a) { return _mm256_castps_si256(_mm256_maskload_ps((float*)a,mask)); } - - static __forceinline void store (void* ptr, const vint8& f) { _mm256_store_ps((float*)ptr,_mm256_castsi256_ps(f)); } - static __forceinline void storeu(void* ptr, const vint8& f) { _mm256_storeu_ps((float*)ptr,_mm256_castsi256_ps(f)); } - -#if !defined(__aarch64__) - static __forceinline void store (const vboolf8& mask, void* ptr, const vint8& f) { _mm256_maskstore_ps((float*)ptr,(__m256i)mask,_mm256_castsi256_ps(f)); } - static __forceinline void storeu(const vboolf8& mask, void* ptr, const vint8& f) { _mm256_maskstore_ps((float*)ptr,(__m256i)mask,_mm256_castsi256_ps(f)); } -#else - static __forceinline void store (const vboolf8& mask, void* ptr, const vint8& f) { _mm256_maskstore_ps((float*)ptr,(__m256i)mask.v,_mm256_castsi256_ps(f)); } - static __forceinline void storeu(const vboolf8& mask, void* ptr, const vint8& f) { _mm256_maskstore_ps((float*)ptr,(__m256i)mask.v,_mm256_castsi256_ps(f)); } -#endif - - static __forceinline void store_nt(void* ptr, const vint8& v) { - _mm256_stream_ps((float*)ptr,_mm256_castsi256_ps(v)); - } - - static __forceinline vint8 load(const uint8_t* ptr) { - vint4 il = vint4::load(ptr+0); - vint4 ih = vint4::load(ptr+4); - return vint8(il,ih); - } - - static __forceinline vint8 loadu(const uint8_t* ptr) { - vint4 il = vint4::loadu(ptr+0); - vint4 ih = vint4::loadu(ptr+4); - return vint8(il,ih); - } - - static __forceinline vint8 load(const unsigned short* ptr) { - vint4 il = vint4::load(ptr+0); - vint4 ih = vint4::load(ptr+4); - return vint8(il,ih); - } - - static __forceinline vint8 loadu(const unsigned short* ptr) { - vint4 il = vint4::loadu(ptr+0); - vint4 ih = vint4::loadu(ptr+4); - return vint8(il,ih); - } - - static __forceinline void store(uint8_t* ptr, const vint8& i) { - vint4 il(i.vl); - vint4 ih(i.vh); - vint4::store(ptr + 0,il); - vint4::store(ptr + 4,ih); - } - - static __forceinline void store(unsigned short* ptr, const vint8& v) { - for (size_t i=0;i<8;i++) - ptr[i] = (unsigned short)v[i]; - } - - template<int scale = 4> - static __forceinline vint8 gather(const int* ptr, const vint8& index) { - return vint8( - *(int*)(((int8_t*)ptr)+scale*index[0]), - *(int*)(((int8_t*)ptr)+scale*index[1]), - *(int*)(((int8_t*)ptr)+scale*index[2]), - *(int*)(((int8_t*)ptr)+scale*index[3]), - *(int*)(((int8_t*)ptr)+scale*index[4]), - *(int*)(((int8_t*)ptr)+scale*index[5]), - *(int*)(((int8_t*)ptr)+scale*index[6]), - *(int*)(((int8_t*)ptr)+scale*index[7])); - } - - template<int scale = 4> - static __forceinline vint8 gather(const vboolf8& mask, const int* ptr, const vint8& index) { - vint8 r = zero; - if (likely(mask[0])) r[0] = *(int*)(((int8_t*)ptr)+scale*index[0]); - if (likely(mask[1])) r[1] = *(int*)(((int8_t*)ptr)+scale*index[1]); - if (likely(mask[2])) r[2] = *(int*)(((int8_t*)ptr)+scale*index[2]); - if (likely(mask[3])) r[3] = *(int*)(((int8_t*)ptr)+scale*index[3]); - if (likely(mask[4])) r[4] = *(int*)(((int8_t*)ptr)+scale*index[4]); - if (likely(mask[5])) r[5] = *(int*)(((int8_t*)ptr)+scale*index[5]); - if (likely(mask[6])) r[6] = *(int*)(((int8_t*)ptr)+scale*index[6]); - if (likely(mask[7])) r[7] = *(int*)(((int8_t*)ptr)+scale*index[7]); - return r; - } - - template<int scale = 4> - static __forceinline void scatter(void* ptr, const vint8& ofs, const vint8& v) - { - *(int*)(((int8_t*)ptr)+scale*ofs[0]) = v[0]; - *(int*)(((int8_t*)ptr)+scale*ofs[1]) = v[1]; - *(int*)(((int8_t*)ptr)+scale*ofs[2]) = v[2]; - *(int*)(((int8_t*)ptr)+scale*ofs[3]) = v[3]; - *(int*)(((int8_t*)ptr)+scale*ofs[4]) = v[4]; - *(int*)(((int8_t*)ptr)+scale*ofs[5]) = v[5]; - *(int*)(((int8_t*)ptr)+scale*ofs[6]) = v[6]; - *(int*)(((int8_t*)ptr)+scale*ofs[7]) = v[7]; - } - - template<int scale = 4> - static __forceinline void scatter(const vboolf8& mask, void* ptr, const vint8& ofs, const vint8& v) - { - if (likely(mask[0])) *(int*)(((int8_t*)ptr)+scale*ofs[0]) = v[0]; - if (likely(mask[1])) *(int*)(((int8_t*)ptr)+scale*ofs[1]) = v[1]; - if (likely(mask[2])) *(int*)(((int8_t*)ptr)+scale*ofs[2]) = v[2]; - if (likely(mask[3])) *(int*)(((int8_t*)ptr)+scale*ofs[3]) = v[3]; - if (likely(mask[4])) *(int*)(((int8_t*)ptr)+scale*ofs[4]) = v[4]; - if (likely(mask[5])) *(int*)(((int8_t*)ptr)+scale*ofs[5]) = v[5]; - if (likely(mask[6])) *(int*)(((int8_t*)ptr)+scale*ofs[6]) = v[6]; - if (likely(mask[7])) *(int*)(((int8_t*)ptr)+scale*ofs[7]) = v[7]; - } - - - static __forceinline vint8 broadcast64(const long long& a) { return _mm256_set1_epi64x(a); } - - //////////////////////////////////////////////////////////////////////////////// - /// Array Access - //////////////////////////////////////////////////////////////////////////////// - - __forceinline const int& operator [](size_t index) const { assert(index < 8); return i[index]; } - __forceinline int& operator [](size_t index) { assert(index < 8); return i[index]; } - }; - - //////////////////////////////////////////////////////////////////////////////// - /// Unary Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboolf8 asBool(const vint8& a) { return _mm256_castsi256_ps(a); } - - __forceinline vint8 operator +(const vint8& a) { return a; } - __forceinline vint8 operator -(const vint8& a) { return vint8(_mm_sub_epi32(_mm_setzero_si128(), a.vl), _mm_sub_epi32(_mm_setzero_si128(), a.vh)); } - __forceinline vint8 abs (const vint8& a) { return vint8(_mm_abs_epi32(a.vl), _mm_abs_epi32(a.vh)); } - - //////////////////////////////////////////////////////////////////////////////// - /// Binary Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vint8 operator +(const vint8& a, const vint8& b) { return vint8(_mm_add_epi32(a.vl, b.vl), _mm_add_epi32(a.vh, b.vh)); } - __forceinline vint8 operator +(const vint8& a, int b) { return a + vint8(b); } - __forceinline vint8 operator +(int a, const vint8& b) { return vint8(a) + b; } - - __forceinline vint8 operator -(const vint8& a, const vint8& b) { return vint8(_mm_sub_epi32(a.vl, b.vl), _mm_sub_epi32(a.vh, b.vh)); } - __forceinline vint8 operator -(const vint8& a, int b) { return a - vint8(b); } - __forceinline vint8 operator -(int a, const vint8& b) { return vint8(a) - b; } - - __forceinline vint8 operator *(const vint8& a, const vint8& b) { return vint8(_mm_mullo_epi32(a.vl, b.vl), _mm_mullo_epi32(a.vh, b.vh)); } - __forceinline vint8 operator *(const vint8& a, int b) { return a * vint8(b); } - __forceinline vint8 operator *(int a, const vint8& b) { return vint8(a) * b; } - - __forceinline vint8 operator &(const vint8& a, const vint8& b) { return _mm256_castps_si256(_mm256_and_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b))); } - __forceinline vint8 operator &(const vint8& a, int b) { return a & vint8(b); } - __forceinline vint8 operator &(int a, const vint8& b) { return vint8(a) & b; } - - __forceinline vint8 operator |(const vint8& a, const vint8& b) { return _mm256_castps_si256(_mm256_or_ps (_mm256_castsi256_ps(a), _mm256_castsi256_ps(b))); } - __forceinline vint8 operator |(const vint8& a, int b) { return a | vint8(b); } - __forceinline vint8 operator |(int a, const vint8& b) { return vint8(a) | b; } - - __forceinline vint8 operator ^(const vint8& a, const vint8& b) { return _mm256_castps_si256(_mm256_xor_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b))); } - __forceinline vint8 operator ^(const vint8& a, int b) { return a ^ vint8(b); } - __forceinline vint8 operator ^(int a, const vint8& b) { return vint8(a) ^ b; } - - __forceinline vint8 operator <<(const vint8& a, int n) { return vint8(_mm_slli_epi32(a.vl, n), _mm_slli_epi32(a.vh, n)); } - __forceinline vint8 operator >>(const vint8& a, int n) { return vint8(_mm_srai_epi32(a.vl, n), _mm_srai_epi32(a.vh, n)); } - - __forceinline vint8 sll (const vint8& a, int b) { return vint8(_mm_slli_epi32(a.vl, b), _mm_slli_epi32(a.vh, b)); } - __forceinline vint8 sra (const vint8& a, int b) { return vint8(_mm_srai_epi32(a.vl, b), _mm_srai_epi32(a.vh, b)); } - __forceinline vint8 srl (const vint8& a, int b) { return vint8(_mm_srli_epi32(a.vl, b), _mm_srli_epi32(a.vh, b)); } - - __forceinline vint8 min(const vint8& a, const vint8& b) { return vint8(_mm_min_epi32(a.vl, b.vl), _mm_min_epi32(a.vh, b.vh)); } - __forceinline vint8 min(const vint8& a, int b) { return min(a,vint8(b)); } - __forceinline vint8 min(int a, const vint8& b) { return min(vint8(a),b); } - - __forceinline vint8 max(const vint8& a, const vint8& b) { return vint8(_mm_max_epi32(a.vl, b.vl), _mm_max_epi32(a.vh, b.vh)); } - __forceinline vint8 max(const vint8& a, int b) { return max(a,vint8(b)); } - __forceinline vint8 max(int a, const vint8& b) { return max(vint8(a),b); } - - __forceinline vint8 umin(const vint8& a, const vint8& b) { return vint8(_mm_min_epu32(a.vl, b.vl), _mm_min_epu32(a.vh, b.vh)); } - __forceinline vint8 umin(const vint8& a, int b) { return umin(a,vint8(b)); } - __forceinline vint8 umin(int a, const vint8& b) { return umin(vint8(a),b); } - - __forceinline vint8 umax(const vint8& a, const vint8& b) { return vint8(_mm_max_epu32(a.vl, b.vl), _mm_max_epu32(a.vh, b.vh)); } - __forceinline vint8 umax(const vint8& a, int b) { return umax(a,vint8(b)); } - __forceinline vint8 umax(int a, const vint8& b) { return umax(vint8(a),b); } - - //////////////////////////////////////////////////////////////////////////////// - /// Assignment Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vint8& operator +=(vint8& a, const vint8& b) { return a = a + b; } - __forceinline vint8& operator +=(vint8& a, int b) { return a = a + b; } - - __forceinline vint8& operator -=(vint8& a, const vint8& b) { return a = a - b; } - __forceinline vint8& operator -=(vint8& a, int b) { return a = a - b; } - - __forceinline vint8& operator *=(vint8& a, const vint8& b) { return a = a * b; } - __forceinline vint8& operator *=(vint8& a, int b) { return a = a * b; } - - __forceinline vint8& operator &=(vint8& a, const vint8& b) { return a = a & b; } - __forceinline vint8& operator &=(vint8& a, int b) { return a = a & b; } - - __forceinline vint8& operator |=(vint8& a, const vint8& b) { return a = a | b; } - __forceinline vint8& operator |=(vint8& a, int b) { return a = a | b; } - - __forceinline vint8& operator <<=(vint8& a, int b) { return a = a << b; } - __forceinline vint8& operator >>=(vint8& a, int b) { return a = a >> b; } - - //////////////////////////////////////////////////////////////////////////////// - /// Comparison Operators + Select - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboolf8 operator ==(const vint8& a, const vint8& b) { return vboolf8(_mm_castsi128_ps(_mm_cmpeq_epi32 (a.vl, b.vl)), - _mm_castsi128_ps(_mm_cmpeq_epi32 (a.vh, b.vh))); } - __forceinline vboolf8 operator ==(const vint8& a, int b) { return a == vint8(b); } - __forceinline vboolf8 operator ==(int a, const vint8& b) { return vint8(a) == b; } - - __forceinline vboolf8 operator !=(const vint8& a, const vint8& b) { return !(a == b); } - __forceinline vboolf8 operator !=(const vint8& a, int b) { return a != vint8(b); } - __forceinline vboolf8 operator !=(int a, const vint8& b) { return vint8(a) != b; } - - __forceinline vboolf8 operator < (const vint8& a, const vint8& b) { return vboolf8(_mm_castsi128_ps(_mm_cmplt_epi32 (a.vl, b.vl)), - _mm_castsi128_ps(_mm_cmplt_epi32 (a.vh, b.vh))); } - __forceinline vboolf8 operator < (const vint8& a, int b) { return a < vint8(b); } - __forceinline vboolf8 operator < (int a, const vint8& b) { return vint8(a) < b; } - - __forceinline vboolf8 operator >=(const vint8& a, const vint8& b) { return !(a < b); } - __forceinline vboolf8 operator >=(const vint8& a, int b) { return a >= vint8(b); } - __forceinline vboolf8 operator >=(int a, const vint8& b) { return vint8(a) >= b; } - - __forceinline vboolf8 operator > (const vint8& a, const vint8& b) { return vboolf8(_mm_castsi128_ps(_mm_cmpgt_epi32 (a.vl, b.vl)), - _mm_castsi128_ps(_mm_cmpgt_epi32 (a.vh, b.vh))); } - __forceinline vboolf8 operator > (const vint8& a, int b) { return a > vint8(b); } - __forceinline vboolf8 operator > (int a, const vint8& b) { return vint8(a) > b; } - - __forceinline vboolf8 operator <=(const vint8& a, const vint8& b) { return !(a > b); } - __forceinline vboolf8 operator <=(const vint8& a, int b) { return a <= vint8(b); } - __forceinline vboolf8 operator <=(int a, const vint8& b) { return vint8(a) <= b; } - - __forceinline vboolf8 eq(const vint8& a, const vint8& b) { return a == b; } - __forceinline vboolf8 ne(const vint8& a, const vint8& b) { return a != b; } - __forceinline vboolf8 lt(const vint8& a, const vint8& b) { return a < b; } - __forceinline vboolf8 ge(const vint8& a, const vint8& b) { return a >= b; } - __forceinline vboolf8 gt(const vint8& a, const vint8& b) { return a > b; } - __forceinline vboolf8 le(const vint8& a, const vint8& b) { return a <= b; } - - __forceinline vboolf8 eq(const vboolf8& mask, const vint8& a, const vint8& b) { return mask & (a == b); } - __forceinline vboolf8 ne(const vboolf8& mask, const vint8& a, const vint8& b) { return mask & (a != b); } - __forceinline vboolf8 lt(const vboolf8& mask, const vint8& a, const vint8& b) { return mask & (a < b); } - __forceinline vboolf8 ge(const vboolf8& mask, const vint8& a, const vint8& b) { return mask & (a >= b); } - __forceinline vboolf8 gt(const vboolf8& mask, const vint8& a, const vint8& b) { return mask & (a > b); } - __forceinline vboolf8 le(const vboolf8& mask, const vint8& a, const vint8& b) { return mask & (a <= b); } - - __forceinline vint8 select(const vboolf8& m, const vint8& t, const vint8& f) { - return _mm256_castps_si256(_mm256_blendv_ps(_mm256_castsi256_ps(f), _mm256_castsi256_ps(t), m)); - } - - __forceinline vint8 notand(const vboolf8& m, const vint8& f) { - return _mm256_castps_si256(_mm256_andnot_ps(m, _mm256_castsi256_ps(f))); - } - - - //////////////////////////////////////////////////////////////////////////////// - /// Movement/Shifting/Shuffling Functions - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vint8 unpacklo(const vint8& a, const vint8& b) { return _mm256_castps_si256(_mm256_unpacklo_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b))); } - __forceinline vint8 unpackhi(const vint8& a, const vint8& b) { return _mm256_castps_si256(_mm256_unpackhi_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b))); } - - template<int i> - __forceinline vint8 shuffle(const vint8& v) { - return _mm256_castps_si256(_mm256_permute_ps(_mm256_castsi256_ps(v), _MM_SHUFFLE(i, i, i, i))); - } - - template<int i0, int i1> - __forceinline vint8 shuffle4(const vint8& v) { - return _mm256_permute2f128_si256(v, v, (i1 << 4) | (i0 << 0)); - } - - template<int i0, int i1> - __forceinline vint8 shuffle4(const vint8& a, const vint8& b) { - return _mm256_permute2f128_si256(a, b, (i1 << 4) | (i0 << 0)); - } - - template<int i0, int i1, int i2, int i3> - __forceinline vint8 shuffle(const vint8& v) { - return _mm256_castps_si256(_mm256_permute_ps(_mm256_castsi256_ps(v), _MM_SHUFFLE(i3, i2, i1, i0))); - } - - template<int i0, int i1, int i2, int i3> - __forceinline vint8 shuffle(const vint8& a, const vint8& b) { - return _mm256_castps_si256(_mm256_shuffle_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b), _MM_SHUFFLE(i3, i2, i1, i0))); - } - - template<> __forceinline vint8 shuffle<0, 0, 2, 2>(const vint8& v) { return _mm256_castps_si256(_mm256_moveldup_ps(_mm256_castsi256_ps(v))); } - template<> __forceinline vint8 shuffle<1, 1, 3, 3>(const vint8& v) { return _mm256_castps_si256(_mm256_movehdup_ps(_mm256_castsi256_ps(v))); } - template<> __forceinline vint8 shuffle<0, 1, 0, 1>(const vint8& v) { return _mm256_castps_si256(_mm256_castpd_ps(_mm256_movedup_pd(_mm256_castps_pd(_mm256_castsi256_ps(v))))); } - - __forceinline vint8 broadcast(const int* ptr) { return _mm256_castps_si256(_mm256_broadcast_ss((const float*)ptr)); } - template<int i> __forceinline vint8 insert4(const vint8& a, const vint4& b) { return _mm256_insertf128_si256(a, b, i); } - template<int i> __forceinline vint4 extract4(const vint8& a) { return _mm256_extractf128_si256(a, i); } - template<> __forceinline vint4 extract4<0>(const vint8& a) { return _mm256_castsi256_si128(a); } - - __forceinline int toScalar(const vint8& v) { return _mm_cvtsi128_si32(_mm256_castsi256_si128(v)); } - - - //////////////////////////////////////////////////////////////////////////////// - /// Reductions - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vint8 vreduce_min2(const vint8& v) { return min(v,shuffle<1,0,3,2>(v)); } - __forceinline vint8 vreduce_min4(const vint8& v) { vint8 v1 = vreduce_min2(v); return min(v1,shuffle<2,3,0,1>(v1)); } - __forceinline vint8 vreduce_min (const vint8& v) { vint8 v1 = vreduce_min4(v); return min(v1,shuffle4<1,0>(v1)); } - - __forceinline vint8 vreduce_max2(const vint8& v) { return max(v,shuffle<1,0,3,2>(v)); } - __forceinline vint8 vreduce_max4(const vint8& v) { vint8 v1 = vreduce_max2(v); return max(v1,shuffle<2,3,0,1>(v1)); } - __forceinline vint8 vreduce_max (const vint8& v) { vint8 v1 = vreduce_max4(v); return max(v1,shuffle4<1,0>(v1)); } - - __forceinline vint8 vreduce_add2(const vint8& v) { return v + shuffle<1,0,3,2>(v); } - __forceinline vint8 vreduce_add4(const vint8& v) { vint8 v1 = vreduce_add2(v); return v1 + shuffle<2,3,0,1>(v1); } - __forceinline vint8 vreduce_add (const vint8& v) { vint8 v1 = vreduce_add4(v); return v1 + shuffle4<1,0>(v1); } - - __forceinline int reduce_min(const vint8& v) { return toScalar(vreduce_min(v)); } - __forceinline int reduce_max(const vint8& v) { return toScalar(vreduce_max(v)); } - __forceinline int reduce_add(const vint8& v) { return toScalar(vreduce_add(v)); } - - __forceinline size_t select_min(const vint8& v) { return bsf(movemask(v == vreduce_min(v))); } - __forceinline size_t select_max(const vint8& v) { return bsf(movemask(v == vreduce_max(v))); } - - __forceinline size_t select_min(const vboolf8& valid, const vint8& v) { const vint8 a = select(valid,v,vint8(pos_inf)); return bsf(movemask(valid & (a == vreduce_min(a)))); } - __forceinline size_t select_max(const vboolf8& valid, const vint8& v) { const vint8 a = select(valid,v,vint8(neg_inf)); return bsf(movemask(valid & (a == vreduce_max(a)))); } - - //////////////////////////////////////////////////////////////////////////////// - /// Sorting networks - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vint8 usort_ascending(const vint8& v) - { - const vint8 a0 = v; - const vint8 b0 = shuffle<1,0,3,2>(a0); - const vint8 c0 = umin(a0,b0); - const vint8 d0 = umax(a0,b0); - const vint8 a1 = select(0x99 /* 0b10011001 */,c0,d0); - const vint8 b1 = shuffle<2,3,0,1>(a1); - const vint8 c1 = umin(a1,b1); - const vint8 d1 = umax(a1,b1); - const vint8 a2 = select(0xc3 /* 0b11000011 */,c1,d1); - const vint8 b2 = shuffle<1,0,3,2>(a2); - const vint8 c2 = umin(a2,b2); - const vint8 d2 = umax(a2,b2); - const vint8 a3 = select(0xa5 /* 0b10100101 */,c2,d2); - const vint8 b3 = shuffle4<1,0>(a3); - const vint8 c3 = umin(a3,b3); - const vint8 d3 = umax(a3,b3); - const vint8 a4 = select(0xf /* 0b00001111 */,c3,d3); - const vint8 b4 = shuffle<2,3,0,1>(a4); - const vint8 c4 = umin(a4,b4); - const vint8 d4 = umax(a4,b4); - const vint8 a5 = select(0x33 /* 0b00110011 */,c4,d4); - const vint8 b5 = shuffle<1,0,3,2>(a5); - const vint8 c5 = umin(a5,b5); - const vint8 d5 = umax(a5,b5); - const vint8 a6 = select(0x55 /* 0b01010101 */,c5,d5); - return a6; - } - - __forceinline vint8 usort_descending(const vint8& v) - { - const vint8 a0 = v; - const vint8 b0 = shuffle<1,0,3,2>(a0); - const vint8 c0 = umax(a0,b0); - const vint8 d0 = umin(a0,b0); - const vint8 a1 = select(0x99 /* 0b10011001 */,c0,d0); - const vint8 b1 = shuffle<2,3,0,1>(a1); - const vint8 c1 = umax(a1,b1); - const vint8 d1 = umin(a1,b1); - const vint8 a2 = select(0xc3 /* 0b11000011 */,c1,d1); - const vint8 b2 = shuffle<1,0,3,2>(a2); - const vint8 c2 = umax(a2,b2); - const vint8 d2 = umin(a2,b2); - const vint8 a3 = select(0xa5 /* 0b10100101 */,c2,d2); - const vint8 b3 = shuffle4<1,0>(a3); - const vint8 c3 = umax(a3,b3); - const vint8 d3 = umin(a3,b3); - const vint8 a4 = select(0xf /* 0b00001111 */,c3,d3); - const vint8 b4 = shuffle<2,3,0,1>(a4); - const vint8 c4 = umax(a4,b4); - const vint8 d4 = umin(a4,b4); - const vint8 a5 = select(0x33 /* 0b00110011 */,c4,d4); - const vint8 b5 = shuffle<1,0,3,2>(a5); - const vint8 c5 = umax(a5,b5); - const vint8 d5 = umin(a5,b5); - const vint8 a6 = select(0x55 /* 0b01010101 */,c5,d5); - return a6; - } - - //////////////////////////////////////////////////////////////////////////////// - /// Output Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline embree_ostream operator <<(embree_ostream cout, const vint8& a) { - return cout << "<" << a[0] << ", " << a[1] << ", " << a[2] << ", " << a[3] << ", " << a[4] << ", " << a[5] << ", " << a[6] << ", " << a[7] << ">"; - } -} diff --git a/thirdparty/embree-aarch64/common/simd/vint8_avx2.h b/thirdparty/embree-aarch64/common/simd/vint8_avx2.h deleted file mode 100644 index 4937d972cf..0000000000 --- a/thirdparty/embree-aarch64/common/simd/vint8_avx2.h +++ /dev/null @@ -1,512 +0,0 @@ -// Copyright 2009-2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 - -#pragma once - -namespace embree -{ - /* 8-wide AVX integer type */ - template<> - struct vint<8> - { - ALIGNED_STRUCT_(32); - - typedef vboolf8 Bool; - typedef vint8 Int; - typedef vfloat8 Float; - - enum { size = 8 }; // number of SIMD elements - union { // data - __m256i v; - int i[8]; - }; - - //////////////////////////////////////////////////////////////////////////////// - /// Constructors, Assignment & Cast Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vint() {} - __forceinline vint(const vint8& a) { v = a.v; } - __forceinline vint8& operator =(const vint8& a) { v = a.v; return *this; } - - __forceinline vint(__m256i a) : v(a) {} - __forceinline operator const __m256i&() const { return v; } - __forceinline operator __m256i&() { return v; } - - __forceinline explicit vint(const vint4& a) : v(_mm256_insertf128_si256(_mm256_castsi128_si256(a),a,1)) {} - __forceinline vint(const vint4& a, const vint4& b) : v(_mm256_insertf128_si256(_mm256_castsi128_si256(a),b,1)) {} - __forceinline vint(const __m128i& a, const __m128i& b) : v(_mm256_insertf128_si256(_mm256_castsi128_si256(a),b,1)) {} - - __forceinline explicit vint(const int* a) : v(_mm256_castps_si256(_mm256_loadu_ps((const float*)a))) {} - __forceinline vint(int a) : v(_mm256_set1_epi32(a)) {} - __forceinline vint(int a, int b) : v(_mm256_set_epi32(b, a, b, a, b, a, b, a)) {} - __forceinline vint(int a, int b, int c, int d) : v(_mm256_set_epi32(d, c, b, a, d, c, b, a)) {} - __forceinline vint(int a, int b, int c, int d, int e, int f, int g, int h) : v(_mm256_set_epi32(h, g, f, e, d, c, b, a)) {} - - __forceinline explicit vint(__m256 a) : v(_mm256_cvtps_epi32(a)) {} - -#if defined(__AVX512VL__) - __forceinline explicit vint(const vboolf8& a) : v(_mm256_movm_epi32(a)) {} -#else - __forceinline explicit vint(const vboolf8& a) : v(_mm256_castps_si256((__m256)a)) {} -#endif - - //////////////////////////////////////////////////////////////////////////////// - /// Constants - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vint(ZeroTy) : v(_mm256_setzero_si256()) {} - __forceinline vint(OneTy) : v(_mm256_set1_epi32(1)) {} - __forceinline vint(PosInfTy) : v(_mm256_set1_epi32(pos_inf)) {} - __forceinline vint(NegInfTy) : v(_mm256_set1_epi32(neg_inf)) {} - __forceinline vint(StepTy) : v(_mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0)) {} - __forceinline vint(ReverseStepTy) : v(_mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7)) {} - __forceinline vint(UndefinedTy) : v(_mm256_undefined_si256()) {} - - //////////////////////////////////////////////////////////////////////////////// - /// Loads and Stores - //////////////////////////////////////////////////////////////////////////////// - - static __forceinline vint8 load(const uint8_t* ptr) { return _mm256_cvtepu8_epi32(_mm_loadl_epi64((__m128i*)ptr)); } - static __forceinline vint8 loadu(const uint8_t* ptr) { return _mm256_cvtepu8_epi32(_mm_loadl_epi64((__m128i*)ptr)); } - static __forceinline vint8 load(const unsigned short* ptr) { return _mm256_cvtepu16_epi32(_mm_load_si128((__m128i*)ptr)); } - static __forceinline vint8 loadu(const unsigned short* ptr) { return _mm256_cvtepu16_epi32(_mm_loadu_si128((__m128i*)ptr)); } - - static __forceinline vint8 load(const void* ptr) { return _mm256_load_si256((__m256i*)ptr); } - static __forceinline vint8 loadu(const void* ptr) { return _mm256_loadu_si256((__m256i*)ptr); } - - static __forceinline void store (void* ptr, const vint8& v) { _mm256_store_si256((__m256i*)ptr,v); } - static __forceinline void storeu(void* ptr, const vint8& v) { _mm256_storeu_ps((float*)ptr,_mm256_castsi256_ps(v)); } - -#if defined(__AVX512VL__) - - static __forceinline vint8 compact(const vboolf8& mask, vint8 &v) { - return _mm256_mask_compress_epi32(v, mask, v); - } - static __forceinline vint8 compact(const vboolf8& mask, vint8 &a, const vint8& b) { - return _mm256_mask_compress_epi32(a, mask, b); - } - - static __forceinline vint8 load (const vboolf8& mask, const void* ptr) { return _mm256_mask_load_epi32 (_mm256_setzero_si256(),mask,ptr); } - static __forceinline vint8 loadu(const vboolf8& mask, const void* ptr) { return _mm256_mask_loadu_epi32(_mm256_setzero_si256(),mask,ptr); } - - static __forceinline void store (const vboolf8& mask, void* ptr, const vint8& v) { _mm256_mask_store_epi32 (ptr,mask,v); } - static __forceinline void storeu(const vboolf8& mask, void* ptr, const vint8& v) { _mm256_mask_storeu_epi32(ptr,mask,v); } -#else - static __forceinline vint8 load (const vboolf8& mask, const void* ptr) { return _mm256_castps_si256(_mm256_maskload_ps((float*)ptr,mask)); } - static __forceinline vint8 loadu(const vboolf8& mask, const void* ptr) { return _mm256_castps_si256(_mm256_maskload_ps((float*)ptr,mask)); } - - static __forceinline void store (const vboolf8& mask, void* ptr, const vint8& v) { _mm256_maskstore_epi32((int*)ptr,mask,v); } - static __forceinline void storeu(const vboolf8& mask, void* ptr, const vint8& v) { _mm256_maskstore_epi32((int*)ptr,mask,v); } -#endif - - static __forceinline vint8 load_nt(void* ptr) { - return _mm256_stream_load_si256((__m256i*)ptr); - } - - static __forceinline void store_nt(void* ptr, const vint8& v) { - _mm256_stream_ps((float*)ptr,_mm256_castsi256_ps(v)); - } - - static __forceinline void store(uint8_t* ptr, const vint8& i) - { - for (size_t j=0; j<8; j++) - ptr[j] = i[j]; - } - - static __forceinline void store(unsigned short* ptr, const vint8& v) { - for (size_t i=0;i<8;i++) - ptr[i] = (unsigned short)v[i]; - } - - template<int scale = 4> - static __forceinline vint8 gather(const int *const ptr, const vint8& index) { - return _mm256_i32gather_epi32(ptr, index, scale); - } - - template<int scale = 4> - static __forceinline vint8 gather(const vboolf8& mask, const int *const ptr, const vint8& index) { - vint8 r = zero; -#if defined(__AVX512VL__) - return _mm256_mmask_i32gather_epi32(r, mask, index, ptr, scale); -#else - return _mm256_mask_i32gather_epi32(r, ptr, index, mask, scale); -#endif - } - - template<int scale = 4> - static __forceinline void scatter(void* ptr, const vint8& ofs, const vint8& v) - { -#if defined(__AVX512VL__) - _mm256_i32scatter_epi32((int*)ptr, ofs, v, scale); -#else - *(int*)(((int8_t*)ptr)+scale*ofs[0]) = v[0]; - *(int*)(((int8_t*)ptr)+scale*ofs[1]) = v[1]; - *(int*)(((int8_t*)ptr)+scale*ofs[2]) = v[2]; - *(int*)(((int8_t*)ptr)+scale*ofs[3]) = v[3]; - *(int*)(((int8_t*)ptr)+scale*ofs[4]) = v[4]; - *(int*)(((int8_t*)ptr)+scale*ofs[5]) = v[5]; - *(int*)(((int8_t*)ptr)+scale*ofs[6]) = v[6]; - *(int*)(((int8_t*)ptr)+scale*ofs[7]) = v[7]; -#endif - } - - template<int scale = 4> - static __forceinline void scatter(const vboolf8& mask, void* ptr, const vint8& ofs, const vint8& v) - { -#if defined(__AVX512VL__) - _mm256_mask_i32scatter_epi32((int*)ptr, mask, ofs, v, scale); -#else - if (likely(mask[0])) *(int*)(((int8_t*)ptr)+scale*ofs[0]) = v[0]; - if (likely(mask[1])) *(int*)(((int8_t*)ptr)+scale*ofs[1]) = v[1]; - if (likely(mask[2])) *(int*)(((int8_t*)ptr)+scale*ofs[2]) = v[2]; - if (likely(mask[3])) *(int*)(((int8_t*)ptr)+scale*ofs[3]) = v[3]; - if (likely(mask[4])) *(int*)(((int8_t*)ptr)+scale*ofs[4]) = v[4]; - if (likely(mask[5])) *(int*)(((int8_t*)ptr)+scale*ofs[5]) = v[5]; - if (likely(mask[6])) *(int*)(((int8_t*)ptr)+scale*ofs[6]) = v[6]; - if (likely(mask[7])) *(int*)(((int8_t*)ptr)+scale*ofs[7]) = v[7]; -#endif - } - - static __forceinline vint8 broadcast64(const long long &a) { return _mm256_set1_epi64x(a); } - - //////////////////////////////////////////////////////////////////////////////// - /// Array Access - //////////////////////////////////////////////////////////////////////////////// - - __forceinline const int& operator [](size_t index) const { assert(index < 8); return i[index]; } - __forceinline int& operator [](size_t index) { assert(index < 8); return i[index]; } - }; - - //////////////////////////////////////////////////////////////////////////////// - /// Unary Operators - //////////////////////////////////////////////////////////////////////////////// - -#if defined(__AVX512VL__) - static __forceinline vboolf8 asBool(const vint8& a) { return _mm256_movepi32_mask(a); } -#else - static __forceinline vboolf8 asBool(const vint8& a) { return _mm256_castsi256_ps(a); } -#endif - - __forceinline vint8 operator +(const vint8& a) { return a; } - __forceinline vint8 operator -(const vint8& a) { return _mm256_sub_epi32(_mm256_setzero_si256(), a); } - __forceinline vint8 abs (const vint8& a) { return _mm256_abs_epi32(a); } - - //////////////////////////////////////////////////////////////////////////////// - /// Binary Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vint8 operator +(const vint8& a, const vint8& b) { return _mm256_add_epi32(a, b); } - __forceinline vint8 operator +(const vint8& a, int b) { return a + vint8(b); } - __forceinline vint8 operator +(int a, const vint8& b) { return vint8(a) + b; } - - __forceinline vint8 operator -(const vint8& a, const vint8& b) { return _mm256_sub_epi32(a, b); } - __forceinline vint8 operator -(const vint8& a, int b) { return a - vint8(b); } - __forceinline vint8 operator -(int a, const vint8& b) { return vint8(a) - b; } - - __forceinline vint8 operator *(const vint8& a, const vint8& b) { return _mm256_mullo_epi32(a, b); } - __forceinline vint8 operator *(const vint8& a, int b) { return a * vint8(b); } - __forceinline vint8 operator *(int a, const vint8& b) { return vint8(a) * b; } - - __forceinline vint8 operator &(const vint8& a, const vint8& b) { return _mm256_and_si256(a, b); } - __forceinline vint8 operator &(const vint8& a, int b) { return a & vint8(b); } - __forceinline vint8 operator &(int a, const vint8& b) { return vint8(a) & b; } - - __forceinline vint8 operator |(const vint8& a, const vint8& b) { return _mm256_or_si256(a, b); } - __forceinline vint8 operator |(const vint8& a, int b) { return a | vint8(b); } - __forceinline vint8 operator |(int a, const vint8& b) { return vint8(a) | b; } - - __forceinline vint8 operator ^(const vint8& a, const vint8& b) { return _mm256_xor_si256(a, b); } - __forceinline vint8 operator ^(const vint8& a, int b) { return a ^ vint8(b); } - __forceinline vint8 operator ^(int a, const vint8& b) { return vint8(a) ^ b; } - - __forceinline vint8 operator <<(const vint8& a, int n) { return _mm256_slli_epi32(a, n); } - __forceinline vint8 operator >>(const vint8& a, int n) { return _mm256_srai_epi32(a, n); } - - __forceinline vint8 operator <<(const vint8& a, const vint8& n) { return _mm256_sllv_epi32(a, n); } - __forceinline vint8 operator >>(const vint8& a, const vint8& n) { return _mm256_srav_epi32(a, n); } - - __forceinline vint8 sll(const vint8& a, int b) { return _mm256_slli_epi32(a, b); } - __forceinline vint8 sra(const vint8& a, int b) { return _mm256_srai_epi32(a, b); } - __forceinline vint8 srl(const vint8& a, int b) { return _mm256_srli_epi32(a, b); } - - __forceinline vint8 sll(const vint8& a, const vint8& b) { return _mm256_sllv_epi32(a, b); } - __forceinline vint8 sra(const vint8& a, const vint8& b) { return _mm256_srav_epi32(a, b); } - __forceinline vint8 srl(const vint8& a, const vint8& b) { return _mm256_srlv_epi32(a, b); } - - __forceinline vint8 min(const vint8& a, const vint8& b) { return _mm256_min_epi32(a, b); } - __forceinline vint8 min(const vint8& a, int b) { return min(a,vint8(b)); } - __forceinline vint8 min(int a, const vint8& b) { return min(vint8(a),b); } - - __forceinline vint8 max(const vint8& a, const vint8& b) { return _mm256_max_epi32(a, b); } - __forceinline vint8 max(const vint8& a, int b) { return max(a,vint8(b)); } - __forceinline vint8 max(int a, const vint8& b) { return max(vint8(a),b); } - - __forceinline vint8 umin(const vint8& a, const vint8& b) { return _mm256_min_epu32(a, b); } - __forceinline vint8 umax(const vint8& a, const vint8& b) { return _mm256_max_epu32(a, b); } - - //////////////////////////////////////////////////////////////////////////////// - /// Assignment Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vint8& operator +=(vint8& a, const vint8& b) { return a = a + b; } - __forceinline vint8& operator +=(vint8& a, int b) { return a = a + b; } - - __forceinline vint8& operator -=(vint8& a, const vint8& b) { return a = a - b; } - __forceinline vint8& operator -=(vint8& a, int b) { return a = a - b; } - - __forceinline vint8& operator *=(vint8& a, const vint8& b) { return a = a * b; } - __forceinline vint8& operator *=(vint8& a, int b) { return a = a * b; } - - __forceinline vint8& operator &=(vint8& a, const vint8& b) { return a = a & b; } - __forceinline vint8& operator &=(vint8& a, int b) { return a = a & b; } - - __forceinline vint8& operator |=(vint8& a, const vint8& b) { return a = a | b; } - __forceinline vint8& operator |=(vint8& a, int b) { return a = a | b; } - - __forceinline vint8& operator <<=(vint8& a, const int b) { return a = a << b; } - __forceinline vint8& operator >>=(vint8& a, const int b) { return a = a >> b; } - - //////////////////////////////////////////////////////////////////////////////// - /// Comparison Operators + Select - //////////////////////////////////////////////////////////////////////////////// - -#if defined(__AVX512VL__) - static __forceinline vboolf8 operator ==(const vint8& a, const vint8& b) { return _mm256_cmp_epi32_mask(a,b,_MM_CMPINT_EQ); } - static __forceinline vboolf8 operator !=(const vint8& a, const vint8& b) { return _mm256_cmp_epi32_mask(a,b,_MM_CMPINT_NE); } - static __forceinline vboolf8 operator < (const vint8& a, const vint8& b) { return _mm256_cmp_epi32_mask(a,b,_MM_CMPINT_LT); } - static __forceinline vboolf8 operator >=(const vint8& a, const vint8& b) { return _mm256_cmp_epi32_mask(a,b,_MM_CMPINT_GE); } - static __forceinline vboolf8 operator > (const vint8& a, const vint8& b) { return _mm256_cmp_epi32_mask(a,b,_MM_CMPINT_GT); } - static __forceinline vboolf8 operator <=(const vint8& a, const vint8& b) { return _mm256_cmp_epi32_mask(a,b,_MM_CMPINT_LE); } - - static __forceinline vint8 select(const vboolf8& m, const vint8& t, const vint8& f) { - return _mm256_mask_blend_epi32(m, (__m256i)f, (__m256i)t); - } -#else - static __forceinline vboolf8 operator ==(const vint8& a, const vint8& b) { return _mm256_castsi256_ps(_mm256_cmpeq_epi32(a, b)); } - static __forceinline vboolf8 operator !=(const vint8& a, const vint8& b) { return !(a == b); } - static __forceinline vboolf8 operator < (const vint8& a, const vint8& b) { return _mm256_castsi256_ps(_mm256_cmpgt_epi32(b, a)); } - static __forceinline vboolf8 operator >=(const vint8& a, const vint8& b) { return !(a < b); } - static __forceinline vboolf8 operator > (const vint8& a, const vint8& b) { return _mm256_castsi256_ps(_mm256_cmpgt_epi32(a, b)); } - static __forceinline vboolf8 operator <=(const vint8& a, const vint8& b) { return !(a > b); } - - static __forceinline vint8 select(const vboolf8& m, const vint8& t, const vint8& f) { - return _mm256_castps_si256(_mm256_blendv_ps(_mm256_castsi256_ps(f), _mm256_castsi256_ps(t), m)); - } -#endif - - template<int mask> - __forceinline vint8 select(const vint8& t, const vint8& f) { - return _mm256_blend_epi32(f, t, mask); - } - - __forceinline vboolf8 operator ==(const vint8& a, int b) { return a == vint8(b); } - __forceinline vboolf8 operator ==(int a, const vint8& b) { return vint8(a) == b; } - - __forceinline vboolf8 operator !=(const vint8& a, int b) { return a != vint8(b); } - __forceinline vboolf8 operator !=(int a, const vint8& b) { return vint8(a) != b; } - - __forceinline vboolf8 operator < (const vint8& a, int b) { return a < vint8(b); } - __forceinline vboolf8 operator < (int a, const vint8& b) { return vint8(a) < b; } - - __forceinline vboolf8 operator >=(const vint8& a, int b) { return a >= vint8(b); } - __forceinline vboolf8 operator >=(int a, const vint8& b) { return vint8(a) >= b; } - - __forceinline vboolf8 operator > (const vint8& a, int b) { return a > vint8(b); } - __forceinline vboolf8 operator > (int a, const vint8& b) { return vint8(a) > b; } - - __forceinline vboolf8 operator <=(const vint8& a, int b) { return a <= vint8(b); } - __forceinline vboolf8 operator <=(int a, const vint8& b) { return vint8(a) <= b; } - - __forceinline vboolf8 eq(const vint8& a, const vint8& b) { return a == b; } - __forceinline vboolf8 ne(const vint8& a, const vint8& b) { return a != b; } - __forceinline vboolf8 lt(const vint8& a, const vint8& b) { return a < b; } - __forceinline vboolf8 ge(const vint8& a, const vint8& b) { return a >= b; } - __forceinline vboolf8 gt(const vint8& a, const vint8& b) { return a > b; } - __forceinline vboolf8 le(const vint8& a, const vint8& b) { return a <= b; } - -#if defined(__AVX512VL__) - static __forceinline vboolf8 eq(const vboolf8& mask, const vint8& a, const vint8& b) { return _mm256_mask_cmp_epi32_mask(mask, a, b, _MM_CMPINT_EQ); } - static __forceinline vboolf8 ne(const vboolf8& mask, const vint8& a, const vint8& b) { return _mm256_mask_cmp_epi32_mask(mask, a, b, _MM_CMPINT_NE); } - static __forceinline vboolf8 lt(const vboolf8& mask, const vint8& a, const vint8& b) { return _mm256_mask_cmp_epi32_mask(mask, a, b, _MM_CMPINT_LT); } - static __forceinline vboolf8 ge(const vboolf8& mask, const vint8& a, const vint8& b) { return _mm256_mask_cmp_epi32_mask(mask, a, b, _MM_CMPINT_GE); } - static __forceinline vboolf8 gt(const vboolf8& mask, const vint8& a, const vint8& b) { return _mm256_mask_cmp_epi32_mask(mask, a, b, _MM_CMPINT_GT); } - static __forceinline vboolf8 le(const vboolf8& mask, const vint8& a, const vint8& b) { return _mm256_mask_cmp_epi32_mask(mask, a, b, _MM_CMPINT_LE); } -#else - static __forceinline vboolf8 eq(const vboolf8& mask, const vint8& a, const vint8& b) { return mask & (a == b); } - static __forceinline vboolf8 ne(const vboolf8& mask, const vint8& a, const vint8& b) { return mask & (a != b); } - static __forceinline vboolf8 lt(const vboolf8& mask, const vint8& a, const vint8& b) { return mask & (a < b); } - static __forceinline vboolf8 ge(const vboolf8& mask, const vint8& a, const vint8& b) { return mask & (a >= b); } - static __forceinline vboolf8 gt(const vboolf8& mask, const vint8& a, const vint8& b) { return mask & (a > b); } - static __forceinline vboolf8 le(const vboolf8& mask, const vint8& a, const vint8& b) { return mask & (a <= b); } -#endif - - //////////////////////////////////////////////////////////////////////////////// - /// Movement/Shifting/Shuffling Functions - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vint8 unpacklo(const vint8& a, const vint8& b) { return _mm256_unpacklo_epi32(a, b); } - __forceinline vint8 unpackhi(const vint8& a, const vint8& b) { return _mm256_unpackhi_epi32(a, b); } - - template<int i> - __forceinline vint8 shuffle(const vint8& v) { - return _mm256_castps_si256(_mm256_permute_ps(_mm256_castsi256_ps(v), _MM_SHUFFLE(i, i, i, i))); - } - - template<int i0, int i1> - __forceinline vint8 shuffle4(const vint8& v) { - return _mm256_permute2f128_si256(v, v, (i1 << 4) | (i0 << 0)); - } - - template<int i0, int i1> - __forceinline vint8 shuffle4(const vint8& a, const vint8& b) { - return _mm256_permute2f128_si256(a, b, (i1 << 4) | (i0 << 0)); - } - - template<int i0, int i1, int i2, int i3> - __forceinline vint8 shuffle(const vint8& v) { - return _mm256_castps_si256(_mm256_permute_ps(_mm256_castsi256_ps(v), _MM_SHUFFLE(i3, i2, i1, i0))); - } - - template<int i0, int i1, int i2, int i3> - __forceinline vint8 shuffle(const vint8& a, const vint8& b) { - return _mm256_castps_si256(_mm256_shuffle_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b), _MM_SHUFFLE(i3, i2, i1, i0))); - } - - template<> __forceinline vint8 shuffle<0, 0, 2, 2>(const vint8& v) { return _mm256_castps_si256(_mm256_moveldup_ps(_mm256_castsi256_ps(v))); } - template<> __forceinline vint8 shuffle<1, 1, 3, 3>(const vint8& v) { return _mm256_castps_si256(_mm256_movehdup_ps(_mm256_castsi256_ps(v))); } - template<> __forceinline vint8 shuffle<0, 1, 0, 1>(const vint8& v) { return _mm256_castps_si256(_mm256_castpd_ps(_mm256_movedup_pd(_mm256_castps_pd(_mm256_castsi256_ps(v))))); } - - __forceinline vint8 broadcast(const int* ptr) { return _mm256_castps_si256(_mm256_broadcast_ss((const float*)ptr)); } - - template<int i> __forceinline vint8 insert4(const vint8& a, const vint4& b) { return _mm256_insertf128_si256(a, b, i); } - template<int i> __forceinline vint4 extract4(const vint8& a) { return _mm256_extractf128_si256(a, i); } - template<> __forceinline vint4 extract4<0>(const vint8& a) { return _mm256_castsi256_si128(a); } - - __forceinline int toScalar(const vint8& v) { return _mm_cvtsi128_si32(_mm256_castsi256_si128(v)); } - -#if !defined(__aarch64__) - -__forceinline vint8 permute(const vint8& v, const __m256i& index) { - return _mm256_permutevar8x32_epi32(v, index); - } - - __forceinline vint8 shuffle(const vint8& v, const __m256i& index) { - return _mm256_castps_si256(_mm256_permutevar_ps(_mm256_castsi256_ps(v), index)); - } - - - - template<int i> - static __forceinline vint8 align_shift_right(const vint8& a, const vint8& b) { -#if defined(__AVX512VL__) - return _mm256_alignr_epi32(a, b, i); -#else - return _mm256_alignr_epi8(a, b, 4*i); -#endif - } - -#endif - - - //////////////////////////////////////////////////////////////////////////////// - /// Reductions - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vint8 vreduce_min2(const vint8& v) { return min(v,shuffle<1,0,3,2>(v)); } - __forceinline vint8 vreduce_min4(const vint8& v) { vint8 v1 = vreduce_min2(v); return min(v1,shuffle<2,3,0,1>(v1)); } - __forceinline vint8 vreduce_min (const vint8& v) { vint8 v1 = vreduce_min4(v); return min(v1,shuffle4<1,0>(v1)); } - - __forceinline vint8 vreduce_max2(const vint8& v) { return max(v,shuffle<1,0,3,2>(v)); } - __forceinline vint8 vreduce_max4(const vint8& v) { vint8 v1 = vreduce_max2(v); return max(v1,shuffle<2,3,0,1>(v1)); } - __forceinline vint8 vreduce_max (const vint8& v) { vint8 v1 = vreduce_max4(v); return max(v1,shuffle4<1,0>(v1)); } - - __forceinline vint8 vreduce_add2(const vint8& v) { return v + shuffle<1,0,3,2>(v); } - __forceinline vint8 vreduce_add4(const vint8& v) { vint8 v1 = vreduce_add2(v); return v1 + shuffle<2,3,0,1>(v1); } - __forceinline vint8 vreduce_add (const vint8& v) { vint8 v1 = vreduce_add4(v); return v1 + shuffle4<1,0>(v1); } - - __forceinline int reduce_min(const vint8& v) { return toScalar(vreduce_min(v)); } - __forceinline int reduce_max(const vint8& v) { return toScalar(vreduce_max(v)); } - __forceinline int reduce_add(const vint8& v) { return toScalar(vreduce_add(v)); } - - __forceinline size_t select_min(const vint8& v) { return bsf(movemask(v == vreduce_min(v))); } - __forceinline size_t select_max(const vint8& v) { return bsf(movemask(v == vreduce_max(v))); } - - __forceinline size_t select_min(const vboolf8& valid, const vint8& v) { const vint8 a = select(valid,v,vint8(pos_inf)); return bsf(movemask(valid & (a == vreduce_min(a)))); } - __forceinline size_t select_max(const vboolf8& valid, const vint8& v) { const vint8 a = select(valid,v,vint8(neg_inf)); return bsf(movemask(valid & (a == vreduce_max(a)))); } - - - __forceinline vint8 assign(const vint4& a) { return _mm256_castsi128_si256(a); } - - //////////////////////////////////////////////////////////////////////////////// - /// Sorting networks - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vint8 usort_ascending(const vint8& v) - { - const vint8 a0 = v; - const vint8 b0 = shuffle<1,0,3,2>(a0); - const vint8 c0 = umin(a0,b0); - const vint8 d0 = umax(a0,b0); - const vint8 a1 = select<0x99 /* 0b10011001 */>(c0,d0); - const vint8 b1 = shuffle<2,3,0,1>(a1); - const vint8 c1 = umin(a1,b1); - const vint8 d1 = umax(a1,b1); - const vint8 a2 = select<0xc3 /* 0b11000011 */>(c1,d1); - const vint8 b2 = shuffle<1,0,3,2>(a2); - const vint8 c2 = umin(a2,b2); - const vint8 d2 = umax(a2,b2); - const vint8 a3 = select<0xa5 /* 0b10100101 */>(c2,d2); - const vint8 b3 = shuffle4<1,0>(a3); - const vint8 c3 = umin(a3,b3); - const vint8 d3 = umax(a3,b3); - const vint8 a4 = select<0xf /* 0b00001111 */>(c3,d3); - const vint8 b4 = shuffle<2,3,0,1>(a4); - const vint8 c4 = umin(a4,b4); - const vint8 d4 = umax(a4,b4); - const vint8 a5 = select<0x33 /* 0b00110011 */>(c4,d4); - const vint8 b5 = shuffle<1,0,3,2>(a5); - const vint8 c5 = umin(a5,b5); - const vint8 d5 = umax(a5,b5); - const vint8 a6 = select<0x55 /* 0b01010101 */>(c5,d5); - return a6; - } - - __forceinline vint8 usort_descending(const vint8& v) - { - const vint8 a0 = v; - const vint8 b0 = shuffle<1,0,3,2>(a0); - const vint8 c0 = umax(a0,b0); - const vint8 d0 = umin(a0,b0); - const vint8 a1 = select<0x99 /* 0b10011001 */>(c0,d0); - const vint8 b1 = shuffle<2,3,0,1>(a1); - const vint8 c1 = umax(a1,b1); - const vint8 d1 = umin(a1,b1); - const vint8 a2 = select<0xc3 /* 0b11000011 */>(c1,d1); - const vint8 b2 = shuffle<1,0,3,2>(a2); - const vint8 c2 = umax(a2,b2); - const vint8 d2 = umin(a2,b2); - const vint8 a3 = select<0xa5 /* 0b10100101 */>(c2,d2); - const vint8 b3 = shuffle4<1,0>(a3); - const vint8 c3 = umax(a3,b3); - const vint8 d3 = umin(a3,b3); - const vint8 a4 = select<0xf /* 0b00001111 */>(c3,d3); - const vint8 b4 = shuffle<2,3,0,1>(a4); - const vint8 c4 = umax(a4,b4); - const vint8 d4 = umin(a4,b4); - const vint8 a5 = select<0x33 /* 0b00110011 */>(c4,d4); - const vint8 b5 = shuffle<1,0,3,2>(a5); - const vint8 c5 = umax(a5,b5); - const vint8 d5 = umin(a5,b5); - const vint8 a6 = select<0x55 /* 0b01010101 */>(c5,d5); - return a6; - } - - //////////////////////////////////////////////////////////////////////////////// - /// Output Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline embree_ostream operator <<(embree_ostream cout, const vint8& a) { - return cout << "<" << a[0] << ", " << a[1] << ", " << a[2] << ", " << a[3] << ", " << a[4] << ", " << a[5] << ", " << a[6] << ", " << a[7] << ">"; - } -} diff --git a/thirdparty/embree-aarch64/common/simd/vllong4_avx2.h b/thirdparty/embree-aarch64/common/simd/vllong4_avx2.h deleted file mode 100644 index de3ebc16a7..0000000000 --- a/thirdparty/embree-aarch64/common/simd/vllong4_avx2.h +++ /dev/null @@ -1,358 +0,0 @@ -// Copyright 2009-2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 - -#pragma once - -namespace embree -{ - /* 4-wide AVX2 64-bit long long type */ - template<> - struct vllong<4> - { - ALIGNED_STRUCT_(32); - - typedef vboold4 Bool; - - enum { size = 4 }; // number of SIMD elements - union { // data - __m256i v; - long long i[4]; - }; - - //////////////////////////////////////////////////////////////////////////////// - /// Constructors, Assignment & Cast Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vllong() {} - __forceinline vllong(const vllong4& t) { v = t.v; } - __forceinline vllong4& operator =(const vllong4& f) { v = f.v; return *this; } - - __forceinline vllong(const __m256i& t) { v = t; } - __forceinline operator __m256i() const { return v; } - __forceinline operator __m256d() const { return _mm256_castsi256_pd(v); } - - - __forceinline vllong(long long i) { - v = _mm256_set1_epi64x(i); - } - - __forceinline vllong(long long a, long long b, long long c, long long d) { - v = _mm256_set_epi64x(d,c,b,a); - } - - - //////////////////////////////////////////////////////////////////////////////// - /// Constants - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vllong(ZeroTy) : v(_mm256_setzero_si256()) {} - __forceinline vllong(OneTy) : v(_mm256_set1_epi64x(1)) {} - __forceinline vllong(StepTy) : v(_mm256_set_epi64x(3,2,1,0)) {} - __forceinline vllong(ReverseStepTy) : v(_mm256_set_epi64x(0,1,2,3)) {} - - //////////////////////////////////////////////////////////////////////////////// - /// Loads and Stores - //////////////////////////////////////////////////////////////////////////////// - - static __forceinline void store_nt(void* __restrict__ ptr, const vllong4& a) { - _mm256_stream_ps((float*)ptr,_mm256_castsi256_ps(a)); - } - - static __forceinline vllong4 loadu(const void* addr) - { - return _mm256_loadu_si256((__m256i*)addr); - } - - static __forceinline vllong4 load(const vllong4* addr) { - return _mm256_load_si256((__m256i*)addr); - } - - static __forceinline vllong4 load(const long long* addr) { - return _mm256_load_si256((__m256i*)addr); - } - - static __forceinline void store(void* ptr, const vllong4& v) { - _mm256_store_si256((__m256i*)ptr,v); - } - - static __forceinline void storeu(void* ptr, const vllong4& v) { - _mm256_storeu_si256((__m256i*)ptr,v); - } - - static __forceinline void storeu(const vboold4& mask, long long* ptr, const vllong4& f) { -#if defined(__AVX512VL__) - _mm256_mask_storeu_epi64(ptr,mask,f); -#else - _mm256_maskstore_pd((double*)ptr,mask,_mm256_castsi256_pd(f)); -#endif - } - - static __forceinline void store(const vboold4& mask, void* ptr, const vllong4& f) { -#if defined(__AVX512VL__) - _mm256_mask_store_epi64(ptr,mask,f); -#else - _mm256_maskstore_pd((double*)ptr,mask,_mm256_castsi256_pd(f)); -#endif - } - - static __forceinline vllong4 broadcast64bit(size_t v) { - return _mm256_set1_epi64x(v); - } - - static __forceinline size_t extract64bit(const vllong4& v) - { - return _mm_cvtsi128_si64(_mm256_castsi256_si128(v)); - } - - - //////////////////////////////////////////////////////////////////////////////// - /// Array Access - //////////////////////////////////////////////////////////////////////////////// - - __forceinline long long& operator [](size_t index) { assert(index < 4); return i[index]; } - __forceinline const long long& operator [](size_t index) const { assert(index < 4); return i[index]; } - - }; - - //////////////////////////////////////////////////////////////////////////////// - /// Select - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vllong4 select(const vboold4& m, const vllong4& t, const vllong4& f) { - #if defined(__AVX512VL__) - return _mm256_mask_blend_epi64(m, f, t); - #else - return _mm256_castpd_si256(_mm256_blendv_pd(_mm256_castsi256_pd(f), _mm256_castsi256_pd(t), m)); - #endif - } - - //////////////////////////////////////////////////////////////////////////////// - /// Unary Operators - //////////////////////////////////////////////////////////////////////////////// - -#if defined(__AVX512VL__) - __forceinline vboold4 asBool(const vllong4& a) { return _mm256_movepi64_mask(a); } -#else - __forceinline vboold4 asBool(const vllong4& a) { return _mm256_castsi256_pd(a); } -#endif - - __forceinline vllong4 operator +(const vllong4& a) { return a; } - __forceinline vllong4 operator -(const vllong4& a) { return _mm256_sub_epi64(_mm256_setzero_si256(), a); } - - //////////////////////////////////////////////////////////////////////////////// - /// Binary Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vllong4 operator +(const vllong4& a, const vllong4& b) { return _mm256_add_epi64(a, b); } - __forceinline vllong4 operator +(const vllong4& a, long long b) { return a + vllong4(b); } - __forceinline vllong4 operator +(long long a, const vllong4& b) { return vllong4(a) + b; } - - __forceinline vllong4 operator -(const vllong4& a, const vllong4& b) { return _mm256_sub_epi64(a, b); } - __forceinline vllong4 operator -(const vllong4& a, long long b) { return a - vllong4(b); } - __forceinline vllong4 operator -(long long a, const vllong4& b) { return vllong4(a) - b; } - - /* only low 32bit part */ - __forceinline vllong4 operator *(const vllong4& a, const vllong4& b) { return _mm256_mul_epi32(a, b); } - __forceinline vllong4 operator *(const vllong4& a, long long b) { return a * vllong4(b); } - __forceinline vllong4 operator *(long long a, const vllong4& b) { return vllong4(a) * b; } - - __forceinline vllong4 operator &(const vllong4& a, const vllong4& b) { return _mm256_and_si256(a, b); } - __forceinline vllong4 operator &(const vllong4& a, long long b) { return a & vllong4(b); } - __forceinline vllong4 operator &(long long a, const vllong4& b) { return vllong4(a) & b; } - - __forceinline vllong4 operator |(const vllong4& a, const vllong4& b) { return _mm256_or_si256(a, b); } - __forceinline vllong4 operator |(const vllong4& a, long long b) { return a | vllong4(b); } - __forceinline vllong4 operator |(long long a, const vllong4& b) { return vllong4(a) | b; } - - __forceinline vllong4 operator ^(const vllong4& a, const vllong4& b) { return _mm256_xor_si256(a, b); } - __forceinline vllong4 operator ^(const vllong4& a, long long b) { return a ^ vllong4(b); } - __forceinline vllong4 operator ^(long long a, const vllong4& b) { return vllong4(a) ^ b; } - - __forceinline vllong4 operator <<(const vllong4& a, long long n) { return _mm256_slli_epi64(a, (int)n); } - //__forceinline vllong4 operator >>(const vllong4& a, long long n) { return _mm256_srai_epi64(a, n); } - - __forceinline vllong4 operator <<(const vllong4& a, const vllong4& n) { return _mm256_sllv_epi64(a, n); } - //__forceinline vllong4 operator >>(const vllong4& a, const vllong4& n) { return _mm256_srav_epi64(a, n); } - //__forceinline vllong4 sra(const vllong4& a, long long b) { return _mm256_srai_epi64(a, b); } - - __forceinline vllong4 srl(const vllong4& a, long long b) { return _mm256_srli_epi64(a, (int)b); } - - //__forceinline vllong4 min(const vllong4& a, const vllong4& b) { return _mm256_min_epi64(a, b); } - //__forceinline vllong4 min(const vllong4& a, long long b) { return min(a,vllong4(b)); } - //__forceinline vllong4 min(long long a, const vllong4& b) { return min(vllong4(a),b); } - - //__forceinline vllong4 max(const vllong4& a, const vllong4& b) { return _mm256_max_epi64(a, b); } - //__forceinline vllong4 max(const vllong4& a, long long b) { return max(a,vllong4(b)); } - //__forceinline vllong4 max(long long a, const vllong4& b) { return max(vllong4(a),b); } - -#if defined(__AVX512VL__) - __forceinline vllong4 mask_and(const vboold4& m, const vllong4& c, const vllong4& a, const vllong4& b) { return _mm256_mask_and_epi64(c,m,a,b); } - __forceinline vllong4 mask_or (const vboold4& m, const vllong4& c, const vllong4& a, const vllong4& b) { return _mm256_mask_or_epi64(c,m,a,b); } -#else - __forceinline vllong4 mask_and(const vboold4& m, const vllong4& c, const vllong4& a, const vllong4& b) { return select(m, a & b, c); } - __forceinline vllong4 mask_or (const vboold4& m, const vllong4& c, const vllong4& a, const vllong4& b) { return select(m, a | b, c); } -#endif - - //////////////////////////////////////////////////////////////////////////////// - /// Assignment Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vllong4& operator +=(vllong4& a, const vllong4& b) { return a = a + b; } - __forceinline vllong4& operator +=(vllong4& a, long long b) { return a = a + b; } - - __forceinline vllong4& operator -=(vllong4& a, const vllong4& b) { return a = a - b; } - __forceinline vllong4& operator -=(vllong4& a, long long b) { return a = a - b; } - - __forceinline vllong4& operator *=(vllong4& a, const vllong4& b) { return a = a * b; } - __forceinline vllong4& operator *=(vllong4& a, long long b) { return a = a * b; } - - __forceinline vllong4& operator &=(vllong4& a, const vllong4& b) { return a = a & b; } - __forceinline vllong4& operator &=(vllong4& a, long long b) { return a = a & b; } - - __forceinline vllong4& operator |=(vllong4& a, const vllong4& b) { return a = a | b; } - __forceinline vllong4& operator |=(vllong4& a, long long b) { return a = a | b; } - - __forceinline vllong4& operator <<=(vllong4& a, long long b) { return a = a << b; } - //__forceinline vllong4& operator >>=(vllong4& a, long long b) { return a = a >> b; } - - //////////////////////////////////////////////////////////////////////////////// - /// Comparison Operators - //////////////////////////////////////////////////////////////////////////////// - -#if defined(__AVX512VL__) - __forceinline vboold4 operator ==(const vllong4& a, const vllong4& b) { return _mm256_cmp_epi64_mask(a,b,_MM_CMPINT_EQ); } - __forceinline vboold4 operator !=(const vllong4& a, const vllong4& b) { return _mm256_cmp_epi64_mask(a,b,_MM_CMPINT_NE); } - __forceinline vboold4 operator < (const vllong4& a, const vllong4& b) { return _mm256_cmp_epi64_mask(a,b,_MM_CMPINT_LT); } - __forceinline vboold4 operator >=(const vllong4& a, const vllong4& b) { return _mm256_cmp_epi64_mask(a,b,_MM_CMPINT_GE); } - __forceinline vboold4 operator > (const vllong4& a, const vllong4& b) { return _mm256_cmp_epi64_mask(a,b,_MM_CMPINT_GT); } - __forceinline vboold4 operator <=(const vllong4& a, const vllong4& b) { return _mm256_cmp_epi64_mask(a,b,_MM_CMPINT_LE); } -#else - __forceinline vboold4 operator ==(const vllong4& a, const vllong4& b) { return _mm256_cmpeq_epi64(a,b); } - __forceinline vboold4 operator !=(const vllong4& a, const vllong4& b) { return !(a == b); } - __forceinline vboold4 operator > (const vllong4& a, const vllong4& b) { return _mm256_cmpgt_epi64(a,b); } - __forceinline vboold4 operator < (const vllong4& a, const vllong4& b) { return _mm256_cmpgt_epi64(b,a); } - __forceinline vboold4 operator >=(const vllong4& a, const vllong4& b) { return !(a < b); } - __forceinline vboold4 operator <=(const vllong4& a, const vllong4& b) { return !(a > b); } -#endif - - __forceinline vboold4 operator ==(const vllong4& a, long long b) { return a == vllong4(b); } - __forceinline vboold4 operator ==(long long a, const vllong4& b) { return vllong4(a) == b; } - - __forceinline vboold4 operator !=(const vllong4& a, long long b) { return a != vllong4(b); } - __forceinline vboold4 operator !=(long long a, const vllong4& b) { return vllong4(a) != b; } - - __forceinline vboold4 operator > (const vllong4& a, long long b) { return a > vllong4(b); } - __forceinline vboold4 operator > (long long a, const vllong4& b) { return vllong4(a) > b; } - - __forceinline vboold4 operator < (const vllong4& a, long long b) { return a < vllong4(b); } - __forceinline vboold4 operator < (long long a, const vllong4& b) { return vllong4(a) < b; } - - __forceinline vboold4 operator >=(const vllong4& a, long long b) { return a >= vllong4(b); } - __forceinline vboold4 operator >=(long long a, const vllong4& b) { return vllong4(a) >= b; } - - __forceinline vboold4 operator <=(const vllong4& a, long long b) { return a <= vllong4(b); } - __forceinline vboold4 operator <=(long long a, const vllong4& b) { return vllong4(a) <= b; } - - __forceinline vboold4 eq(const vllong4& a, const vllong4& b) { return a == b; } - __forceinline vboold4 ne(const vllong4& a, const vllong4& b) { return a != b; } - __forceinline vboold4 lt(const vllong4& a, const vllong4& b) { return a < b; } - __forceinline vboold4 ge(const vllong4& a, const vllong4& b) { return a >= b; } - __forceinline vboold4 gt(const vllong4& a, const vllong4& b) { return a > b; } - __forceinline vboold4 le(const vllong4& a, const vllong4& b) { return a <= b; } - -#if defined(__AVX512VL__) - __forceinline vboold4 eq(const vboold4& mask, const vllong4& a, const vllong4& b) { return _mm256_mask_cmp_epi64_mask(mask, a, b, _MM_CMPINT_EQ); } - __forceinline vboold4 ne(const vboold4& mask, const vllong4& a, const vllong4& b) { return _mm256_mask_cmp_epi64_mask(mask, a, b, _MM_CMPINT_NE); } - __forceinline vboold4 lt(const vboold4& mask, const vllong4& a, const vllong4& b) { return _mm256_mask_cmp_epi64_mask(mask, a, b, _MM_CMPINT_LT); } - __forceinline vboold4 ge(const vboold4& mask, const vllong4& a, const vllong4& b) { return _mm256_mask_cmp_epi64_mask(mask, a, b, _MM_CMPINT_GE); } - __forceinline vboold4 gt(const vboold4& mask, const vllong4& a, const vllong4& b) { return _mm256_mask_cmp_epi64_mask(mask, a, b, _MM_CMPINT_GT); } - __forceinline vboold4 le(const vboold4& mask, const vllong4& a, const vllong4& b) { return _mm256_mask_cmp_epi64_mask(mask, a, b, _MM_CMPINT_LE); } -#else - __forceinline vboold4 eq(const vboold4& mask, const vllong4& a, const vllong4& b) { return mask & (a == b); } - __forceinline vboold4 ne(const vboold4& mask, const vllong4& a, const vllong4& b) { return mask & (a != b); } - __forceinline vboold4 lt(const vboold4& mask, const vllong4& a, const vllong4& b) { return mask & (a < b); } - __forceinline vboold4 ge(const vboold4& mask, const vllong4& a, const vllong4& b) { return mask & (a >= b); } - __forceinline vboold4 gt(const vboold4& mask, const vllong4& a, const vllong4& b) { return mask & (a > b); } - __forceinline vboold4 le(const vboold4& mask, const vllong4& a, const vllong4& b) { return mask & (a <= b); } -#endif - - __forceinline void xchg(const vboold4& m, vllong4& a, vllong4& b) { - const vllong4 c = a; a = select(m,b,a); b = select(m,c,b); - } - - __forceinline vboold4 test(const vllong4& a, const vllong4& b) { -#if defined(__AVX512VL__) - return _mm256_test_epi64_mask(a,b); -#else - return _mm256_testz_si256(a,b); -#endif - } - - //////////////////////////////////////////////////////////////////////////////// - // Movement/Shifting/Shuffling Functions - //////////////////////////////////////////////////////////////////////////////// - - template<int i0, int i1> - __forceinline vllong4 shuffle(const vllong4& v) { - return _mm256_castpd_si256(_mm256_permute_pd(_mm256_castsi256_pd(v), (i1 << 3) | (i0 << 2) | (i1 << 1) | i0)); - } - - template<int i> - __forceinline vllong4 shuffle(const vllong4& v) { - return shuffle<i, i>(v); - } - - template<int i0, int i1> - __forceinline vllong4 shuffle2(const vllong4& v) { - return _mm256_castpd_si256(_mm256_permute2f128_pd(_mm256_castsi256_pd(v), _mm256_castsi256_pd(v), (i1 << 4) | i0)); - } - - __forceinline long long toScalar(const vllong4& v) { - return _mm_cvtsi128_si64(_mm256_castsi256_si128(v)); - } - -#if defined(__AVX512VL__) - __forceinline vllong4 permute(const vllong4& a, const __m256i& index) { - // workaround for GCC 7.x -#if defined(__GNUC__) && !defined(__INTEL_COMPILER) && !defined(__clang__) - return _mm256_permutex2var_epi64(a,index,a); -#else - return _mm256_permutexvar_epi64(index,a); -#endif - } - - __forceinline vllong4 permutex2var(const vllong4& index, const vllong4& a, const vllong4& b) { - return _mm256_permutex2var_epi64(a,index,b); - } - -#endif - //////////////////////////////////////////////////////////////////////////////// - /// Reductions - //////////////////////////////////////////////////////////////////////////////// - - - __forceinline vllong4 vreduce_and2(const vllong4& x) { return x & shuffle<1,0>(x); } - __forceinline vllong4 vreduce_and (const vllong4& y) { const vllong4 x = vreduce_and2(y); return x & shuffle2<1,0>(x); } - - __forceinline vllong4 vreduce_or2(const vllong4& x) { return x | shuffle<1,0>(x); } - __forceinline vllong4 vreduce_or (const vllong4& y) { const vllong4 x = vreduce_or2(y); return x | shuffle2<1,0>(x); } - - __forceinline vllong4 vreduce_add2(const vllong4& x) { return x + shuffle<1,0>(x); } - __forceinline vllong4 vreduce_add (const vllong4& y) { const vllong4 x = vreduce_add2(y); return x + shuffle2<1,0>(x); } - - __forceinline long long reduce_add(const vllong4& a) { return toScalar(vreduce_add(a)); } - __forceinline long long reduce_or (const vllong4& a) { return toScalar(vreduce_or(a)); } - __forceinline long long reduce_and(const vllong4& a) { return toScalar(vreduce_and(a)); } - - //////////////////////////////////////////////////////////////////////////////// - /// Output Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline embree_ostream operator <<(embree_ostream cout, const vllong4& v) - { - cout << "<" << v[0]; - for (size_t i=1; i<4; i++) cout << ", " << v[i]; - cout << ">"; - return cout; - } -} diff --git a/thirdparty/embree-aarch64/common/simd/vllong8_avx512.h b/thirdparty/embree-aarch64/common/simd/vllong8_avx512.h deleted file mode 100644 index 76dddd8991..0000000000 --- a/thirdparty/embree-aarch64/common/simd/vllong8_avx512.h +++ /dev/null @@ -1,381 +0,0 @@ -// Copyright 2009-2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 - -#pragma once - -namespace embree -{ - /* 8-wide AVX-512 64-bit long long type */ - template<> - struct vllong<8> - { - ALIGNED_STRUCT_(64); - - typedef vboold8 Bool; - - enum { size = 8 }; // number of SIMD elements - union { // data - __m512i v; - long long i[8]; - }; - - //////////////////////////////////////////////////////////////////////////////// - /// Constructors, Assignment & Cast Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vllong() {} - __forceinline vllong(const vllong8& t) { v = t.v; } - __forceinline vllong8& operator =(const vllong8& f) { v = f.v; return *this; } - - __forceinline vllong(const __m512i& t) { v = t; } - __forceinline operator __m512i() const { return v; } - __forceinline operator __m256i() const { return _mm512_castsi512_si256(v); } - - __forceinline vllong(long long i) { - v = _mm512_set1_epi64(i); - } - - __forceinline vllong(long long a, long long b, long long c, long long d) { - v = _mm512_set4_epi64(d,c,b,a); - } - - __forceinline vllong(long long a0, long long a1, long long a2, long long a3, - long long a4, long long a5, long long a6, long long a7) - { - v = _mm512_set_epi64(a7,a6,a5,a4,a3,a2,a1,a0); - } - - __forceinline vllong(const vllong<4>& i) { - v = _mm512_broadcast_i64x4(i); - } - - //////////////////////////////////////////////////////////////////////////////// - /// Constants - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vllong(ZeroTy) : v(_mm512_setzero_epi32()) {} - __forceinline vllong(OneTy) : v(_mm512_set1_epi64(1)) {} - __forceinline vllong(StepTy) : v(_mm512_set_epi64(7,6,5,4,3,2,1,0)) {} - __forceinline vllong(ReverseStepTy) : v(_mm512_setr_epi64(7,6,5,4,3,2,1,0)) {} - - //////////////////////////////////////////////////////////////////////////////// - /// Loads and Stores - //////////////////////////////////////////////////////////////////////////////// - - static __forceinline void store_nt(void* __restrict__ ptr, const vllong8& a) { - _mm512_stream_si512((__m512i*)ptr,a); - } - - static __forceinline vllong8 loadu(const void* addr) { - return _mm512_loadu_si512(addr); - } - - static __forceinline vllong8 load(const vllong8* addr) { - return _mm512_load_si512(addr); - } - - static __forceinline vllong8 load(const long long* addr) { - return _mm512_load_si512(addr); - } - - static __forceinline vllong8 load(const uint8_t* ptr) { - return _mm512_cvtepu8_epi64(*(__m128i*)ptr); - } - - static __forceinline void store(void* ptr, const vllong8& v) { - _mm512_store_si512(ptr,v); - } - - static __forceinline void storeu(void* ptr, const vllong8& v) { - _mm512_storeu_si512(ptr,v); - } - - static __forceinline void storeu(const vboold8& mask, long long* ptr, const vllong8& f) { - _mm512_mask_storeu_epi64(ptr,mask,f); - } - - static __forceinline void store(const vboold8& mask, void* addr, const vllong8& v2) { - _mm512_mask_store_epi64(addr,mask,v2); - } - - /* pass by value to avoid compiler generating inefficient code */ - static __forceinline void storeu_compact(const vboold8 mask, void* addr, const vllong8& reg) { - _mm512_mask_compressstoreu_epi64(addr,mask,reg); - } - - static __forceinline vllong8 compact64bit(const vboold8& mask, vllong8& v) { - return _mm512_mask_compress_epi64(v,mask,v); - } - - static __forceinline vllong8 compact64bit(const vboold8& mask, vllong8& dest, const vllong8& source) { - return _mm512_mask_compress_epi64(dest,mask,source); - } - - static __forceinline vllong8 compact(const vboold8& mask, vllong8& v) { - return _mm512_mask_compress_epi64(v,mask,v); - } - - static __forceinline vllong8 compact(const vboold8& mask, const vllong8& a, vllong8& b) { - return _mm512_mask_compress_epi64(a,mask,b); - } - - static __forceinline vllong8 expand(const vboold8& mask, const vllong8& a, vllong8& b) { - return _mm512_mask_expand_epi64(b,mask,a); - } - - static __forceinline vllong8 broadcast64bit(size_t v) { - return _mm512_set1_epi64(v); - } - - static __forceinline size_t extract64bit(const vllong8& v) - { - return _mm_cvtsi128_si64(_mm512_castsi512_si128(v)); - } - - - //////////////////////////////////////////////////////////////////////////////// - /// Array Access - //////////////////////////////////////////////////////////////////////////////// - - __forceinline long long& operator [](size_t index) { assert(index < 8); return i[index]; } - __forceinline const long long& operator [](size_t index) const { assert(index < 8); return i[index]; } - - }; - - //////////////////////////////////////////////////////////////////////////////// - /// Unary Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboold8 asBool(const vllong8& a) { return _mm512_movepi64_mask(a); } - - __forceinline vllong8 operator +(const vllong8& a) { return a; } - __forceinline vllong8 operator -(const vllong8& a) { return _mm512_sub_epi64(_mm512_setzero_epi32(), a); } - - //////////////////////////////////////////////////////////////////////////////// - /// Binary Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vllong8 operator +(const vllong8& a, const vllong8& b) { return _mm512_add_epi64(a, b); } - __forceinline vllong8 operator +(const vllong8& a, long long b) { return a + vllong8(b); } - __forceinline vllong8 operator +(long long a, const vllong8& b) { return vllong8(a) + b; } - - __forceinline vllong8 operator -(const vllong8& a, const vllong8& b) { return _mm512_sub_epi64(a, b); } - __forceinline vllong8 operator -(const vllong8& a, long long b) { return a - vllong8(b); } - __forceinline vllong8 operator -(long long a, const vllong8& b) { return vllong8(a) - b; } - - __forceinline vllong8 operator *(const vllong8& a, const vllong8& b) { return _mm512_mullo_epi64(a, b); } - __forceinline vllong8 operator *(const vllong8& a, long long b) { return a * vllong8(b); } - __forceinline vllong8 operator *(long long a, const vllong8& b) { return vllong8(a) * b; } - - __forceinline vllong8 operator &(const vllong8& a, const vllong8& b) { return _mm512_and_epi64(a, b); } - __forceinline vllong8 operator &(const vllong8& a, long long b) { return a & vllong8(b); } - __forceinline vllong8 operator &(long long a, const vllong8& b) { return vllong8(a) & b; } - - __forceinline vllong8 operator |(const vllong8& a, const vllong8& b) { return _mm512_or_epi64(a, b); } - __forceinline vllong8 operator |(const vllong8& a, long long b) { return a | vllong8(b); } - __forceinline vllong8 operator |(long long a, const vllong8& b) { return vllong8(a) | b; } - - __forceinline vllong8 operator ^(const vllong8& a, const vllong8& b) { return _mm512_xor_epi64(a, b); } - __forceinline vllong8 operator ^(const vllong8& a, long long b) { return a ^ vllong8(b); } - __forceinline vllong8 operator ^(long long a, const vllong8& b) { return vllong8(a) ^ b; } - - __forceinline vllong8 operator <<(const vllong8& a, long long n) { return _mm512_slli_epi64(a, n); } - __forceinline vllong8 operator >>(const vllong8& a, long long n) { return _mm512_srai_epi64(a, n); } - - __forceinline vllong8 operator <<(const vllong8& a, const vllong8& n) { return _mm512_sllv_epi64(a, n); } - __forceinline vllong8 operator >>(const vllong8& a, const vllong8& n) { return _mm512_srav_epi64(a, n); } - - __forceinline vllong8 sll (const vllong8& a, long long b) { return _mm512_slli_epi64(a, b); } - __forceinline vllong8 sra (const vllong8& a, long long b) { return _mm512_srai_epi64(a, b); } - __forceinline vllong8 srl (const vllong8& a, long long b) { return _mm512_srli_epi64(a, b); } - - __forceinline vllong8 min(const vllong8& a, const vllong8& b) { return _mm512_min_epi64(a, b); } - __forceinline vllong8 min(const vllong8& a, long long b) { return min(a,vllong8(b)); } - __forceinline vllong8 min(long long a, const vllong8& b) { return min(vllong8(a),b); } - - __forceinline vllong8 max(const vllong8& a, const vllong8& b) { return _mm512_max_epi64(a, b); } - __forceinline vllong8 max(const vllong8& a, long long b) { return max(a,vllong8(b)); } - __forceinline vllong8 max(long long a, const vllong8& b) { return max(vllong8(a),b); } - - __forceinline vllong8 mask_add(const vboold8& m, const vllong8& c, const vllong8& a, const vllong8& b) { return _mm512_mask_add_epi64(c,m,a,b); } - __forceinline vllong8 mask_sub(const vboold8& m, const vllong8& c, const vllong8& a, const vllong8& b) { return _mm512_mask_sub_epi64(c,m,a,b); } - - __forceinline vllong8 mask_and(const vboold8& m, const vllong8& c, const vllong8& a, const vllong8& b) { return _mm512_mask_and_epi64(c,m,a,b); } - __forceinline vllong8 mask_or (const vboold8& m, const vllong8& c, const vllong8& a, const vllong8& b) { return _mm512_mask_or_epi64(c,m,a,b); } - - //////////////////////////////////////////////////////////////////////////////// - /// Assignment Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vllong8& operator +=(vllong8& a, const vllong8& b) { return a = a + b; } - __forceinline vllong8& operator +=(vllong8& a, long long b) { return a = a + b; } - - __forceinline vllong8& operator -=(vllong8& a, const vllong8& b) { return a = a - b; } - __forceinline vllong8& operator -=(vllong8& a, long long b) { return a = a - b; } - - __forceinline vllong8& operator *=(vllong8& a, const vllong8& b) { return a = a * b; } - __forceinline vllong8& operator *=(vllong8& a, long long b) { return a = a * b; } - - __forceinline vllong8& operator &=(vllong8& a, const vllong8& b) { return a = a & b; } - __forceinline vllong8& operator &=(vllong8& a, long long b) { return a = a & b; } - - __forceinline vllong8& operator |=(vllong8& a, const vllong8& b) { return a = a | b; } - __forceinline vllong8& operator |=(vllong8& a, long long b) { return a = a | b; } - - __forceinline vllong8& operator <<=(vllong8& a, long long b) { return a = a << b; } - __forceinline vllong8& operator >>=(vllong8& a, long long b) { return a = a >> b; } - - //////////////////////////////////////////////////////////////////////////////// - /// Comparison Operators + Select - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboold8 operator ==(const vllong8& a, const vllong8& b) { return _mm512_cmp_epi64_mask(a,b,_MM_CMPINT_EQ); } - __forceinline vboold8 operator ==(const vllong8& a, long long b) { return a == vllong8(b); } - __forceinline vboold8 operator ==(long long a, const vllong8& b) { return vllong8(a) == b; } - - __forceinline vboold8 operator !=(const vllong8& a, const vllong8& b) { return _mm512_cmp_epi64_mask(a,b,_MM_CMPINT_NE); } - __forceinline vboold8 operator !=(const vllong8& a, long long b) { return a != vllong8(b); } - __forceinline vboold8 operator !=(long long a, const vllong8& b) { return vllong8(a) != b; } - - __forceinline vboold8 operator < (const vllong8& a, const vllong8& b) { return _mm512_cmp_epi64_mask(a,b,_MM_CMPINT_LT); } - __forceinline vboold8 operator < (const vllong8& a, long long b) { return a < vllong8(b); } - __forceinline vboold8 operator < (long long a, const vllong8& b) { return vllong8(a) < b; } - - __forceinline vboold8 operator >=(const vllong8& a, const vllong8& b) { return _mm512_cmp_epi64_mask(a,b,_MM_CMPINT_GE); } - __forceinline vboold8 operator >=(const vllong8& a, long long b) { return a >= vllong8(b); } - __forceinline vboold8 operator >=(long long a, const vllong8& b) { return vllong8(a) >= b; } - - __forceinline vboold8 operator > (const vllong8& a, const vllong8& b) { return _mm512_cmp_epi64_mask(a,b,_MM_CMPINT_GT); } - __forceinline vboold8 operator > (const vllong8& a, long long b) { return a > vllong8(b); } - __forceinline vboold8 operator > (long long a, const vllong8& b) { return vllong8(a) > b; } - - __forceinline vboold8 operator <=(const vllong8& a, const vllong8& b) { return _mm512_cmp_epi64_mask(a,b,_MM_CMPINT_LE); } - __forceinline vboold8 operator <=(const vllong8& a, long long b) { return a <= vllong8(b); } - __forceinline vboold8 operator <=(long long a, const vllong8& b) { return vllong8(a) <= b; } - - __forceinline vboold8 eq(const vllong8& a, const vllong8& b) { return _mm512_cmp_epi64_mask(a,b,_MM_CMPINT_EQ); } - __forceinline vboold8 ne(const vllong8& a, const vllong8& b) { return _mm512_cmp_epi64_mask(a,b,_MM_CMPINT_NE); } - __forceinline vboold8 lt(const vllong8& a, const vllong8& b) { return _mm512_cmp_epi64_mask(a,b,_MM_CMPINT_LT); } - __forceinline vboold8 ge(const vllong8& a, const vllong8& b) { return _mm512_cmp_epi64_mask(a,b,_MM_CMPINT_GE); } - __forceinline vboold8 gt(const vllong8& a, const vllong8& b) { return _mm512_cmp_epi64_mask(a,b,_MM_CMPINT_GT); } - __forceinline vboold8 le(const vllong8& a, const vllong8& b) { return _mm512_cmp_epi64_mask(a,b,_MM_CMPINT_LE); } - - __forceinline vboold8 eq(const vboold8 mask, const vllong8& a, const vllong8& b) { return _mm512_mask_cmp_epi64_mask(mask,a,b,_MM_CMPINT_EQ); } - __forceinline vboold8 ne(const vboold8 mask, const vllong8& a, const vllong8& b) { return _mm512_mask_cmp_epi64_mask(mask,a,b,_MM_CMPINT_NE); } - __forceinline vboold8 lt(const vboold8 mask, const vllong8& a, const vllong8& b) { return _mm512_mask_cmp_epi64_mask(mask,a,b,_MM_CMPINT_LT); } - __forceinline vboold8 ge(const vboold8 mask, const vllong8& a, const vllong8& b) { return _mm512_mask_cmp_epi64_mask(mask,a,b,_MM_CMPINT_GE); } - __forceinline vboold8 gt(const vboold8 mask, const vllong8& a, const vllong8& b) { return _mm512_mask_cmp_epi64_mask(mask,a,b,_MM_CMPINT_GT); } - __forceinline vboold8 le(const vboold8 mask, const vllong8& a, const vllong8& b) { return _mm512_mask_cmp_epi64_mask(mask,a,b,_MM_CMPINT_LE); } - - __forceinline vllong8 select(const vboold8& m, const vllong8& t, const vllong8& f) { - return _mm512_mask_or_epi64(f,m,t,t); - } - - __forceinline void xchg(const vboold8& m, vllong8& a, vllong8& b) { - const vllong8 c = a; a = select(m,b,a); b = select(m,c,b); - } - - __forceinline vboold8 test(const vboold8& m, const vllong8& a, const vllong8& b) { - return _mm512_mask_test_epi64_mask(m,a,b); - } - - __forceinline vboold8 test(const vllong8& a, const vllong8& b) { - return _mm512_test_epi64_mask(a,b); - } - - //////////////////////////////////////////////////////////////////////////////// - // Movement/Shifting/Shuffling Functions - //////////////////////////////////////////////////////////////////////////////// - - template<int i0, int i1> - __forceinline vllong8 shuffle(const vllong8& v) { - return _mm512_castpd_si512(_mm512_permute_pd(_mm512_castsi512_pd(v), (i1 << 7) | (i0 << 6) | (i1 << 5) | (i0 << 4) | (i1 << 3) | (i0 << 2) | (i1 << 1) | i0)); - } - - template<int i> - __forceinline vllong8 shuffle(const vllong8& v) { - return shuffle<i, i>(v); - } - - template<int i0, int i1, int i2, int i3> - __forceinline vllong8 shuffle(const vllong8& v) { - return _mm512_permutex_epi64(v, _MM_SHUFFLE(i3, i2, i1, i0)); - } - - template<int i0, int i1> - __forceinline vllong8 shuffle4(const vllong8& v) { - return _mm512_shuffle_i64x2(v, v, _MM_SHUFFLE(i1*2+1, i1*2, i0*2+1, i0*2)); - } - - template<int i> - __forceinline vllong8 shuffle4(const vllong8& v) { - return shuffle4<i, i>(v); - } - - template<int i> - __forceinline vllong8 align_shift_right(const vllong8& a, const vllong8& b) { - return _mm512_alignr_epi64(a, b, i); - }; - - __forceinline long long toScalar(const vllong8& v) { - return _mm_cvtsi128_si64(_mm512_castsi512_si128(v)); - } - - __forceinline vllong8 zeroExtend32Bit(const __m512i& a) { - return _mm512_cvtepu32_epi64(_mm512_castsi512_si256(a)); - } - - //////////////////////////////////////////////////////////////////////////////// - /// Reductions - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vllong8 vreduce_min2(vllong8 x) { return min(x, shuffle<1,0,3,2>(x)); } - __forceinline vllong8 vreduce_min4(vllong8 x) { x = vreduce_min2(x); return min(x, shuffle<2,3,0,1>(x)); } - __forceinline vllong8 vreduce_min (vllong8 x) { x = vreduce_min4(x); return min(x, shuffle4<1,0>(x)); } - - __forceinline vllong8 vreduce_max2(vllong8 x) { return max(x, shuffle<1,0,3,2>(x)); } - __forceinline vllong8 vreduce_max4(vllong8 x) { x = vreduce_max2(x); return max(x, shuffle<2,3,0,1>(x)); } - __forceinline vllong8 vreduce_max (vllong8 x) { x = vreduce_max4(x); return max(x, shuffle4<1,0>(x)); } - - __forceinline vllong8 vreduce_and2(vllong8 x) { return x & shuffle<1,0,3,2>(x); } - __forceinline vllong8 vreduce_and4(vllong8 x) { x = vreduce_and2(x); return x & shuffle<2,3,0,1>(x); } - __forceinline vllong8 vreduce_and (vllong8 x) { x = vreduce_and4(x); return x & shuffle4<1,0>(x); } - - __forceinline vllong8 vreduce_or2(vllong8 x) { return x | shuffle<1,0,3,2>(x); } - __forceinline vllong8 vreduce_or4(vllong8 x) { x = vreduce_or2(x); return x | shuffle<2,3,0,1>(x); } - __forceinline vllong8 vreduce_or (vllong8 x) { x = vreduce_or4(x); return x | shuffle4<1,0>(x); } - - __forceinline vllong8 vreduce_add2(vllong8 x) { return x + shuffle<1,0,3,2>(x); } - __forceinline vllong8 vreduce_add4(vllong8 x) { x = vreduce_add2(x); return x + shuffle<2,3,0,1>(x); } - __forceinline vllong8 vreduce_add (vllong8 x) { x = vreduce_add4(x); return x + shuffle4<1,0>(x); } - - __forceinline long long reduce_min(const vllong8& v) { return toScalar(vreduce_min(v)); } - __forceinline long long reduce_max(const vllong8& v) { return toScalar(vreduce_max(v)); } - __forceinline long long reduce_and(const vllong8& v) { return toScalar(vreduce_and(v)); } - __forceinline long long reduce_or (const vllong8& v) { return toScalar(vreduce_or (v)); } - __forceinline long long reduce_add(const vllong8& v) { return toScalar(vreduce_add(v)); } - - //////////////////////////////////////////////////////////////////////////////// - /// Memory load and store operations - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vllong8 permute(const vllong8& v, const vllong8& index) { - return _mm512_permutexvar_epi64(index,v); - } - - __forceinline vllong8 reverse(const vllong8& a) { - return permute(a,vllong8(reverse_step)); - } - - //////////////////////////////////////////////////////////////////////////////// - /// Output Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline embree_ostream operator <<(embree_ostream cout, const vllong8& v) - { - cout << "<" << v[0]; - for (size_t i=1; i<8; i++) cout << ", " << v[i]; - cout << ">"; - return cout; - } -} diff --git a/thirdparty/embree-aarch64/common/simd/vuint16_avx512.h b/thirdparty/embree-aarch64/common/simd/vuint16_avx512.h deleted file mode 100644 index 39752611bb..0000000000 --- a/thirdparty/embree-aarch64/common/simd/vuint16_avx512.h +++ /dev/null @@ -1,443 +0,0 @@ -// Copyright 2009-2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 - -#pragma once - -namespace embree -{ - /* 16-wide AVX-512 unsigned integer type */ - template<> - struct vuint<16> - { - ALIGNED_STRUCT_(64); - - typedef vboolf16 Bool; - typedef vuint16 UInt; - typedef vfloat16 Float; - - enum { size = 16 }; // number of SIMD elements - union { // data - __m512i v; - unsigned int i[16]; - }; - - //////////////////////////////////////////////////////////////////////////////// - /// Constructors, Assignment & Cast Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vuint() {} - __forceinline vuint(const vuint16& t) { v = t.v; } - __forceinline vuint16& operator =(const vuint16& f) { v = f.v; return *this; } - - __forceinline vuint(const __m512i& t) { v = t; } - __forceinline operator __m512i() const { return v; } - __forceinline operator __m256i() const { return _mm512_castsi512_si256(v); } - - __forceinline vuint(unsigned int i) { - v = _mm512_set1_epi32(i); - } - - __forceinline vuint(const vuint4& i) { - v = _mm512_broadcast_i32x4(i); - } - - __forceinline vuint(const vuint8& i) { - v = _mm512_castps_si512(_mm512_castpd_ps(_mm512_broadcast_f64x4(_mm256_castsi256_pd(i)))); - } - - __forceinline vuint(unsigned int a, unsigned int b, unsigned int c, unsigned int d) { - v = _mm512_set4_epi32(d,c,b,a); - } - - __forceinline vuint(unsigned int a0 , unsigned int a1 , unsigned int a2 , unsigned int a3, - unsigned int a4 , unsigned int a5 , unsigned int a6 , unsigned int a7, - unsigned int a8 , unsigned int a9 , unsigned int a10, unsigned int a11, - unsigned int a12, unsigned int a13, unsigned int a14, unsigned int a15) - { - v = _mm512_set_epi32(a15,a14,a13,a12,a11,a10,a9,a8,a7,a6,a5,a4,a3,a2,a1,a0); - } - - __forceinline explicit vuint(const __m512& f) { - v = _mm512_cvtps_epu32(f); - } - - //////////////////////////////////////////////////////////////////////////////// - /// Constants - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vuint(ZeroTy) : v(_mm512_setzero_epi32()) {} - __forceinline vuint(OneTy) : v(_mm512_set1_epi32(1)) {} - __forceinline vuint(StepTy) : v(_mm512_set_epi32(15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0)) {} - __forceinline vuint(ReverseStepTy) : v(_mm512_setr_epi32(15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0)) {} - - //////////////////////////////////////////////////////////////////////////////// - /// Loads and Stores - //////////////////////////////////////////////////////////////////////////////// - - static __forceinline void store_nt(void* __restrict__ ptr, const vuint16& a) { - _mm512_stream_si512((__m512i*)ptr,a); - } - - static __forceinline vuint16 loadu(const void* addr) - { - return _mm512_loadu_si512(addr); - } - - static __forceinline vuint16 loadu(const uint8_t* ptr) { return _mm512_cvtepu8_epi32(_mm_loadu_si128((__m128i*)ptr)); } - static __forceinline vuint16 loadu(const unsigned short* ptr) { return _mm512_cvtepu16_epi32(_mm256_loadu_si256((__m256i*)ptr)); } - - static __forceinline vuint16 load(const vuint16* addr) { - return _mm512_load_si512(addr); - } - - static __forceinline vuint16 load(const unsigned int* addr) { - return _mm512_load_si512(addr); - } - - static __forceinline vuint16 load(unsigned short* ptr) { return _mm512_cvtepu16_epi32(*(__m256i*)ptr); } - - - static __forceinline void store(void* ptr, const vuint16& v) { - _mm512_store_si512(ptr,v); - } - - static __forceinline void storeu(void* ptr, const vuint16& v) { - _mm512_storeu_si512(ptr,v); - } - - static __forceinline void storeu(const vboolf16& mask, void* ptr, const vuint16& f) { - _mm512_mask_storeu_epi32(ptr,mask,f); - } - - static __forceinline void store(const vboolf16& mask, void* addr, const vuint16& v2) { - _mm512_mask_store_epi32(addr,mask,v2); - } - - /* pass by value to avoid compiler generating inefficient code */ - static __forceinline void storeu_compact(const vboolf16 mask, void* addr, const vuint16 reg) { - _mm512_mask_compressstoreu_epi32(addr,mask,reg); - } - - static __forceinline void storeu_compact_single(const vboolf16 mask, void* addr, vuint16 reg) { - //_mm512_mask_compressstoreu_epi32(addr,mask,reg); - *(float*)addr = mm512_cvtss_f32(_mm512_mask_compress_ps(_mm512_castsi512_ps(reg),mask,_mm512_castsi512_ps(reg))); - } - - static __forceinline vuint16 compact64bit(const vboolf16& mask, vuint16& v) { - return _mm512_mask_compress_epi64(v,mask,v); - } - - static __forceinline vuint16 compact(const vboolf16& mask, vuint16& v) { - return _mm512_mask_compress_epi32(v,mask,v); - } - - static __forceinline vuint16 compact(const vboolf16& mask, const vuint16& a, vuint16& b) { - return _mm512_mask_compress_epi32(a,mask,b); - } - - static __forceinline vuint16 expand(const vboolf16& mask, const vuint16& a, vuint16& b) { - return _mm512_mask_expand_epi32(b,mask,a); - } - - template<int scale = 4> - static __forceinline vuint16 gather(const unsigned int* ptr, const vint16& index) { - return _mm512_i32gather_epi32(index,ptr,scale); - } - - template<int scale = 4> - static __forceinline vuint16 gather(const vboolf16& mask, const unsigned int* ptr, const vint16& index) { - return _mm512_mask_i32gather_epi32(_mm512_undefined_epi32(),mask,index,ptr,scale); - } - - template<int scale = 4> - static __forceinline vuint16 gather(const vboolf16& mask, vuint16& dest, const unsigned int* ptr, const vint16& index) { - return _mm512_mask_i32gather_epi32(dest,mask,index,ptr,scale); - } - - template<int scale = 4> - static __forceinline void scatter(unsigned int* ptr, const vint16& index, const vuint16& v) { - _mm512_i32scatter_epi32((int*)ptr,index,v,scale); - } - - template<int scale = 4> - static __forceinline void scatter(const vboolf16& mask, unsigned int* ptr, const vint16& index, const vuint16& v) { - _mm512_mask_i32scatter_epi32((int*)ptr,mask,index,v,scale); - } - - static __forceinline vuint16 broadcast64bit(size_t v) { - return _mm512_set1_epi64(v); - } - - static __forceinline size_t extract64bit(const vuint16& v) - { - return _mm_cvtsi128_si64(_mm512_castsi512_si128(v)); - } - - //////////////////////////////////////////////////////////////////////////////// - /// Array Access - //////////////////////////////////////////////////////////////////////////////// - - __forceinline unsigned int& operator [](size_t index) { assert(index < 16); return i[index]; } - __forceinline const unsigned int& operator [](size_t index) const { assert(index < 16); return i[index]; } - - __forceinline unsigned int uint (size_t index) const { assert(index < 16); return ((unsigned int*)i)[index]; } - __forceinline size_t& uint64_t(size_t index) const { assert(index < 8); return ((size_t*)i)[index]; } - }; - - //////////////////////////////////////////////////////////////////////////////// - /// Unary Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboolf16 asBool(const vuint16& a) { return _mm512_movepi32_mask(a); } - - __forceinline vuint16 operator +(const vuint16& a) { return a; } - __forceinline vuint16 operator -(const vuint16& a) { return _mm512_sub_epi32(_mm512_setzero_epi32(), a); } - - //////////////////////////////////////////////////////////////////////////////// - /// Binary Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vuint16 operator +(const vuint16& a, const vuint16& b) { return _mm512_add_epi32(a, b); } - __forceinline vuint16 operator +(const vuint16& a, unsigned int b) { return a + vuint16(b); } - __forceinline vuint16 operator +(unsigned int a, const vuint16& b) { return vuint16(a) + b; } - - __forceinline vuint16 operator -(const vuint16& a, const vuint16& b) { return _mm512_sub_epi32(a, b); } - __forceinline vuint16 operator -(const vuint16& a, unsigned int b) { return a - vuint16(b); } - __forceinline vuint16 operator -(unsigned int a, const vuint16& b) { return vuint16(a) - b; } - - __forceinline vuint16 operator *(const vuint16& a, const vuint16& b) { return _mm512_mul_epu32(a, b); } - __forceinline vuint16 operator *(const vuint16& a, unsigned int b) { return a * vuint16(b); } - __forceinline vuint16 operator *(unsigned int a, const vuint16& b) { return vuint16(a) * b; } - - __forceinline vuint16 operator &(const vuint16& a, const vuint16& b) { return _mm512_and_epi32(a, b); } - __forceinline vuint16 operator &(const vuint16& a, unsigned int b) { return a & vuint16(b); } - __forceinline vuint16 operator &(unsigned int a, const vuint16& b) { return vuint16(a) & b; } - - __forceinline vuint16 operator |(const vuint16& a, const vuint16& b) { return _mm512_or_epi32(a, b); } - __forceinline vuint16 operator |(const vuint16& a, unsigned int b) { return a | vuint16(b); } - __forceinline vuint16 operator |(unsigned int a, const vuint16& b) { return vuint16(a) | b; } - - __forceinline vuint16 operator ^(const vuint16& a, const vuint16& b) { return _mm512_xor_epi32(a, b); } - __forceinline vuint16 operator ^(const vuint16& a, unsigned int b) { return a ^ vuint16(b); } - __forceinline vuint16 operator ^(unsigned int a, const vuint16& b) { return vuint16(a) ^ b; } - - __forceinline vuint16 operator <<(const vuint16& a, unsigned int n) { return _mm512_slli_epi32(a, n); } - __forceinline vuint16 operator >>(const vuint16& a, unsigned int n) { return _mm512_srli_epi32(a, n); } - - __forceinline vuint16 operator <<(const vuint16& a, const vuint16& n) { return _mm512_sllv_epi32(a, n); } - __forceinline vuint16 operator >>(const vuint16& a, const vuint16& n) { return _mm512_srlv_epi32(a, n); } - - __forceinline vuint16 sll (const vuint16& a, unsigned int b) { return _mm512_slli_epi32(a, b); } - __forceinline vuint16 sra (const vuint16& a, unsigned int b) { return _mm512_srai_epi32(a, b); } - __forceinline vuint16 srl (const vuint16& a, unsigned int b) { return _mm512_srli_epi32(a, b); } - - __forceinline vuint16 min(const vuint16& a, const vuint16& b) { return _mm512_min_epu32(a, b); } - __forceinline vuint16 min(const vuint16& a, unsigned int b) { return min(a,vuint16(b)); } - __forceinline vuint16 min(unsigned int a, const vuint16& b) { return min(vuint16(a),b); } - - __forceinline vuint16 max(const vuint16& a, const vuint16& b) { return _mm512_max_epu32(a, b); } - __forceinline vuint16 max(const vuint16& a, unsigned int b) { return max(a,vuint16(b)); } - __forceinline vuint16 max(unsigned int a, const vuint16& b) { return max(vuint16(a),b); } - - __forceinline vuint16 mask_add(const vboolf16& mask, vuint16& c, const vuint16& a, const vuint16& b) { return _mm512_mask_add_epi32(c,mask,a,b); } - __forceinline vuint16 mask_sub(const vboolf16& mask, vuint16& c, const vuint16& a, const vuint16& b) { return _mm512_mask_sub_epi32(c,mask,a,b); } - - __forceinline vuint16 mask_and(const vboolf16& m, vuint16& c, const vuint16& a, const vuint16& b) { return _mm512_mask_and_epi32(c,m,a,b); } - __forceinline vuint16 mask_or (const vboolf16& m, vuint16& c, const vuint16& a, const vuint16& b) { return _mm512_mask_or_epi32(c,m,a,b); } - - //////////////////////////////////////////////////////////////////////////////// - /// Assignment Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vuint16& operator +=(vuint16& a, const vuint16& b) { return a = a + b; } - __forceinline vuint16& operator +=(vuint16& a, unsigned int b) { return a = a + b; } - - __forceinline vuint16& operator -=(vuint16& a, const vuint16& b) { return a = a - b; } - __forceinline vuint16& operator -=(vuint16& a, unsigned int b) { return a = a - b; } - - __forceinline vuint16& operator *=(vuint16& a, const vuint16& b) { return a = a * b; } - __forceinline vuint16& operator *=(vuint16& a, unsigned int b) { return a = a * b; } - - __forceinline vuint16& operator &=(vuint16& a, const vuint16& b) { return a = a & b; } - __forceinline vuint16& operator &=(vuint16& a, unsigned int b) { return a = a & b; } - - __forceinline vuint16& operator |=(vuint16& a, const vuint16& b) { return a = a | b; } - __forceinline vuint16& operator |=(vuint16& a, unsigned int b) { return a = a | b; } - - __forceinline vuint16& operator <<=(vuint16& a, unsigned int b) { return a = a << b; } - __forceinline vuint16& operator >>=(vuint16& a, unsigned int b) { return a = a >> b; } - - - //////////////////////////////////////////////////////////////////////////////// - /// Comparison Operators + Select - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboolf16 operator ==(const vuint16& a, const vuint16& b) { return _mm512_cmp_epu32_mask(a,b,_MM_CMPINT_EQ); } - __forceinline vboolf16 operator ==(const vuint16& a, unsigned int b) { return a == vuint16(b); } - __forceinline vboolf16 operator ==(unsigned int a, const vuint16& b) { return vuint16(a) == b; } - - __forceinline vboolf16 operator !=(const vuint16& a, const vuint16& b) { return _mm512_cmp_epu32_mask(a,b,_MM_CMPINT_NE); } - __forceinline vboolf16 operator !=(const vuint16& a, unsigned int b) { return a != vuint16(b); } - __forceinline vboolf16 operator !=(unsigned int a, const vuint16& b) { return vuint16(a) != b; } - - __forceinline vboolf16 operator < (const vuint16& a, const vuint16& b) { return _mm512_cmp_epu32_mask(a,b,_MM_CMPINT_LT); } - __forceinline vboolf16 operator < (const vuint16& a, unsigned int b) { return a < vuint16(b); } - __forceinline vboolf16 operator < (unsigned int a, const vuint16& b) { return vuint16(a) < b; } - - __forceinline vboolf16 operator >=(const vuint16& a, const vuint16& b) { return _mm512_cmp_epu32_mask(a,b,_MM_CMPINT_GE); } - __forceinline vboolf16 operator >=(const vuint16& a, unsigned int b) { return a >= vuint16(b); } - __forceinline vboolf16 operator >=(unsigned int a, const vuint16& b) { return vuint16(a) >= b; } - - __forceinline vboolf16 operator > (const vuint16& a, const vuint16& b) { return _mm512_cmp_epu32_mask(a,b,_MM_CMPINT_GT); } - __forceinline vboolf16 operator > (const vuint16& a, unsigned int b) { return a > vuint16(b); } - __forceinline vboolf16 operator > (unsigned int a, const vuint16& b) { return vuint16(a) > b; } - - __forceinline vboolf16 operator <=(const vuint16& a, const vuint16& b) { return _mm512_cmp_epu32_mask(a,b,_MM_CMPINT_LE); } - __forceinline vboolf16 operator <=(const vuint16& a, unsigned int b) { return a <= vuint16(b); } - __forceinline vboolf16 operator <=(unsigned int a, const vuint16& b) { return vuint16(a) <= b; } - - __forceinline vboolf16 eq(const vuint16& a, const vuint16& b) { return _mm512_cmp_epu32_mask(a,b,_MM_CMPINT_EQ); } - __forceinline vboolf16 ne(const vuint16& a, const vuint16& b) { return _mm512_cmp_epu32_mask(a,b,_MM_CMPINT_NE); } - __forceinline vboolf16 lt(const vuint16& a, const vuint16& b) { return _mm512_cmp_epu32_mask(a,b,_MM_CMPINT_LT); } - __forceinline vboolf16 ge(const vuint16& a, const vuint16& b) { return _mm512_cmp_epu32_mask(a,b,_MM_CMPINT_GE); } - __forceinline vboolf16 gt(const vuint16& a, const vuint16& b) { return _mm512_cmp_epu32_mask(a,b,_MM_CMPINT_GT); } - __forceinline vboolf16 le(const vuint16& a, const vuint16& b) { return _mm512_cmp_epu32_mask(a,b,_MM_CMPINT_LE); } - - __forceinline vboolf16 eq(const vboolf16 mask, const vuint16& a, const vuint16& b) { return _mm512_mask_cmp_epu32_mask(mask,a,b,_MM_CMPINT_EQ); } - __forceinline vboolf16 ne(const vboolf16 mask, const vuint16& a, const vuint16& b) { return _mm512_mask_cmp_epu32_mask(mask,a,b,_MM_CMPINT_NE); } - __forceinline vboolf16 lt(const vboolf16 mask, const vuint16& a, const vuint16& b) { return _mm512_mask_cmp_epu32_mask(mask,a,b,_MM_CMPINT_LT); } - __forceinline vboolf16 ge(const vboolf16 mask, const vuint16& a, const vuint16& b) { return _mm512_mask_cmp_epu32_mask(mask,a,b,_MM_CMPINT_GE); } - __forceinline vboolf16 gt(const vboolf16 mask, const vuint16& a, const vuint16& b) { return _mm512_mask_cmp_epu32_mask(mask,a,b,_MM_CMPINT_GT); } - __forceinline vboolf16 le(const vboolf16 mask, const vuint16& a, const vuint16& b) { return _mm512_mask_cmp_epu32_mask(mask,a,b,_MM_CMPINT_LE); } - - - __forceinline vuint16 select(const vboolf16& m, const vuint16& t, const vuint16& f) { - return _mm512_mask_or_epi32(f,m,t,t); - } - - __forceinline void xchg(const vboolf16& m, vuint16& a, vuint16& b) { - const vuint16 c = a; a = select(m,b,a); b = select(m,c,b); - } - - __forceinline vboolf16 test(const vboolf16& m, const vuint16& a, const vuint16& b) { - return _mm512_mask_test_epi32_mask(m,a,b); - } - - __forceinline vboolf16 test(const vuint16& a, const vuint16& b) { - return _mm512_test_epi32_mask(a,b); - } - - //////////////////////////////////////////////////////////////////////////////// - // Movement/Shifting/Shuffling Functions - //////////////////////////////////////////////////////////////////////////////// - - template<int i> - __forceinline vuint16 shuffle(const vuint16& v) { - return _mm512_castps_si512(_mm512_permute_ps(_mm512_castsi512_ps(v), _MM_SHUFFLE(i, i, i, i))); - } - - template<int i0, int i1, int i2, int i3> - __forceinline vuint16 shuffle(const vuint16& v) { - return _mm512_castps_si512(_mm512_permute_ps(_mm512_castsi512_ps(v), _MM_SHUFFLE(i3, i2, i1, i0))); - } - - template<int i> - __forceinline vuint16 shuffle4(const vuint16& v) { - return _mm512_castps_si512(_mm512_shuffle_f32x4(_mm512_castsi512_ps(v), _mm512_castsi512_ps(v) ,_MM_SHUFFLE(i, i, i, i))); - } - - template<int i0, int i1, int i2, int i3> - __forceinline vuint16 shuffle4(const vuint16& v) { - return _mm512_castps_si512(_mm512_shuffle_f32x4(_mm512_castsi512_ps(v), _mm512_castsi512_ps(v), _MM_SHUFFLE(i3, i2, i1, i0))); - } - - template<int i> - __forceinline vuint16 align_shift_right(const vuint16& a, const vuint16& b) { - return _mm512_alignr_epi32(a, b, i); - }; - - __forceinline unsigned int toScalar(const vuint16& v) { - return _mm_cvtsi128_si32(_mm512_castsi512_si128(v)); - } - - //////////////////////////////////////////////////////////////////////////////// - /// Reductions - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vuint16 vreduce_min2(vuint16 x) { return min(x, shuffle<1,0,3,2>(x)); } - __forceinline vuint16 vreduce_min4(vuint16 x) { x = vreduce_min2(x); return min(x, shuffle<2,3,0,1>(x)); } - __forceinline vuint16 vreduce_min8(vuint16 x) { x = vreduce_min4(x); return min(x, shuffle4<1,0,3,2>(x)); } - __forceinline vuint16 vreduce_min (vuint16 x) { x = vreduce_min8(x); return min(x, shuffle4<2,3,0,1>(x)); } - - __forceinline vuint16 vreduce_max2(vuint16 x) { return max(x, shuffle<1,0,3,2>(x)); } - __forceinline vuint16 vreduce_max4(vuint16 x) { x = vreduce_max2(x); return max(x, shuffle<2,3,0,1>(x)); } - __forceinline vuint16 vreduce_max8(vuint16 x) { x = vreduce_max4(x); return max(x, shuffle4<1,0,3,2>(x)); } - __forceinline vuint16 vreduce_max (vuint16 x) { x = vreduce_max8(x); return max(x, shuffle4<2,3,0,1>(x)); } - - __forceinline vuint16 vreduce_and2(vuint16 x) { return x & shuffle<1,0,3,2>(x); } - __forceinline vuint16 vreduce_and4(vuint16 x) { x = vreduce_and2(x); return x & shuffle<2,3,0,1>(x); } - __forceinline vuint16 vreduce_and8(vuint16 x) { x = vreduce_and4(x); return x & shuffle4<1,0,3,2>(x); } - __forceinline vuint16 vreduce_and (vuint16 x) { x = vreduce_and8(x); return x & shuffle4<2,3,0,1>(x); } - - __forceinline vuint16 vreduce_or2(vuint16 x) { return x | shuffle<1,0,3,2>(x); } - __forceinline vuint16 vreduce_or4(vuint16 x) { x = vreduce_or2(x); return x | shuffle<2,3,0,1>(x); } - __forceinline vuint16 vreduce_or8(vuint16 x) { x = vreduce_or4(x); return x | shuffle4<1,0,3,2>(x); } - __forceinline vuint16 vreduce_or (vuint16 x) { x = vreduce_or8(x); return x | shuffle4<2,3,0,1>(x); } - - __forceinline vuint16 vreduce_add2(vuint16 x) { return x + shuffle<1,0,3,2>(x); } - __forceinline vuint16 vreduce_add4(vuint16 x) { x = vreduce_add2(x); return x + shuffle<2,3,0,1>(x); } - __forceinline vuint16 vreduce_add8(vuint16 x) { x = vreduce_add4(x); return x + shuffle4<1,0,3,2>(x); } - __forceinline vuint16 vreduce_add (vuint16 x) { x = vreduce_add8(x); return x + shuffle4<2,3,0,1>(x); } - - __forceinline unsigned int reduce_min(const vuint16& v) { return toScalar(vreduce_min(v)); } - __forceinline unsigned int reduce_max(const vuint16& v) { return toScalar(vreduce_max(v)); } - __forceinline unsigned int reduce_and(const vuint16& v) { return toScalar(vreduce_and(v)); } - __forceinline unsigned int reduce_or (const vuint16& v) { return toScalar(vreduce_or (v)); } - __forceinline unsigned int reduce_add(const vuint16& v) { return toScalar(vreduce_add(v)); } - - //////////////////////////////////////////////////////////////////////////////// - /// Memory load and store operations - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vuint16 permute(vuint16 v, vuint16 index) { - return _mm512_permutexvar_epi32(index,v); - } - - __forceinline vuint16 reverse(const vuint16& a) { - return permute(a,vuint16(reverse_step)); - } - - __forceinline vuint16 prefix_sum(const vuint16& a) - { - const vuint16 z(zero); - vuint16 v = a; - v = v + align_shift_right<16-1>(v,z); - v = v + align_shift_right<16-2>(v,z); - v = v + align_shift_right<16-4>(v,z); - v = v + align_shift_right<16-8>(v,z); - return v; - } - - __forceinline vuint16 reverse_prefix_sum(const vuint16& a) - { - const vuint16 z(zero); - vuint16 v = a; - v = v + align_shift_right<1>(z,v); - v = v + align_shift_right<2>(z,v); - v = v + align_shift_right<4>(z,v); - v = v + align_shift_right<8>(z,v); - return v; - } - - //////////////////////////////////////////////////////////////////////////////// - /// Output Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline embree_ostream operator <<(embree_ostream cout, const vuint16& v) - { - cout << "<" << v[0]; - for (int i=1; i<16; i++) cout << ", " << v[i]; - cout << ">"; - return cout; - } -} diff --git a/thirdparty/embree-aarch64/common/simd/vuint4_sse2.h b/thirdparty/embree-aarch64/common/simd/vuint4_sse2.h deleted file mode 100644 index a3f393ebf2..0000000000 --- a/thirdparty/embree-aarch64/common/simd/vuint4_sse2.h +++ /dev/null @@ -1,499 +0,0 @@ -// Copyright 2009-2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 - -#pragma once - -#include "../math/math.h" - -namespace embree -{ - /* 4-wide SSE integer type */ - template<> - struct vuint<4> - { - ALIGNED_STRUCT_(16); - - typedef vboolf4 Bool; - typedef vuint4 Int; - typedef vfloat4 Float; - - enum { size = 4 }; // number of SIMD elements - union { __m128i v; unsigned int i[4]; }; // data - - //////////////////////////////////////////////////////////////////////////////// - /// Constructors, Assignment & Cast Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vuint() {} - __forceinline vuint(const vuint4& a) { v = a.v; } - __forceinline vuint4& operator =(const vuint4& a) { v = a.v; return *this; } - - __forceinline vuint(const __m128i a) : v(a) {} - __forceinline operator const __m128i&() const { return v; } - __forceinline operator __m128i&() { return v; } - - - __forceinline vuint(unsigned int a) : v(_mm_set1_epi32(a)) {} - __forceinline vuint(unsigned int a, unsigned int b, unsigned int c, unsigned int d) : v(_mm_set_epi32(d, c, b, a)) {} - -#if defined(__AVX512VL__) - __forceinline explicit vuint(__m128 a) : v(_mm_cvtps_epu32(a)) {} -#endif - -#if defined(__AVX512VL__) - __forceinline explicit vuint(const vboolf4& a) : v(_mm_movm_epi32(a)) {} -#else - __forceinline explicit vuint(const vboolf4& a) : v(_mm_castps_si128((__m128)a)) {} -#endif - - //////////////////////////////////////////////////////////////////////////////// - /// Constants - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vuint(ZeroTy) : v(_mm_setzero_si128()) {} - __forceinline vuint(OneTy) : v(_mm_set1_epi32(1)) {} - __forceinline vuint(PosInfTy) : v(_mm_set1_epi32(unsigned(pos_inf))) {} - __forceinline vuint(StepTy) : v(_mm_set_epi32(3, 2, 1, 0)) {} - __forceinline vuint(TrueTy) { v = _mm_cmpeq_epi32(v,v); } - __forceinline vuint(UndefinedTy) : v(_mm_castps_si128(_mm_undefined_ps())) {} - - //////////////////////////////////////////////////////////////////////////////// - /// Loads and Stores - //////////////////////////////////////////////////////////////////////////////// - - static __forceinline vuint4 load (const void* a) { return _mm_load_si128((__m128i*)a); } - static __forceinline vuint4 loadu(const void* a) { return _mm_loadu_si128((__m128i*)a); } - - static __forceinline void store (void* ptr, const vuint4& v) { _mm_store_si128((__m128i*)ptr,v); } - static __forceinline void storeu(void* ptr, const vuint4& v) { _mm_storeu_si128((__m128i*)ptr,v); } - -#if defined(__AVX512VL__) - static __forceinline vuint4 load (const vboolf4& mask, const void* ptr) { return _mm_mask_load_epi32 (_mm_setzero_si128(),mask,ptr); } - static __forceinline vuint4 loadu(const vboolf4& mask, const void* ptr) { return _mm_mask_loadu_epi32(_mm_setzero_si128(),mask,ptr); } - - static __forceinline void store (const vboolf4& mask, void* ptr, const vuint4& v) { _mm_mask_store_epi32 (ptr,mask,v); } - static __forceinline void storeu(const vboolf4& mask, void* ptr, const vuint4& v) { _mm_mask_storeu_epi32(ptr,mask,v); } -#elif defined(__AVX__) - static __forceinline vuint4 load (const vbool4& mask, const void* a) { return _mm_castps_si128(_mm_maskload_ps((float*)a,mask)); } - static __forceinline vuint4 loadu(const vbool4& mask, const void* a) { return _mm_castps_si128(_mm_maskload_ps((float*)a,mask)); } - - static __forceinline void store (const vboolf4& mask, void* ptr, const vuint4& i) { _mm_maskstore_ps((float*)ptr,(__m128i)mask,_mm_castsi128_ps(i)); } - static __forceinline void storeu(const vboolf4& mask, void* ptr, const vuint4& i) { _mm_maskstore_ps((float*)ptr,(__m128i)mask,_mm_castsi128_ps(i)); } -#else - static __forceinline vuint4 load (const vbool4& mask, const void* a) { return _mm_and_si128(_mm_load_si128 ((__m128i*)a),mask); } - static __forceinline vuint4 loadu(const vbool4& mask, const void* a) { return _mm_and_si128(_mm_loadu_si128((__m128i*)a),mask); } - - static __forceinline void store (const vboolf4& mask, void* ptr, const vuint4& i) { store (ptr,select(mask,i,load (ptr))); } - static __forceinline void storeu(const vboolf4& mask, void* ptr, const vuint4& i) { storeu(ptr,select(mask,i,loadu(ptr))); } -#endif - -#if defined(__aarch64__) - static __forceinline vuint4 load(const uint8_t* ptr) { - return _mm_load4epu8_epi32(((__m128i*)ptr)); - } - static __forceinline vuint4 loadu(const uint8_t* ptr) { - return _mm_load4epu8_epi32(((__m128i*)ptr)); - } -#elif defined(__SSE4_1__) - static __forceinline vuint4 load(const uint8_t* ptr) { - return _mm_cvtepu8_epi32(_mm_loadl_epi64((__m128i*)ptr)); - } - - static __forceinline vuint4 loadu(const uint8_t* ptr) { - return _mm_cvtepu8_epi32(_mm_loadl_epi64((__m128i*)ptr)); - } - -#endif - - static __forceinline vuint4 load(const unsigned short* ptr) { -#if defined(__aarch64__) - return _mm_load4epu16_epi32(((__m128i*)ptr)); -#elif defined (__SSE4_1__) - return _mm_cvtepu16_epi32(_mm_loadu_si128((__m128i*)ptr)); -#else - return vuint4(ptr[0],ptr[1],ptr[2],ptr[3]); -#endif - } - - static __forceinline void store_uint8(uint8_t* ptr, const vuint4& v) { -#if defined(__aarch64__) - uint32x4_t x = uint32x4_t(v.v); - uint16x4_t y = vqmovn_u32(x); - uint8x8_t z = vqmovn_u16(vcombine_u16(y, y)); - vst1_lane_u32((uint32_t *)ptr, uint32x2_t(z), 0); -#elif defined(__SSE4_1__) - __m128i x = v; - x = _mm_packus_epi32(x, x); - x = _mm_packus_epi16(x, x); - *(unsigned*)ptr = _mm_cvtsi128_si32(x); -#else - for (size_t i=0;i<4;i++) - ptr[i] = (uint8_t)v[i]; -#endif - } - - static __forceinline void store_uint8(unsigned short* ptr, const vuint4& v) { -#if defined(__aarch64__) - uint32x4_t x = (uint32x4_t)v.v; - uint16x4_t y = vqmovn_u32(x); - vst1_u16(ptr, y); -#else - for (size_t i=0;i<4;i++) - ptr[i] = (unsigned short)v[i]; -#endif - } - - static __forceinline vuint4 load_nt(void* ptr) { -#if (defined(__aarch64__)) || defined(__SSE4_1__) - return _mm_stream_load_si128((__m128i*)ptr); -#else - return _mm_load_si128((__m128i*)ptr); -#endif - } - - static __forceinline void store_nt(void* ptr, const vuint4& v) { -#if !defined(__aarch64__) && defined(__SSE4_1__) - _mm_stream_ps((float*)ptr, _mm_castsi128_ps(v)); -#else - _mm_store_si128((__m128i*)ptr,v); -#endif - } - - template<int scale = 4> - static __forceinline vuint4 gather(const unsigned int* ptr, const vint4& index) { -#if defined(__AVX2__) && !defined(__aarch64__) - return _mm_i32gather_epi32((const int*)ptr, index, scale); -#else - return vuint4( - *(unsigned int*)(((int8_t*)ptr)+scale*index[0]), - *(unsigned int*)(((int8_t*)ptr)+scale*index[1]), - *(unsigned int*)(((int8_t*)ptr)+scale*index[2]), - *(unsigned int*)(((int8_t*)ptr)+scale*index[3])); -#endif - } - - template<int scale = 4> - static __forceinline vuint4 gather(const vboolf4& mask, const unsigned int* ptr, const vint4& index) { - vuint4 r = zero; -#if defined(__AVX512VL__) - return _mm_mmask_i32gather_epi32(r, mask, index, ptr, scale); -#elif defined(__AVX2__) && !defined(__aarch64__) - return _mm_mask_i32gather_epi32(r, (const int*)ptr, index, mask, scale); -#else - if (likely(mask[0])) r[0] = *(unsigned int*)(((int8_t*)ptr)+scale*index[0]); - if (likely(mask[1])) r[1] = *(unsigned int*)(((int8_t*)ptr)+scale*index[1]); - if (likely(mask[2])) r[2] = *(unsigned int*)(((int8_t*)ptr)+scale*index[2]); - if (likely(mask[3])) r[3] = *(unsigned int*)(((int8_t*)ptr)+scale*index[3]); - return r; -#endif - } - - //////////////////////////////////////////////////////////////////////////////// - /// Array Access - //////////////////////////////////////////////////////////////////////////////// - - __forceinline const unsigned int& operator [](size_t index) const { assert(index < 4); return i[index]; } - __forceinline unsigned int& operator [](size_t index) { assert(index < 4); return i[index]; } - - friend __forceinline vuint4 select(const vboolf4& m, const vuint4& t, const vuint4& f) { -#if defined(__AVX512VL__) - return _mm_mask_blend_epi32(m, (__m128i)f, (__m128i)t); -#elif defined(__SSE4_1__) - return _mm_castps_si128(_mm_blendv_ps(_mm_castsi128_ps(f), _mm_castsi128_ps(t), m)); -#else - return _mm_or_si128(_mm_and_si128(m, t), _mm_andnot_si128(m, f)); -#endif - } - }; - - //////////////////////////////////////////////////////////////////////////////// - /// Unary Operators - //////////////////////////////////////////////////////////////////////////////// - -#if defined(__AVX512VL__) - __forceinline vboolf4 asBool(const vuint4& a) { return _mm_movepi32_mask(a); } -#else - __forceinline vboolf4 asBool(const vuint4& a) { return _mm_castsi128_ps(a); } -#endif - - __forceinline vuint4 operator +(const vuint4& a) { return a; } - __forceinline vuint4 operator -(const vuint4& a) { return _mm_sub_epi32(_mm_setzero_si128(), a); } - - //////////////////////////////////////////////////////////////////////////////// - /// Binary Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vuint4 operator +(const vuint4& a, const vuint4& b) { return _mm_add_epi32(a, b); } - __forceinline vuint4 operator +(const vuint4& a, unsigned int b) { return a + vuint4(b); } - __forceinline vuint4 operator +(unsigned int a, const vuint4& b) { return vuint4(a) + b; } - - __forceinline vuint4 operator -(const vuint4& a, const vuint4& b) { return _mm_sub_epi32(a, b); } - __forceinline vuint4 operator -(const vuint4& a, unsigned int b) { return a - vuint4(b); } - __forceinline vuint4 operator -(unsigned int a, const vuint4& b) { return vuint4(a) - b; } - -//#if defined(__SSE4_1__) -// __forceinline vuint4 operator *(const vuint4& a, const vuint4& b) { return _mm_mullo_epu32(a, b); } -//#else -// __forceinline vuint4 operator *(const vuint4& a, const vuint4& b) { return vuint4(a[0]*b[0],a[1]*b[1],a[2]*b[2],a[3]*b[3]); } -//#endif -// __forceinline vuint4 operator *(const vuint4& a, unsigned int b) { return a * vuint4(b); } -// __forceinline vuint4 operator *(unsigned int a, const vuint4& b) { return vuint4(a) * b; } - - __forceinline vuint4 operator &(const vuint4& a, const vuint4& b) { return _mm_and_si128(a, b); } - __forceinline vuint4 operator &(const vuint4& a, unsigned int b) { return a & vuint4(b); } - __forceinline vuint4 operator &(unsigned int a, const vuint4& b) { return vuint4(a) & b; } - - __forceinline vuint4 operator |(const vuint4& a, const vuint4& b) { return _mm_or_si128(a, b); } - __forceinline vuint4 operator |(const vuint4& a, unsigned int b) { return a | vuint4(b); } - __forceinline vuint4 operator |(unsigned int a, const vuint4& b) { return vuint4(a) | b; } - - __forceinline vuint4 operator ^(const vuint4& a, const vuint4& b) { return _mm_xor_si128(a, b); } - __forceinline vuint4 operator ^(const vuint4& a, unsigned int b) { return a ^ vuint4(b); } - __forceinline vuint4 operator ^(unsigned int a, const vuint4& b) { return vuint4(a) ^ b; } - - __forceinline vuint4 operator <<(const vuint4& a, unsigned int n) { return _mm_slli_epi32(a, n); } - __forceinline vuint4 operator >>(const vuint4& a, unsigned int n) { return _mm_srli_epi32(a, n); } - - __forceinline vuint4 sll (const vuint4& a, unsigned int b) { return _mm_slli_epi32(a, b); } - __forceinline vuint4 sra (const vuint4& a, unsigned int b) { return _mm_srai_epi32(a, b); } - __forceinline vuint4 srl (const vuint4& a, unsigned int b) { return _mm_srli_epi32(a, b); } - - //////////////////////////////////////////////////////////////////////////////// - /// Assignment Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vuint4& operator +=(vuint4& a, const vuint4& b) { return a = a + b; } - __forceinline vuint4& operator +=(vuint4& a, unsigned int b) { return a = a + b; } - - __forceinline vuint4& operator -=(vuint4& a, const vuint4& b) { return a = a - b; } - __forceinline vuint4& operator -=(vuint4& a, unsigned int b) { return a = a - b; } - -//#if defined(__SSE4_1__) -// __forceinline vuint4& operator *=(vuint4& a, const vuint4& b) { return a = a * b; } -// __forceinline vuint4& operator *=(vuint4& a, unsigned int b) { return a = a * b; } -//#endif - - __forceinline vuint4& operator &=(vuint4& a, const vuint4& b) { return a = a & b; } - __forceinline vuint4& operator &=(vuint4& a, unsigned int b) { return a = a & b; } - - __forceinline vuint4& operator |=(vuint4& a, const vuint4& b) { return a = a | b; } - __forceinline vuint4& operator |=(vuint4& a, unsigned int b) { return a = a | b; } - - __forceinline vuint4& operator <<=(vuint4& a, unsigned int b) { return a = a << b; } - __forceinline vuint4& operator >>=(vuint4& a, unsigned int b) { return a = a >> b; } - - //////////////////////////////////////////////////////////////////////////////// - /// Comparison Operators + Select - //////////////////////////////////////////////////////////////////////////////// - -#if defined(__AVX512VL__) - __forceinline vboolf4 operator ==(const vuint4& a, const vuint4& b) { return _mm_cmp_epu32_mask(a,b,_MM_CMPINT_EQ); } - __forceinline vboolf4 operator !=(const vuint4& a, const vuint4& b) { return _mm_cmp_epu32_mask(a,b,_MM_CMPINT_NE); } - //__forceinline vboolf4 operator < (const vuint4& a, const vuint4& b) { return _mm_cmp_epu32_mask(a,b,_MM_CMPINT_LT); } - //__forceinline vboolf4 operator >=(const vuint4& a, const vuint4& b) { return _mm_cmp_epu32_mask(a,b,_MM_CMPINT_GE); } - //__forceinline vboolf4 operator > (const vuint4& a, const vuint4& b) { return _mm_cmp_epu32_mask(a,b,_MM_CMPINT_GT); } - //__forceinline vboolf4 operator <=(const vuint4& a, const vuint4& b) { return _mm_cmp_epu32_mask(a,b,_MM_CMPINT_LE); } -#else - __forceinline vboolf4 operator ==(const vuint4& a, const vuint4& b) { return _mm_castsi128_ps(_mm_cmpeq_epi32(a, b)); } - __forceinline vboolf4 operator !=(const vuint4& a, const vuint4& b) { return !(a == b); } - //__forceinline vboolf4 operator < (const vuint4& a, const vuint4& b) { return _mm_castsi128_ps(_mm_cmplt_epu32(a, b)); } - //__forceinline vboolf4 operator >=(const vuint4& a, const vuint4& b) { return !(a < b); } - //__forceinline vboolf4 operator > (const vuint4& a, const vuint4& b) { return _mm_castsi128_ps(_mm_cmpgt_epu32(a, b)); } - //__forceinline vboolf4 operator <=(const vuint4& a, const vuint4& b) { return !(a > b); } -#endif - - __forceinline vboolf4 operator ==(const vuint4& a, unsigned int b) { return a == vuint4(b); } - __forceinline vboolf4 operator ==(unsigned int a, const vuint4& b) { return vuint4(a) == b; } - - __forceinline vboolf4 operator !=(const vuint4& a, unsigned int b) { return a != vuint4(b); } - __forceinline vboolf4 operator !=(unsigned int a, const vuint4& b) { return vuint4(a) != b; } - - //__forceinline vboolf4 operator < (const vuint4& a, unsigned int b) { return a < vuint4(b); } - //__forceinline vboolf4 operator < (unsigned int a, const vuint4& b) { return vuint4(a) < b; } - - //__forceinline vboolf4 operator >=(const vuint4& a, unsigned int b) { return a >= vuint4(b); } - //__forceinline vboolf4 operator >=(unsigned int a, const vuint4& b) { return vuint4(a) >= b; } - - //__forceinline vboolf4 operator > (const vuint4& a, unsigned int b) { return a > vuint4(b); } - //__forceinline vboolf4 operator > (unsigned int a, const vuint4& b) { return vuint4(a) > b; } - - //__forceinline vboolf4 operator <=(const vuint4& a, unsigned int b) { return a <= vuint4(b); } - //__forceinline vboolf4 operator <=(unsigned int a, const vuint4& b) { return vuint4(a) <= b; } - - __forceinline vboolf4 eq(const vuint4& a, const vuint4& b) { return a == b; } - __forceinline vboolf4 ne(const vuint4& a, const vuint4& b) { return a != b; } - //__forceinline vboolf4 lt(const vuint4& a, const vuint4& b) { return a < b; } - //__forceinline vboolf4 ge(const vuint4& a, const vuint4& b) { return a >= b; } - //__forceinline vboolf4 gt(const vuint4& a, const vuint4& b) { return a > b; } - //__forceinline vboolf4 le(const vuint4& a, const vuint4& b) { return a <= b; } - -#if defined(__AVX512VL__) - __forceinline vboolf4 eq(const vboolf4& mask, const vuint4& a, const vuint4& b) { return _mm_mask_cmp_epu32_mask(mask, a, b, _MM_CMPINT_EQ); } - __forceinline vboolf4 ne(const vboolf4& mask, const vuint4& a, const vuint4& b) { return _mm_mask_cmp_epu32_mask(mask, a, b, _MM_CMPINT_NE); } - //__forceinline vboolf4 lt(const vboolf4& mask, const vuint4& a, const vuint4& b) { return _mm_mask_cmp_epu32_mask(mask, a, b, _MM_CMPINT_LT); } - //__forceinline vboolf4 ge(const vboolf4& mask, const vuint4& a, const vuint4& b) { return _mm_mask_cmp_epu32_mask(mask, a, b, _MM_CMPINT_GE); } - //__forceinline vboolf4 gt(const vboolf4& mask, const vuint4& a, const vuint4& b) { return _mm_mask_cmp_epu32_mask(mask, a, b, _MM_CMPINT_GT); } - //__forceinline vboolf4 le(const vboolf4& mask, const vuint4& a, const vuint4& b) { return _mm_mask_cmp_epu32_mask(mask, a, b, _MM_CMPINT_LE); } -#else - __forceinline vboolf4 eq(const vboolf4& mask, const vuint4& a, const vuint4& b) { return mask & (a == b); } - __forceinline vboolf4 ne(const vboolf4& mask, const vuint4& a, const vuint4& b) { return mask & (a != b); } - //__forceinline vboolf4 lt(const vboolf4& mask, const vuint4& a, const vuint4& b) { return mask & (a < b); } - //__forceinline vboolf4 ge(const vboolf4& mask, const vuint4& a, const vuint4& b) { return mask & (a >= b); } - //__forceinline vboolf4 gt(const vboolf4& mask, const vuint4& a, const vuint4& b) { return mask & (a > b); } - //__forceinline vboolf4 le(const vboolf4& mask, const vuint4& a, const vuint4& b) { return mask & (a <= b); } -#endif - - template<int mask> - __forceinline vuint4 select(const vuint4& t, const vuint4& f) { -#if defined(__SSE4_1__) - return _mm_castps_si128(_mm_blend_ps(_mm_castsi128_ps(f), _mm_castsi128_ps(t), mask)); -#else - return select(vboolf4(mask), t, f); -#endif - } - -/*#if defined(__SSE4_1__) - __forceinline vuint4 min(const vuint4& a, const vuint4& b) { return _mm_min_epu32(a, b); } - __forceinline vuint4 max(const vuint4& a, const vuint4& b) { return _mm_max_epu32(a, b); } - -#else - __forceinline vuint4 min(const vuint4& a, const vuint4& b) { return select(a < b,a,b); } - __forceinline vuint4 max(const vuint4& a, const vuint4& b) { return select(a < b,b,a); } -#endif - - __forceinline vuint4 min(const vuint4& a, unsigned int b) { return min(a,vuint4(b)); } - __forceinline vuint4 min(unsigned int a, const vuint4& b) { return min(vuint4(a),b); } - __forceinline vuint4 max(const vuint4& a, unsigned int b) { return max(a,vuint4(b)); } - __forceinline vuint4 max(unsigned int a, const vuint4& b) { return max(vuint4(a),b); }*/ - - //////////////////////////////////////////////////////////////////////////////// - // Movement/Shifting/Shuffling Functions - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vuint4 unpacklo(const vuint4& a, const vuint4& b) { return _mm_castps_si128(_mm_unpacklo_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b))); } - __forceinline vuint4 unpackhi(const vuint4& a, const vuint4& b) { return _mm_castps_si128(_mm_unpackhi_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b))); } - -#if defined(__aarch64__) - template<int i0, int i1, int i2, int i3> - __forceinline vuint4 shuffle(const vuint4& v) { - return vreinterpretq_s32_u8(vqtbl1q_u8( (uint8x16_t)v.v, _MN_SHUFFLE(i0, i1, i2, i3))); - } - template<int i0, int i1, int i2, int i3> - __forceinline vuint4 shuffle(const vuint4& a, const vuint4& b) { - return vreinterpretq_s32_u8(vqtbl2q_u8( (uint8x16x2_t){(uint8x16_t)a.v, (uint8x16_t)b.v}, _MF_SHUFFLE(i0, i1, i2, i3))); - } -#else - template<int i0, int i1, int i2, int i3> - __forceinline vuint4 shuffle(const vuint4& v) { - return _mm_shuffle_epi32(v, _MM_SHUFFLE(i3, i2, i1, i0)); - } - template<int i0, int i1, int i2, int i3> - __forceinline vuint4 shuffle(const vuint4& a, const vuint4& b) { - return _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b), _MM_SHUFFLE(i3, i2, i1, i0))); - } -#endif -#if defined(__SSE3__) - template<> __forceinline vuint4 shuffle<0, 0, 2, 2>(const vuint4& v) { return _mm_castps_si128(_mm_moveldup_ps(_mm_castsi128_ps(v))); } - template<> __forceinline vuint4 shuffle<1, 1, 3, 3>(const vuint4& v) { return _mm_castps_si128(_mm_movehdup_ps(_mm_castsi128_ps(v))); } - template<> __forceinline vuint4 shuffle<0, 1, 0, 1>(const vuint4& v) { return _mm_castpd_si128(_mm_movedup_pd (_mm_castsi128_pd(v))); } -#endif - - template<int i> - __forceinline vuint4 shuffle(const vuint4& v) { - return shuffle<i,i,i,i>(v); - } - -#if defined(__aarch64__) - template<int src> __forceinline unsigned int extract(const vuint4& b); - template<int dst> __forceinline vuint4 insert(const vuint4& a, const unsigned b); -#elif defined(__SSE4_1__) - template<int src> __forceinline unsigned int extract(const vuint4& b) { return _mm_extract_epi32(b, src); } - template<int dst> __forceinline vuint4 insert(const vuint4& a, const unsigned b) { return _mm_insert_epi32(a, b, dst); } -#else - template<int src> __forceinline unsigned int extract(const vuint4& b) { return b[src&3]; } - template<int dst> __forceinline vuint4 insert(const vuint4& a, const unsigned b) { vuint4 c = a; c[dst&3] = b; return c; } -#endif - -#if defined(__aarch64__) - template<> __forceinline unsigned int extract<0>(const vuint4& b) { - return b[0]; - } - template<> __forceinline unsigned int extract<1>(const vuint4& b) { - return b[1]; - } - template<> __forceinline unsigned int extract<2>(const vuint4& b) { - return b[2]; - } - template<> __forceinline unsigned int extract<3>(const vuint4& b) { - return b[3]; - } - - template<> __forceinline vuint4 insert<0>(const vuint4& a, unsigned b){ - vuint4 c = a; - c[0] = b; - return c; - } - template<> __forceinline vuint4 insert<1>(const vuint4& a, unsigned b){ - vuint4 c = a; - c[1] = b; - return c; - } - template<> __forceinline vuint4 insert<2>(const vuint4& a, unsigned b){ - vuint4 c = a; - c[2] = b; - return c; - } - template<> __forceinline vuint4 insert<3>(const vuint4& a, unsigned b){ - vuint4 c = a; - c[3] = b; - return c; - } - - __forceinline unsigned int toScalar(const vuint4& v) { - return v[0]; - } -#else - template<> __forceinline unsigned int extract<0>(const vuint4& b) { return _mm_cvtsi128_si32(b); } - - __forceinline unsigned int toScalar(const vuint4& v) { return _mm_cvtsi128_si32(v); } -#endif - - //////////////////////////////////////////////////////////////////////////////// - /// Reductions - //////////////////////////////////////////////////////////////////////////////// - -#if 0 -#if defined(__SSE4_1__) - - __forceinline vuint4 vreduce_min(const vuint4& v) { vuint4 h = min(shuffle<1,0,3,2>(v),v); return min(shuffle<2,3,0,1>(h),h); } - __forceinline vuint4 vreduce_max(const vuint4& v) { vuint4 h = max(shuffle<1,0,3,2>(v),v); return max(shuffle<2,3,0,1>(h),h); } - __forceinline vuint4 vreduce_add(const vuint4& v) { vuint4 h = shuffle<1,0,3,2>(v) + v ; return shuffle<2,3,0,1>(h) + h ; } - - __forceinline unsigned int reduce_min(const vuint4& v) { return toScalar(vreduce_min(v)); } - __forceinline unsigned int reduce_max(const vuint4& v) { return toScalar(vreduce_max(v)); } - __forceinline unsigned int reduce_add(const vuint4& v) { return toScalar(vreduce_add(v)); } - - __forceinline size_t select_min(const vuint4& v) { return bsf(movemask(v == vreduce_min(v))); } - __forceinline size_t select_max(const vuint4& v) { return bsf(movemask(v == vreduce_max(v))); } - - //__forceinline size_t select_min(const vboolf4& valid, const vuint4& v) { const vuint4 a = select(valid,v,vuint4(pos_inf)); return bsf(movemask(valid & (a == vreduce_min(a)))); } - //__forceinline size_t select_max(const vboolf4& valid, const vuint4& v) { const vuint4 a = select(valid,v,vuint4(neg_inf)); return bsf(movemask(valid & (a == vreduce_max(a)))); } - -#else - - __forceinline unsigned int reduce_min(const vuint4& v) { return min(v[0],v[1],v[2],v[3]); } - __forceinline unsigned int reduce_max(const vuint4& v) { return max(v[0],v[1],v[2],v[3]); } - __forceinline unsigned int reduce_add(const vuint4& v) { return v[0]+v[1]+v[2]+v[3]; } - -#endif -#endif - - //////////////////////////////////////////////////////////////////////////////// - /// Output Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline embree_ostream operator <<(embree_ostream cout, const vuint4& a) { - return cout << "<" << a[0] << ", " << a[1] << ", " << a[2] << ", " << a[3] << ">"; - } -} - diff --git a/thirdparty/embree-aarch64/common/simd/vuint8_avx.h b/thirdparty/embree-aarch64/common/simd/vuint8_avx.h deleted file mode 100644 index d4e86ae92d..0000000000 --- a/thirdparty/embree-aarch64/common/simd/vuint8_avx.h +++ /dev/null @@ -1,379 +0,0 @@ -// Copyright 2009-2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 - -#pragma once - -namespace embree -{ - /* 8-wide AVX integer type */ - template<> - struct vuint<8> - { - ALIGNED_STRUCT_(32); - - typedef vboolf8 Bool; - typedef vuint8 Int; - typedef vfloat8 Float; - - enum { size = 8 }; // number of SIMD elements - union { // data - __m256i v; - struct { __m128i vl,vh; }; - unsigned int i[8]; - }; - - //////////////////////////////////////////////////////////////////////////////// - /// Constructors, Assignment & Cast Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vuint() {} - __forceinline vuint(const vuint8& a) { v = a.v; } - __forceinline vuint8& operator =(const vuint8& a) { v = a.v; return *this; } - - __forceinline vuint(__m256i a) : v(a) {} - __forceinline operator const __m256i&() const { return v; } - __forceinline operator __m256i&() { return v; } - - __forceinline explicit vuint(const vuint4& a) : v(_mm256_insertf128_si256(_mm256_castsi128_si256(a),a,1)) {} - __forceinline vuint(const vuint4& a, const vuint4& b) : v(_mm256_insertf128_si256(_mm256_castsi128_si256(a),b,1)) {} - __forceinline vuint(const __m128i& a, const __m128i& b) : vl(a), vh(b) {} - - __forceinline explicit vuint(const unsigned int* a) : v(_mm256_castps_si256(_mm256_loadu_ps((const float*)a))) {} - __forceinline vuint(unsigned int a) : v(_mm256_set1_epi32(a)) {} - __forceinline vuint(unsigned int a, unsigned int b) : v(_mm256_set_epi32(b, a, b, a, b, a, b, a)) {} - __forceinline vuint(unsigned int a, unsigned int b, unsigned int c, unsigned int d) : v(_mm256_set_epi32(d, c, b, a, d, c, b, a)) {} - __forceinline vuint(unsigned int a, unsigned int b, unsigned int c, unsigned int d, unsigned int e, unsigned int f, unsigned int g, unsigned int vh) : v(_mm256_set_epi32(vh, g, f, e, d, c, b, a)) {} - - __forceinline explicit vuint(__m256 a) : v(_mm256_cvtps_epi32(a)) {} - - //////////////////////////////////////////////////////////////////////////////// - /// Constants - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vuint(ZeroTy) : v(_mm256_setzero_si256()) {} - __forceinline vuint(OneTy) : v(_mm256_set1_epi32(1)) {} - __forceinline vuint(PosInfTy) : v(_mm256_set1_epi32(0xFFFFFFFF)) {} - __forceinline vuint(StepTy) : v(_mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0)) {} - __forceinline vuint(UndefinedTy) : v(_mm256_undefined_si256()) {} - - //////////////////////////////////////////////////////////////////////////////// - /// Loads and Stores - //////////////////////////////////////////////////////////////////////////////// - - static __forceinline vuint8 load (const void* a) { return _mm256_castps_si256(_mm256_load_ps((float*)a)); } - static __forceinline vuint8 loadu(const void* a) { return _mm256_castps_si256(_mm256_loadu_ps((float*)a)); } - - static __forceinline vuint8 load (const vboolf8& mask, const void* a) { return _mm256_castps_si256(_mm256_maskload_ps((float*)a,mask)); } - static __forceinline vuint8 loadu(const vboolf8& mask, const void* a) { return _mm256_castps_si256(_mm256_maskload_ps((float*)a,mask)); } - - static __forceinline void store (void* ptr, const vuint8& f) { _mm256_store_ps((float*)ptr,_mm256_castsi256_ps(f)); } - static __forceinline void storeu(void* ptr, const vuint8& f) { _mm256_storeu_ps((float*)ptr,_mm256_castsi256_ps(f)); } - -#if !defined(__aarch64__) - static __forceinline void store (const vboolf8& mask, void* ptr, const vuint8& f) { _mm256_maskstore_ps((float*)ptr,(__m256i)mask,_mm256_castsi256_ps(f)); } - static __forceinline void storeu(const vboolf8& mask, void* ptr, const vuint8& f) { _mm256_maskstore_ps((float*)ptr,(__m256i)mask,_mm256_castsi256_ps(f)); } -#else - static __forceinline void store (const vboolf8& mask, void* ptr, const vuint8& f) { _mm256_maskstore_ps((float*)ptr,(__m256i)mask.v,_mm256_castsi256_ps(f)); } - static __forceinline void storeu(const vboolf8& mask, void* ptr, const vuint8& f) { _mm256_maskstore_ps((float*)ptr,(__m256i)mask.v,_mm256_castsi256_ps(f)); } -#endif - static __forceinline void store_nt(void* ptr, const vuint8& v) { - _mm256_stream_ps((float*)ptr,_mm256_castsi256_ps(v)); - } - - static __forceinline vuint8 load(const uint8_t* ptr) { - vuint4 il = vuint4::load(ptr+0); - vuint4 ih = vuint4::load(ptr+4); - return vuint8(il,ih); - } - - static __forceinline vuint8 loadu(const uint8_t* ptr) { - vuint4 il = vuint4::loadu(ptr+0); - vuint4 ih = vuint4::loadu(ptr+4); - return vuint8(il,ih); - } - - static __forceinline vuint8 load(const unsigned short* ptr) { - vuint4 il = vuint4::load(ptr+0); - vuint4 ih = vuint4::load(ptr+4); - return vuint8(il,ih); - } - - static __forceinline vuint8 loadu(const unsigned short* ptr) { - vuint4 il = vuint4::loadu(ptr+0); - vuint4 ih = vuint4::loadu(ptr+4); - return vuint8(il,ih); - } - - static __forceinline void store(uint8_t* ptr, const vuint8& i) { - vuint4 il(i.vl); - vuint4 ih(i.vh); - vuint4::store(ptr + 0,il); - vuint4::store(ptr + 4,ih); - } - - static __forceinline void store(unsigned short* ptr, const vuint8& v) { - for (size_t i=0;i<8;i++) - ptr[i] = (unsigned short)v[i]; - } - - template<int scale = 4> - static __forceinline vuint8 gather(const unsigned int* ptr, const vint8& index) { - return vuint8( - *(unsigned int*)(((int8_t*)ptr)+scale*index[0]), - *(unsigned int*)(((int8_t*)ptr)+scale*index[1]), - *(unsigned int*)(((int8_t*)ptr)+scale*index[2]), - *(unsigned int*)(((int8_t*)ptr)+scale*index[3]), - *(unsigned int*)(((int8_t*)ptr)+scale*index[4]), - *(unsigned int*)(((int8_t*)ptr)+scale*index[5]), - *(unsigned int*)(((int8_t*)ptr)+scale*index[6]), - *(unsigned int*)(((int8_t*)ptr)+scale*index[7])); - } - - template<int scale = 4> - static __forceinline vuint8 gather(const vboolf8& mask, const unsigned int* ptr, const vint8& index) { - vuint8 r = zero; - if (likely(mask[0])) r[0] = *(unsigned int*)(((int8_t*)ptr)+scale*index[0]); - if (likely(mask[1])) r[1] = *(unsigned int*)(((int8_t*)ptr)+scale*index[1]); - if (likely(mask[2])) r[2] = *(unsigned int*)(((int8_t*)ptr)+scale*index[2]); - if (likely(mask[3])) r[3] = *(unsigned int*)(((int8_t*)ptr)+scale*index[3]); - if (likely(mask[4])) r[4] = *(unsigned int*)(((int8_t*)ptr)+scale*index[4]); - if (likely(mask[5])) r[5] = *(unsigned int*)(((int8_t*)ptr)+scale*index[5]); - if (likely(mask[6])) r[6] = *(unsigned int*)(((int8_t*)ptr)+scale*index[6]); - if (likely(mask[7])) r[7] = *(unsigned int*)(((int8_t*)ptr)+scale*index[7]); - return r; - } - - template<int scale = 4> - static __forceinline void scatter(void* ptr, const vint8& ofs, const vuint8& v) - { - *(unsigned int*)(((int8_t*)ptr)+scale*ofs[0]) = v[0]; - *(unsigned int*)(((int8_t*)ptr)+scale*ofs[1]) = v[1]; - *(unsigned int*)(((int8_t*)ptr)+scale*ofs[2]) = v[2]; - *(unsigned int*)(((int8_t*)ptr)+scale*ofs[3]) = v[3]; - *(unsigned int*)(((int8_t*)ptr)+scale*ofs[4]) = v[4]; - *(unsigned int*)(((int8_t*)ptr)+scale*ofs[5]) = v[5]; - *(unsigned int*)(((int8_t*)ptr)+scale*ofs[6]) = v[6]; - *(unsigned int*)(((int8_t*)ptr)+scale*ofs[7]) = v[7]; - } - - template<int scale = 4> - static __forceinline void scatter(const vboolf8& mask, void* ptr, const vint8& ofs, const vuint8& v) - { - if (likely(mask[0])) *(unsigned int*)(((int8_t*)ptr)+scale*ofs[0]) = v[0]; - if (likely(mask[1])) *(unsigned int*)(((int8_t*)ptr)+scale*ofs[1]) = v[1]; - if (likely(mask[2])) *(unsigned int*)(((int8_t*)ptr)+scale*ofs[2]) = v[2]; - if (likely(mask[3])) *(unsigned int*)(((int8_t*)ptr)+scale*ofs[3]) = v[3]; - if (likely(mask[4])) *(unsigned int*)(((int8_t*)ptr)+scale*ofs[4]) = v[4]; - if (likely(mask[5])) *(unsigned int*)(((int8_t*)ptr)+scale*ofs[5]) = v[5]; - if (likely(mask[6])) *(unsigned int*)(((int8_t*)ptr)+scale*ofs[6]) = v[6]; - if (likely(mask[7])) *(unsigned int*)(((int8_t*)ptr)+scale*ofs[7]) = v[7]; - } - - - static __forceinline vuint8 broadcast64(const long long& a) { return _mm256_set1_epi64x(a); } - - //////////////////////////////////////////////////////////////////////////////// - /// Array Access - //////////////////////////////////////////////////////////////////////////////// - - __forceinline const unsigned int& operator [](size_t index) const { assert(index < 8); return i[index]; } - __forceinline unsigned int& operator [](size_t index) { assert(index < 8); return i[index]; } - }; - - //////////////////////////////////////////////////////////////////////////////// - /// Unary Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboolf8 asBool(const vuint8& a) { return _mm256_castsi256_ps(a); } - - __forceinline vuint8 operator +(const vuint8& a) { return a; } - - //////////////////////////////////////////////////////////////////////////////// - /// Binary Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vuint8 operator +(const vuint8& a, const vuint8& b) { return vuint8(_mm_add_epi32(a.vl, b.vl), _mm_add_epi32(a.vh, b.vh)); } - __forceinline vuint8 operator +(const vuint8& a, unsigned int b) { return a + vuint8(b); } - __forceinline vuint8 operator +(unsigned int a, const vuint8& b) { return vuint8(a) + b; } - - __forceinline vuint8 operator -(const vuint8& a, const vuint8& b) { return vuint8(_mm_sub_epi32(a.vl, b.vl), _mm_sub_epi32(a.vh, b.vh)); } - __forceinline vuint8 operator -(const vuint8& a, unsigned int b) { return a - vuint8(b); } - __forceinline vuint8 operator -(unsigned int a, const vuint8& b) { return vuint8(a) - b; } - - //__forceinline vuint8 operator *(const vuint8& a, const vuint8& b) { return vuint8(_mm_mullo_epu32(a.vl, b.vl), _mm_mullo_epu32(a.vh, b.vh)); } - //__forceinline vuint8 operator *(const vuint8& a, unsigned int b) { return a * vuint8(b); } - //__forceinline vuint8 operator *(unsigned int a, const vuint8& b) { return vuint8(a) * b; } - - __forceinline vuint8 operator &(const vuint8& a, const vuint8& b) { return _mm256_castps_si256(_mm256_and_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b))); } - __forceinline vuint8 operator &(const vuint8& a, unsigned int b) { return a & vuint8(b); } - __forceinline vuint8 operator &(unsigned int a, const vuint8& b) { return vuint8(a) & b; } - - __forceinline vuint8 operator |(const vuint8& a, const vuint8& b) { return _mm256_castps_si256(_mm256_or_ps (_mm256_castsi256_ps(a), _mm256_castsi256_ps(b))); } - __forceinline vuint8 operator |(const vuint8& a, unsigned int b) { return a | vuint8(b); } - __forceinline vuint8 operator |(unsigned int a, const vuint8& b) { return vuint8(a) | b; } - - __forceinline vuint8 operator ^(const vuint8& a, const vuint8& b) { return _mm256_castps_si256(_mm256_xor_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b))); } - __forceinline vuint8 operator ^(const vuint8& a, unsigned int b) { return a ^ vuint8(b); } - __forceinline vuint8 operator ^(unsigned int a, const vuint8& b) { return vuint8(a) ^ b; } - - __forceinline vuint8 operator <<(const vuint8& a, unsigned int n) { return vuint8(_mm_slli_epi32(a.vl, n), _mm_slli_epi32(a.vh, n)); } - __forceinline vuint8 operator >>(const vuint8& a, unsigned int n) { return vuint8(_mm_srai_epi32(a.vl, n), _mm_srli_epi32(a.vh, n)); } - - __forceinline vuint8 sll (const vuint8& a, unsigned int b) { return vuint8(_mm_slli_epi32(a.vl, b), _mm_slli_epi32(a.vh, b)); } - __forceinline vuint8 sra (const vuint8& a, unsigned int b) { return vuint8(_mm_srai_epi32(a.vl, b), _mm_srai_epi32(a.vh, b)); } - __forceinline vuint8 srl (const vuint8& a, unsigned int b) { return vuint8(_mm_srli_epi32(a.vl, b), _mm_srli_epi32(a.vh, b)); } - - __forceinline vuint8 min(const vuint8& a, const vuint8& b) { return vuint8(_mm_min_epu32(a.vl, b.vl), _mm_min_epu32(a.vh, b.vh)); } - __forceinline vuint8 min(const vuint8& a, unsigned int b) { return min(a,vuint8(b)); } - __forceinline vuint8 min(unsigned int a, const vuint8& b) { return min(vuint8(a),b); } - - __forceinline vuint8 max(const vuint8& a, const vuint8& b) { return vuint8(_mm_max_epu32(a.vl, b.vl), _mm_max_epu32(a.vh, b.vh)); } - __forceinline vuint8 max(const vuint8& a, unsigned int b) { return max(a,vuint8(b)); } - __forceinline vuint8 max(unsigned int a, const vuint8& b) { return max(vuint8(a),b); } - - //////////////////////////////////////////////////////////////////////////////// - /// Assignment Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vuint8& operator +=(vuint8& a, const vuint8& b) { return a = a + b; } - __forceinline vuint8& operator +=(vuint8& a, unsigned int b) { return a = a + b; } - - __forceinline vuint8& operator -=(vuint8& a, const vuint8& b) { return a = a - b; } - __forceinline vuint8& operator -=(vuint8& a, unsigned int b) { return a = a - b; } - - //__forceinline vuint8& operator *=(vuint8& a, const vuint8& b) { return a = a * b; } - //__forceinline vuint8& operator *=(vuint8& a, unsigned int b) { return a = a * b; } - - __forceinline vuint8& operator &=(vuint8& a, const vuint8& b) { return a = a & b; } - __forceinline vuint8& operator &=(vuint8& a, unsigned int b) { return a = a & b; } - - __forceinline vuint8& operator |=(vuint8& a, const vuint8& b) { return a = a | b; } - __forceinline vuint8& operator |=(vuint8& a, unsigned int b) { return a = a | b; } - - __forceinline vuint8& operator <<=(vuint8& a, unsigned int b) { return a = a << b; } - __forceinline vuint8& operator >>=(vuint8& a, unsigned int b) { return a = a >> b; } - - //////////////////////////////////////////////////////////////////////////////// - /// Comparison Operators + Select - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vboolf8 operator ==(const vuint8& a, const vuint8& b) { return vboolf8(_mm_castsi128_ps(_mm_cmpeq_epi32 (a.vl, b.vl)), - _mm_castsi128_ps(_mm_cmpeq_epi32 (a.vh, b.vh))); } - __forceinline vboolf8 operator ==(const vuint8& a, unsigned int b) { return a == vuint8(b); } - __forceinline vboolf8 operator ==(unsigned int a, const vuint8& b) { return vuint8(a) == b; } - - __forceinline vboolf8 operator !=(const vuint8& a, const vuint8& b) { return !(a == b); } - __forceinline vboolf8 operator !=(const vuint8& a, unsigned int b) { return a != vuint8(b); } - __forceinline vboolf8 operator !=(unsigned int a, const vuint8& b) { return vuint8(a) != b; } - - //__forceinline vboolf8 operator < (const vuint8& a, const vuint8& b) { return vboolf8(_mm_castsi128_ps(_mm_cmplt_epu32 (a.vl, b.vl)), - // _mm_castsi128_ps(_mm_cmplt_epu32 (a.vh, b.vh))); } - //__forceinline vboolf8 operator < (const vuint8& a, unsigned int b) { return a < vuint8(b); } - //__forceinline vboolf8 operator < (unsigned int a, const vuint8& b) { return vuint8(a) < b; } - - //__forceinline vboolf8 operator >=(const vuint8& a, const vuint8& b) { return !(a < b); } - //__forceinline vboolf8 operator >=(const vuint8& a, unsigned int b) { return a >= vuint8(b); } - //__forceinline vboolf8 operator >=(unsigned int a, const vuint8& b) { return vuint8(a) >= b; } - - //__forceinline vboolf8 operator > (const vuint8& a, const vuint8& b) { return vboolf8(_mm_castsi128_ps(_mm_cmpgt_epu32 (a.vl, b.vl)), - // _mm_castsi128_ps(_mm_cmpgt_epu32 (a.vh, b.vh))); } - //__forceinline vboolf8 operator > (const vuint8& a, unsigned int b) { return a > vuint8(b); } - //__forceinline vboolf8 operator > (unsigned int a, const vuint8& b) { return vuint8(a) > b; } - - //__forceinline vboolf8 operator <=(const vuint8& a, const vuint8& b) { return !(a > b); } - //__forceinline vboolf8 operator <=(const vuint8& a, unsigned int b) { return a <= vuint8(b); } - //__forceinline vboolf8 operator <=(unsigned int a, const vuint8& b) { return vuint8(a) <= b; } - - __forceinline vboolf8 eq(const vuint8& a, const vuint8& b) { return a == b; } - __forceinline vboolf8 ne(const vuint8& a, const vuint8& b) { return a != b; } - - __forceinline vboolf8 eq(const vboolf8& mask, const vuint8& a, const vuint8& b) { return mask & (a == b); } - __forceinline vboolf8 ne(const vboolf8& mask, const vuint8& a, const vuint8& b) { return mask & (a != b); } - - __forceinline vuint8 select(const vboolf8& m, const vuint8& t, const vuint8& f) { - return _mm256_castps_si256(_mm256_blendv_ps(_mm256_castsi256_ps(f), _mm256_castsi256_ps(t), m)); - } - - __forceinline vuint8 notand(const vboolf8& m, const vuint8& f) { - return _mm256_castps_si256(_mm256_andnot_ps(m, _mm256_castsi256_ps(f))); - } - - - //////////////////////////////////////////////////////////////////////////////// - /// Movement/Shifting/Shuffling Functions - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vuint8 unpacklo(const vuint8& a, const vuint8& b) { return _mm256_castps_si256(_mm256_unpacklo_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b))); } - __forceinline vuint8 unpackhi(const vuint8& a, const vuint8& b) { return _mm256_castps_si256(_mm256_unpackhi_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b))); } - - template<int i> - __forceinline vuint8 shuffle(const vuint8& v) { - return _mm256_castps_si256(_mm256_permute_ps(_mm256_castsi256_ps(v), _MM_SHUFFLE(i, i, i, i))); - } - - template<int i0, int i1> - __forceinline vuint8 shuffle4(const vuint8& v) { - return _mm256_permute2f128_si256(v, v, (i1 << 4) | (i0 << 0)); - } - - template<int i0, int i1> - __forceinline vuint8 shuffle4(const vuint8& a, const vuint8& b) { - return _mm256_permute2f128_si256(a, b, (i1 << 4) | (i0 << 0)); - } - - template<int i0, int i1, int i2, int i3> - __forceinline vuint8 shuffle(const vuint8& v) { - return _mm256_castps_si256(_mm256_permute_ps(_mm256_castsi256_ps(v), _MM_SHUFFLE(i3, i2, i1, i0))); - } - - template<int i0, int i1, int i2, int i3> - __forceinline vuint8 shuffle(const vuint8& a, const vuint8& b) { - return _mm256_castps_si256(_mm256_shuffle_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b), _MM_SHUFFLE(i3, i2, i1, i0))); - } - - template<> __forceinline vuint8 shuffle<0, 0, 2, 2>(const vuint8& v) { return _mm256_castps_si256(_mm256_moveldup_ps(_mm256_castsi256_ps(v))); } - template<> __forceinline vuint8 shuffle<1, 1, 3, 3>(const vuint8& v) { return _mm256_castps_si256(_mm256_movehdup_ps(_mm256_castsi256_ps(v))); } - template<> __forceinline vuint8 shuffle<0, 1, 0, 1>(const vuint8& v) { return _mm256_castps_si256(_mm256_castpd_ps(_mm256_movedup_pd(_mm256_castps_pd(_mm256_castsi256_ps(v))))); } - - __forceinline vuint8 broadcast(const unsigned int* ptr) { return _mm256_castps_si256(_mm256_broadcast_ss((const float*)ptr)); } - template<int i> __forceinline vuint8 insert4(const vuint8& a, const vuint4& b) { return _mm256_insertf128_si256(a, b, i); } - template<int i> __forceinline vuint4 extract4(const vuint8& a) { return _mm256_extractf128_si256(a, i); } - template<> __forceinline vuint4 extract4<0>(const vuint8& a) { return _mm256_castsi256_si128(a); } - - __forceinline int toScalar(const vuint8& v) { return _mm_cvtsi128_si32(_mm256_castsi256_si128(v)); } - - - //////////////////////////////////////////////////////////////////////////////// - /// Reductions - //////////////////////////////////////////////////////////////////////////////// - - //__forceinline vuint8 vreduce_min2(const vuint8& v) { return min(v,shuffle<1,0,3,2>(v)); } - //__forceinline vuint8 vreduce_min4(const vuint8& v) { vuint8 v1 = vreduce_min2(v); return min(v1,shuffle<2,3,0,1>(v1)); } - //__forceinline vuint8 vreduce_min (const vuint8& v) { vuint8 v1 = vreduce_min4(v); return min(v1,shuffle4<1,0>(v1)); } - - //__forceinline vuint8 vreduce_max2(const vuint8& v) { return max(v,shuffle<1,0,3,2>(v)); } - //__forceinline vuint8 vreduce_max4(const vuint8& v) { vuint8 v1 = vreduce_max2(v); return max(v1,shuffle<2,3,0,1>(v1)); } - //__forceinline vuint8 vreduce_max (const vuint8& v) { vuint8 v1 = vreduce_max4(v); return max(v1,shuffle4<1,0>(v1)); } - - __forceinline vuint8 vreduce_add2(const vuint8& v) { return v + shuffle<1,0,3,2>(v); } - __forceinline vuint8 vreduce_add4(const vuint8& v) { vuint8 v1 = vreduce_add2(v); return v1 + shuffle<2,3,0,1>(v1); } - __forceinline vuint8 vreduce_add (const vuint8& v) { vuint8 v1 = vreduce_add4(v); return v1 + shuffle4<1,0>(v1); } - - //__forceinline int reduce_min(const vuint8& v) { return toScalar(vreduce_min(v)); } - //__forceinline int reduce_max(const vuint8& v) { return toScalar(vreduce_max(v)); } - __forceinline int reduce_add(const vuint8& v) { return toScalar(vreduce_add(v)); } - - //__forceinline size_t select_min(const vuint8& v) { return bsf(movemask(v == vreduce_min(v))); } - //__forceinline size_t select_max(const vuint8& v) { return bsf(movemask(v == vreduce_max(v))); } - - //__forceinline size_t select_min(const vboolf8& valid, const vuint8& v) { const vuint8 a = select(valid,v,vuint8(pos_inf)); return bsf(movemask(valid & (a == vreduce_min(a)))); } - //__forceinline size_t select_max(const vboolf8& valid, const vuint8& v) { const vuint8 a = select(valid,v,vuint8(neg_inf)); return bsf(movemask(valid & (a == vreduce_max(a)))); } - - //////////////////////////////////////////////////////////////////////////////// - /// Output Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline embree_ostream operator <<(embree_ostream cout, const vuint8& a) { - return cout << "<" << a[0] << ", " << a[1] << ", " << a[2] << ", " << a[3] << ", " << a[4] << ", " << a[5] << ", " << a[6] << ", " << a[7] << ">"; - } -} diff --git a/thirdparty/embree-aarch64/common/simd/vuint8_avx2.h b/thirdparty/embree-aarch64/common/simd/vuint8_avx2.h deleted file mode 100644 index b2a965448d..0000000000 --- a/thirdparty/embree-aarch64/common/simd/vuint8_avx2.h +++ /dev/null @@ -1,439 +0,0 @@ -// Copyright 2009-2020 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 - -#pragma once - -namespace embree -{ - /* 8-wide AVX integer type */ - template<> - struct vuint<8> - { - ALIGNED_STRUCT_(32); - - typedef vboolf8 Bool; - typedef vuint8 Int; - typedef vfloat8 Float; - - enum { size = 8 }; // number of SIMD elements - union { // data - __m256i v; - unsigned int i[8]; - }; - - //////////////////////////////////////////////////////////////////////////////// - /// Constructors, Assignment & Cast Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vuint() {} - __forceinline vuint(const vuint8& a) { v = a.v; } - __forceinline vuint8& operator =(const vuint8& a) { v = a.v; return *this; } - - __forceinline vuint(__m256i a) : v(a) {} - __forceinline operator const __m256i&() const { return v; } - __forceinline operator __m256i&() { return v; } - - __forceinline explicit vuint(const vuint4& a) : v(_mm256_insertf128_si256(_mm256_castsi128_si256(a),a,1)) {} - __forceinline vuint(const vuint4& a, const vuint4& b) : v(_mm256_insertf128_si256(_mm256_castsi128_si256(a),b,1)) {} - __forceinline vuint(const __m128i& a, const __m128i& b) : v(_mm256_insertf128_si256(_mm256_castsi128_si256(a),b,1)) {} - - __forceinline explicit vuint(const unsigned int* a) : v(_mm256_castps_si256(_mm256_loadu_ps((const float*)a))) {} - __forceinline vuint(unsigned int a) : v(_mm256_set1_epi32(a)) {} - __forceinline vuint(unsigned int a, unsigned int b) : v(_mm256_set_epi32(b, a, b, a, b, a, b, a)) {} - __forceinline vuint(unsigned int a, unsigned int b, unsigned int c, unsigned int d) : v(_mm256_set_epi32(d, c, b, a, d, c, b, a)) {} - __forceinline vuint(unsigned int a, unsigned int b, unsigned int c, unsigned int d, unsigned int e, unsigned int f, unsigned int g, unsigned int h) : v(_mm256_set_epi32(h, g, f, e, d, c, b, a)) {} - - __forceinline explicit vuint(__m256 a) : v(_mm256_cvtps_epi32(a)) {} - -#if defined(__AVX512VL__) - __forceinline explicit vuint(const vboolf8& a) : v(_mm256_movm_epi32(a)) {} -#else - __forceinline explicit vuint(const vboolf8& a) : v(_mm256_castps_si256((__m256)a)) {} -#endif - - //////////////////////////////////////////////////////////////////////////////// - /// Constants - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vuint(ZeroTy) : v(_mm256_setzero_si256()) {} - __forceinline vuint(OneTy) : v(_mm256_set1_epi32(1)) {} - __forceinline vuint(PosInfTy) : v(_mm256_set1_epi32(pos_inf)) {} - __forceinline vuint(NegInfTy) : v(_mm256_set1_epi32(neg_inf)) {} - __forceinline vuint(StepTy) : v(_mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0)) {} - __forceinline vuint(UndefinedTy) : v(_mm256_undefined_si256()) {} - - //////////////////////////////////////////////////////////////////////////////// - /// Loads and Stores - //////////////////////////////////////////////////////////////////////////////// - - static __forceinline vuint8 load(const uint8_t* ptr) { return _mm256_cvtepu8_epi32(_mm_loadl_epi64((__m128i*)ptr)); } - static __forceinline vuint8 loadu(const uint8_t* ptr) { return _mm256_cvtepu8_epi32(_mm_loadl_epi64((__m128i*)ptr)); } - static __forceinline vuint8 load(const unsigned short* ptr) { return _mm256_cvtepu16_epi32(_mm_load_si128((__m128i*)ptr)); } - static __forceinline vuint8 loadu(const unsigned short* ptr) { return _mm256_cvtepu16_epi32(_mm_loadu_si128((__m128i*)ptr)); } - - static __forceinline vuint8 load(const void* ptr) { return _mm256_load_si256((__m256i*)ptr); } - static __forceinline vuint8 loadu(const void* ptr) { return _mm256_loadu_si256((__m256i*)ptr); } - - static __forceinline void store (void* ptr, const vuint8& v) { _mm256_store_si256((__m256i*)ptr,v); } - static __forceinline void storeu(void* ptr, const vuint8& v) { _mm256_storeu_ps((float*)ptr,_mm256_castsi256_ps(v)); } - -#if defined(__AVX512VL__) - - static __forceinline vuint8 compact(const vboolf8& mask, vuint8 &v) { - return _mm256_mask_compress_epi32(v, mask, v); - } - static __forceinline vuint8 compact(const vboolf8& mask, vuint8 &a, const vuint8& b) { - return _mm256_mask_compress_epi32(a, mask, b); - } - - static __forceinline vuint8 load (const vboolf8& mask, const void* ptr) { return _mm256_mask_load_epi32 (_mm256_setzero_si256(),mask,ptr); } - static __forceinline vuint8 loadu(const vboolf8& mask, const void* ptr) { return _mm256_mask_loadu_epi32(_mm256_setzero_si256(),mask,ptr); } - - static __forceinline void store (const vboolf8& mask, void* ptr, const vuint8& v) { _mm256_mask_store_epi32 (ptr,mask,v); } - static __forceinline void storeu(const vboolf8& mask, void* ptr, const vuint8& v) { _mm256_mask_storeu_epi32(ptr,mask,v); } -#else - static __forceinline vuint8 load (const vboolf8& mask, const void* ptr) { return _mm256_castps_si256(_mm256_maskload_ps((float*)ptr,mask)); } - static __forceinline vuint8 loadu(const vboolf8& mask, const void* ptr) { return _mm256_castps_si256(_mm256_maskload_ps((float*)ptr,mask)); } - - static __forceinline void store (const vboolf8& mask, void* ptr, const vuint8& v) { _mm256_maskstore_epi32((int*)ptr,mask,v); } - static __forceinline void storeu(const vboolf8& mask, void* ptr, const vuint8& v) { _mm256_maskstore_epi32((int*)ptr,mask,v); } -#endif - - static __forceinline vuint8 load_nt(void* ptr) { - return _mm256_stream_load_si256((__m256i*)ptr); - } - - static __forceinline void store_nt(void* ptr, const vuint8& v) { - _mm256_stream_ps((float*)ptr,_mm256_castsi256_ps(v)); - } - - static __forceinline void store(uint8_t* ptr, const vuint8& i) - { - for (size_t j=0; j<8; j++) - ptr[j] = i[j]; - } - - static __forceinline void store(unsigned short* ptr, const vuint8& v) { - for (size_t i=0;i<8;i++) - ptr[i] = (unsigned short)v[i]; - } - - template<int scale = 4> - static __forceinline vuint8 gather(const unsigned int *const ptr, const vint8& index) { - return _mm256_i32gather_epi32((const int*) ptr, index, scale); - } - - template<int scale = 4> - static __forceinline vuint8 gather(const vboolf8& mask, const unsigned int *const ptr, const vint8& index) { - vuint8 r = zero; -#if defined(__AVX512VL__) - return _mm256_mmask_i32gather_epi32(r, mask, index, (const int*) ptr, scale); -#else - return _mm256_mask_i32gather_epi32(r, (const int*) ptr, index, mask, scale); -#endif - } - - template<int scale = 4> - static __forceinline void scatter(void* ptr, const vint8& ofs, const vuint8& v) - { -#if defined(__AVX512VL__) - _mm256_i32scatter_epi32((int*)ptr, ofs, v, scale); -#else - *(unsigned int*)(((int8_t*)ptr) + scale * ofs[0]) = v[0]; - *(unsigned int*)(((int8_t*)ptr) + scale * ofs[1]) = v[1]; - *(unsigned int*)(((int8_t*)ptr) + scale * ofs[2]) = v[2]; - *(unsigned int*)(((int8_t*)ptr) + scale * ofs[3]) = v[3]; - *(unsigned int*)(((int8_t*)ptr) + scale * ofs[4]) = v[4]; - *(unsigned int*)(((int8_t*)ptr) + scale * ofs[5]) = v[5]; - *(unsigned int*)(((int8_t*)ptr) + scale * ofs[6]) = v[6]; - *(unsigned int*)(((int8_t*)ptr) + scale * ofs[7]) = v[7]; -#endif - } - - template<int scale = 4> - static __forceinline void scatter(const vboolf8& mask, void* ptr, const vint8& ofs, const vuint8& v) - { -#if defined(__AVX512VL__) - _mm256_mask_i32scatter_epi32((int*)ptr, mask, ofs, v, scale); -#else - if (likely(mask[0])) *(unsigned int*)(((int8_t*)ptr)+scale*ofs[0]) = v[0]; - if (likely(mask[1])) *(unsigned int*)(((int8_t*)ptr)+scale*ofs[1]) = v[1]; - if (likely(mask[2])) *(unsigned int*)(((int8_t*)ptr)+scale*ofs[2]) = v[2]; - if (likely(mask[3])) *(unsigned int*)(((int8_t*)ptr)+scale*ofs[3]) = v[3]; - if (likely(mask[4])) *(unsigned int*)(((int8_t*)ptr)+scale*ofs[4]) = v[4]; - if (likely(mask[5])) *(unsigned int*)(((int8_t*)ptr)+scale*ofs[5]) = v[5]; - if (likely(mask[6])) *(unsigned int*)(((int8_t*)ptr)+scale*ofs[6]) = v[6]; - if (likely(mask[7])) *(unsigned int*)(((int8_t*)ptr)+scale*ofs[7]) = v[7]; -#endif - } - - static __forceinline vuint8 broadcast64(const long long &a) { return _mm256_set1_epi64x(a); } - - //////////////////////////////////////////////////////////////////////////////// - /// Array Access - //////////////////////////////////////////////////////////////////////////////// - - __forceinline const unsigned int& operator [](size_t index) const { assert(index < 8); return i[index]; } - __forceinline unsigned int& operator [](size_t index) { assert(index < 8); return i[index]; } - }; - - //////////////////////////////////////////////////////////////////////////////// - /// Unary Operators - //////////////////////////////////////////////////////////////////////////////// - -#if defined(__AVX512VL__) - __forceinline vboolf8 asBool(const vuint8& a) { return _mm256_movepi32_mask(a); } -#else - __forceinline vboolf8 asBool(const vuint8& a) { return _mm256_castsi256_ps(a); } -#endif - - __forceinline vuint8 operator +(const vuint8& a) { return a; } - - //////////////////////////////////////////////////////////////////////////////// - /// Binary Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vuint8 operator +(const vuint8& a, const vuint8& b) { return _mm256_add_epi32(a, b); } - __forceinline vuint8 operator +(const vuint8& a, unsigned int b) { return a + vuint8(b); } - __forceinline vuint8 operator +(unsigned int a, const vuint8& b) { return vuint8(a) + b; } - - __forceinline vuint8 operator -(const vuint8& a, const vuint8& b) { return _mm256_sub_epi32(a, b); } - __forceinline vuint8 operator -(const vuint8& a, unsigned int b) { return a - vuint8(b); } - __forceinline vuint8 operator -(unsigned int a, const vuint8& b) { return vuint8(a) - b; } - - //__forceinline vuint8 operator *(const vuint8& a, const vuint8& b) { return _mm256_mullo_epu32(a, b); } - //__forceinline vuint8 operator *(const vuint8& a, unsigned int b) { return a * vuint8(b); } - //__forceinline vuint8 operator *(unsigned int a, const vuint8& b) { return vuint8(a) * b; } - - __forceinline vuint8 operator &(const vuint8& a, const vuint8& b) { return _mm256_and_si256(a, b); } - __forceinline vuint8 operator &(const vuint8& a, unsigned int b) { return a & vuint8(b); } - __forceinline vuint8 operator &(unsigned int a, const vuint8& b) { return vuint8(a) & b; } - - __forceinline vuint8 operator |(const vuint8& a, const vuint8& b) { return _mm256_or_si256(a, b); } - __forceinline vuint8 operator |(const vuint8& a, unsigned int b) { return a | vuint8(b); } - __forceinline vuint8 operator |(unsigned int a, const vuint8& b) { return vuint8(a) | b; } - - __forceinline vuint8 operator ^(const vuint8& a, const vuint8& b) { return _mm256_xor_si256(a, b); } - __forceinline vuint8 operator ^(const vuint8& a, unsigned int b) { return a ^ vuint8(b); } - __forceinline vuint8 operator ^(unsigned int a, const vuint8& b) { return vuint8(a) ^ b; } - - __forceinline vuint8 operator <<(const vuint8& a, unsigned int n) { return _mm256_slli_epi32(a, n); } - __forceinline vuint8 operator >>(const vuint8& a, unsigned int n) { return _mm256_srli_epi32(a, n); } - - __forceinline vuint8 operator <<(const vuint8& a, const vuint8& n) { return _mm256_sllv_epi32(a, n); } - __forceinline vuint8 operator >>(const vuint8& a, const vuint8& n) { return _mm256_srlv_epi32(a, n); } - - __forceinline vuint8 sll(const vuint8& a, unsigned int b) { return _mm256_slli_epi32(a, b); } - __forceinline vuint8 sra(const vuint8& a, unsigned int b) { return _mm256_srai_epi32(a, b); } - __forceinline vuint8 srl(const vuint8& a, unsigned int b) { return _mm256_srli_epi32(a, b); } - - __forceinline vuint8 sll(const vuint8& a, const vuint8& b) { return _mm256_sllv_epi32(a, b); } - __forceinline vuint8 sra(const vuint8& a, const vuint8& b) { return _mm256_srav_epi32(a, b); } - __forceinline vuint8 srl(const vuint8& a, const vuint8& b) { return _mm256_srlv_epi32(a, b); } - - __forceinline vuint8 min(const vuint8& a, const vuint8& b) { return _mm256_min_epu32(a, b); } - __forceinline vuint8 min(const vuint8& a, unsigned int b) { return min(a,vuint8(b)); } - __forceinline vuint8 min(unsigned int a, const vuint8& b) { return min(vuint8(a),b); } - - __forceinline vuint8 max(const vuint8& a, const vuint8& b) { return _mm256_max_epu32(a, b); } - __forceinline vuint8 max(const vuint8& a, unsigned int b) { return max(a,vuint8(b)); } - __forceinline vuint8 max(unsigned int a, const vuint8& b) { return max(vuint8(a),b); } - - //////////////////////////////////////////////////////////////////////////////// - /// Assignment Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vuint8& operator +=(vuint8& a, const vuint8& b) { return a = a + b; } - __forceinline vuint8& operator +=(vuint8& a, unsigned int b) { return a = a + b; } - - __forceinline vuint8& operator -=(vuint8& a, const vuint8& b) { return a = a - b; } - __forceinline vuint8& operator -=(vuint8& a, unsigned int b) { return a = a - b; } - - //__forceinline vuint8& operator *=(vuint8& a, const vuint8& b) { return a = a * b; } - //__forceinline vuint8& operator *=(vuint8& a, unsigned int b) { return a = a * b; } - - __forceinline vuint8& operator &=(vuint8& a, const vuint8& b) { return a = a & b; } - __forceinline vuint8& operator &=(vuint8& a, unsigned int b) { return a = a & b; } - - __forceinline vuint8& operator |=(vuint8& a, const vuint8& b) { return a = a | b; } - __forceinline vuint8& operator |=(vuint8& a, unsigned int b) { return a = a | b; } - - __forceinline vuint8& operator <<=(vuint8& a, const unsigned int b) { return a = a << b; } - __forceinline vuint8& operator >>=(vuint8& a, const unsigned int b) { return a = a >> b; } - - //////////////////////////////////////////////////////////////////////////////// - /// Comparison Operators + Select - //////////////////////////////////////////////////////////////////////////////// - -#if defined(__AVX512VL__) - __forceinline vboolf8 operator ==(const vuint8& a, const vuint8& b) { return _mm256_cmp_epu32_mask(a,b,_MM_CMPINT_EQ); } - __forceinline vboolf8 operator !=(const vuint8& a, const vuint8& b) { return _mm256_cmp_epu32_mask(a,b,_MM_CMPINT_NE); } - __forceinline vboolf8 operator < (const vuint8& a, const vuint8& b) { return _mm256_cmp_epu32_mask(a,b,_MM_CMPINT_LT); } - __forceinline vboolf8 operator >=(const vuint8& a, const vuint8& b) { return _mm256_cmp_epu32_mask(a,b,_MM_CMPINT_GE); } - __forceinline vboolf8 operator > (const vuint8& a, const vuint8& b) { return _mm256_cmp_epu32_mask(a,b,_MM_CMPINT_GT); } - __forceinline vboolf8 operator <=(const vuint8& a, const vuint8& b) { return _mm256_cmp_epu32_mask(a,b,_MM_CMPINT_LE); } - - __forceinline vuint8 select(const vboolf8& m, const vuint8& t, const vuint8& f) { - return _mm256_mask_blend_epi32(m, (__m256i)f, (__m256i)t); - } -#else - __forceinline vboolf8 operator ==(const vuint8& a, const vuint8& b) { return _mm256_castsi256_ps(_mm256_cmpeq_epi32(a, b)); } - __forceinline vboolf8 operator !=(const vuint8& a, const vuint8& b) { return !(a == b); } - //__forceinline vboolf8 operator < (const vuint8& a, const vuint8& b) { return _mm256_castsi256_ps(_mm256_cmpgt_epu32(b, a)); } - //__forceinline vboolf8 operator >=(const vuint8& a, const vuint8& b) { return !(a < b); } - //__forceinline vboolf8 operator > (const vuint8& a, const vuint8& b) { return _mm256_castsi256_ps(_mm256_cmpgt_epu32(a, b)); } - //__forceinline vboolf8 operator <=(const vuint8& a, const vuint8& b) { return !(a > b); } - - __forceinline vuint8 select(const vboolf8& m, const vuint8& t, const vuint8& f) { - return _mm256_castps_si256(_mm256_blendv_ps(_mm256_castsi256_ps(f), _mm256_castsi256_ps(t), m)); - } -#endif - - template<int mask> - __forceinline vuint8 select(const vuint8& t, const vuint8& f) { - return _mm256_blend_epi32(f, t, mask); - } - - __forceinline vboolf8 operator ==(const vuint8& a, unsigned int b) { return a == vuint8(b); } - __forceinline vboolf8 operator ==(unsigned int a, const vuint8& b) { return vuint8(a) == b; } - - __forceinline vboolf8 operator !=(const vuint8& a, unsigned int b) { return a != vuint8(b); } - __forceinline vboolf8 operator !=(unsigned int a, const vuint8& b) { return vuint8(a) != b; } - - //__forceinline vboolf8 operator < (const vuint8& a, unsigned int b) { return a < vuint8(b); } - //__forceinline vboolf8 operator < (unsigned int a, const vuint8& b) { return vuint8(a) < b; } - - //__forceinline vboolf8 operator >=(const vuint8& a, unsigned int b) { return a >= vuint8(b); } - //__forceinline vboolf8 operator >=(unsigned int a, const vuint8& b) { return vuint8(a) >= b; } - - //__forceinline vboolf8 operator > (const vuint8& a, unsigned int b) { return a > vuint8(b); } - //__forceinline vboolf8 operator > (unsigned int a, const vuint8& b) { return vuint8(a) > b; } - - //__forceinline vboolf8 operator <=(const vuint8& a, unsigned int b) { return a <= vuint8(b); } - //__forceinline vboolf8 operator <=(unsigned int a, const vuint8& b) { return vuint8(a) <= b; } - - __forceinline vboolf8 eq(const vuint8& a, const vuint8& b) { return a == b; } - __forceinline vboolf8 ne(const vuint8& a, const vuint8& b) { return a != b; } - //__forceinline vboolf8 lt(const vuint8& a, const vuint8& b) { return a < b; } - //__forceinline vboolf8 ge(const vuint8& a, const vuint8& b) { return a >= b; } - //__forceinline vboolf8 gt(const vuint8& a, const vuint8& b) { return a > b; } - //__forceinline vboolf8 le(const vuint8& a, const vuint8& b) { return a <= b; } - -#if defined(__AVX512VL__) - __forceinline vboolf8 eq(const vboolf8& mask, const vuint8& a, const vuint8& b) { return _mm256_mask_cmp_epu32_mask(mask, a, b, _MM_CMPINT_EQ); } - __forceinline vboolf8 ne(const vboolf8& mask, const vuint8& a, const vuint8& b) { return _mm256_mask_cmp_epu32_mask(mask, a, b, _MM_CMPINT_NE); } - __forceinline vboolf8 lt(const vboolf8& mask, const vuint8& a, const vuint8& b) { return _mm256_mask_cmp_epu32_mask(mask, a, b, _MM_CMPINT_LT); } - __forceinline vboolf8 ge(const vboolf8& mask, const vuint8& a, const vuint8& b) { return _mm256_mask_cmp_epu32_mask(mask, a, b, _MM_CMPINT_GE); } - __forceinline vboolf8 gt(const vboolf8& mask, const vuint8& a, const vuint8& b) { return _mm256_mask_cmp_epu32_mask(mask, a, b, _MM_CMPINT_GT); } - __forceinline vboolf8 le(const vboolf8& mask, const vuint8& a, const vuint8& b) { return _mm256_mask_cmp_epu32_mask(mask, a, b, _MM_CMPINT_LE); } -#else - __forceinline vboolf8 eq(const vboolf8& mask, const vuint8& a, const vuint8& b) { return mask & (a == b); } - __forceinline vboolf8 ne(const vboolf8& mask, const vuint8& a, const vuint8& b) { return mask & (a != b); } - //__forceinline vboolf8 lt(const vboolf8& mask, const vuint8& a, const vuint8& b) { return mask & (a < b); } - //__forceinline vboolf8 ge(const vboolf8& mask, const vuint8& a, const vuint8& b) { return mask & (a >= b); } - //__forceinline vboolf8 gt(const vboolf8& mask, const vuint8& a, const vuint8& b) { return mask & (a > b); } - //__forceinline vboolf8 le(const vboolf8& mask, const vuint8& a, const vuint8& b) { return mask & (a <= b); } -#endif - - //////////////////////////////////////////////////////////////////////////////// - /// Movement/Shifting/Shuffling Functions - //////////////////////////////////////////////////////////////////////////////// - - __forceinline vuint8 unpacklo(const vuint8& a, const vuint8& b) { return _mm256_unpacklo_epi32(a, b); } - __forceinline vuint8 unpackhi(const vuint8& a, const vuint8& b) { return _mm256_unpackhi_epi32(a, b); } - - template<int i> - __forceinline vuint8 shuffle(const vuint8& v) { - return _mm256_castps_si256(_mm256_permute_ps(_mm256_castsi256_ps(v), _MM_SHUFFLE(i, i, i, i))); - } - - template<int i0, int i1> - __forceinline vuint8 shuffle4(const vuint8& v) { - return _mm256_permute2f128_si256(v, v, (i1 << 4) | (i0 << 0)); - } - - template<int i0, int i1> - __forceinline vuint8 shuffle4(const vuint8& a, const vuint8& b) { - return _mm256_permute2f128_si256(a, b, (i1 << 4) | (i0 << 0)); - } - - template<int i0, int i1, int i2, int i3> - __forceinline vuint8 shuffle(const vuint8& v) { - return _mm256_castps_si256(_mm256_permute_ps(_mm256_castsi256_ps(v), _MM_SHUFFLE(i3, i2, i1, i0))); - } - - template<int i0, int i1, int i2, int i3> - __forceinline vuint8 shuffle(const vuint8& a, const vuint8& b) { - return _mm256_castps_si256(_mm256_shuffle_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b), _MM_SHUFFLE(i3, i2, i1, i0))); - } - - template<> __forceinline vuint8 shuffle<0, 0, 2, 2>(const vuint8& v) { return _mm256_castps_si256(_mm256_moveldup_ps(_mm256_castsi256_ps(v))); } - template<> __forceinline vuint8 shuffle<1, 1, 3, 3>(const vuint8& v) { return _mm256_castps_si256(_mm256_movehdup_ps(_mm256_castsi256_ps(v))); } - template<> __forceinline vuint8 shuffle<0, 1, 0, 1>(const vuint8& v) { return _mm256_castps_si256(_mm256_castpd_ps(_mm256_movedup_pd(_mm256_castps_pd(_mm256_castsi256_ps(v))))); } - - __forceinline vuint8 broadcast(const unsigned int* ptr) { return _mm256_castps_si256(_mm256_broadcast_ss((const float*)ptr)); } - - template<int i> __forceinline vuint8 insert4(const vuint8& a, const vuint4& b) { return _mm256_insertf128_si256(a, b, i); } - template<int i> __forceinline vuint4 extract4(const vuint8& a) { return _mm256_extractf128_si256(a, i); } - template<> __forceinline vuint4 extract4<0>(const vuint8& a) { return _mm256_castsi256_si128(a); } - - __forceinline int toScalar(const vuint8& v) { return _mm_cvtsi128_si32(_mm256_castsi256_si128(v)); } - -#if !defined(__aarch64__) - - __forceinline vuint8 permute(const vuint8& v, const __m256i& index) { - return _mm256_permutevar8x32_epi32(v, index); - } - - __forceinline vuint8 shuffle(const vuint8& v, const __m256i& index) { - return _mm256_castps_si256(_mm256_permutevar_ps(_mm256_castsi256_ps(v), index)); - } - - template<int i> - __forceinline vuint8 align_shift_right(const vuint8& a, const vuint8& b) { -#if defined(__AVX512VL__) - return _mm256_alignr_epi32(a, b, i); -#else - return _mm256_alignr_epi8(a, b, 4*i); -#endif - } - -#endif - - - //////////////////////////////////////////////////////////////////////////////// - /// Reductions - //////////////////////////////////////////////////////////////////////////////// - - //__forceinline vuint8 vreduce_min2(const vuint8& v) { return min(v,shuffle<1,0,3,2>(v)); } - //__forceinline vuint8 vreduce_min4(const vuint8& v) { vuint8 v1 = vreduce_min2(v); return min(v1,shuffle<2,3,0,1>(v1)); } - //__forceinline vuint8 vreduce_min (const vuint8& v) { vuint8 v1 = vreduce_min4(v); return min(v1,shuffle4<1,0>(v1)); } - - //__forceinline vuint8 vreduce_max2(const vuint8& v) { return max(v,shuffle<1,0,3,2>(v)); } - //__forceinline vuint8 vreduce_max4(const vuint8& v) { vuint8 v1 = vreduce_max2(v); return max(v1,shuffle<2,3,0,1>(v1)); } - //__forceinline vuint8 vreduce_max (const vuint8& v) { vuint8 v1 = vreduce_max4(v); return max(v1,shuffle4<1,0>(v1)); } - - __forceinline vuint8 vreduce_add2(const vuint8& v) { return v + shuffle<1,0,3,2>(v); } - __forceinline vuint8 vreduce_add4(const vuint8& v) { vuint8 v1 = vreduce_add2(v); return v1 + shuffle<2,3,0,1>(v1); } - __forceinline vuint8 vreduce_add (const vuint8& v) { vuint8 v1 = vreduce_add4(v); return v1 + shuffle4<1,0>(v1); } - - //__forceinline int reduce_min(const vuint8& v) { return toScalar(vreduce_min(v)); } - //__forceinline int reduce_max(const vuint8& v) { return toScalar(vreduce_max(v)); } - __forceinline int reduce_add(const vuint8& v) { return toScalar(vreduce_add(v)); } - - //__forceinline size_t select_min(const vuint8& v) { return bsf(movemask(v == vreduce_min(v))); } - //__forceinline size_t select_max(const vuint8& v) { return bsf(movemask(v == vreduce_max(v))); } - - //__forceinline size_t select_min(const vboolf8& valid, const vuint8& v) { const vuint8 a = select(valid,v,vuint8(pos_inf)); return bsf(movemask(valid & (a == vreduce_min(a)))); } - //__forceinline size_t select_max(const vboolf8& valid, const vuint8& v) { const vuint8 a = select(valid,v,vuint8(neg_inf)); return bsf(movemask(valid & (a == vreduce_max(a)))); } - - __forceinline vuint8 assign(const vuint4& a) { return _mm256_castsi128_si256(a); } - - //////////////////////////////////////////////////////////////////////////////// - /// Output Operators - //////////////////////////////////////////////////////////////////////////////// - - __forceinline embree_ostream operator <<(embree_ostream cout, const vuint8& a) { - return cout << "<" << a[0] << ", " << a[1] << ", " << a[2] << ", " << a[3] << ", " << a[4] << ", " << a[5] << ", " << a[6] << ", " << a[7] << ">"; - } -} |