summaryrefslogtreecommitdiff
path: root/thirdparty/embree-aarch64/common/simd
diff options
context:
space:
mode:
authorjfons <joan.fonssanchez@gmail.com>2021-04-20 18:38:09 +0200
committerjfons <joan.fonssanchez@gmail.com>2021-04-23 15:57:28 +0200
commit34b3e8f9e2ae076990ecf3b2827eff759ba2abf9 (patch)
tree854a526a5ba2d6128e44d995d1bc138cf84ee722 /thirdparty/embree-aarch64/common/simd
parenteeccab26c5641409092547e02ad11e6253ac1b87 (diff)
Add Embree-aarch64 thirdparty library
Diffstat (limited to 'thirdparty/embree-aarch64/common/simd')
-rw-r--r--thirdparty/embree-aarch64/common/simd/avx.h34
-rw-r--r--thirdparty/embree-aarch64/common/simd/avx512.h41
-rw-r--r--thirdparty/embree-aarch64/common/simd/simd.h110
-rw-r--r--thirdparty/embree-aarch64/common/simd/sse.cpp34
-rw-r--r--thirdparty/embree-aarch64/common/simd/sse.h35
-rw-r--r--thirdparty/embree-aarch64/common/simd/varying.h132
-rw-r--r--thirdparty/embree-aarch64/common/simd/vboold4_avx.h160
-rw-r--r--thirdparty/embree-aarch64/common/simd/vboold4_avx512.h140
-rw-r--r--thirdparty/embree-aarch64/common/simd/vboold8_avx512.h148
-rw-r--r--thirdparty/embree-aarch64/common/simd/vboolf16_avx512.h150
-rw-r--r--thirdparty/embree-aarch64/common/simd/vboolf4_avx512.h143
-rw-r--r--thirdparty/embree-aarch64/common/simd/vboolf4_sse2.h198
-rw-r--r--thirdparty/embree-aarch64/common/simd/vboolf8_avx.h189
-rw-r--r--thirdparty/embree-aarch64/common/simd/vboolf8_avx512.h143
-rw-r--r--thirdparty/embree-aarch64/common/simd/vdouble4_avx.h324
-rw-r--r--thirdparty/embree-aarch64/common/simd/vdouble8_avx512.h356
-rw-r--r--thirdparty/embree-aarch64/common/simd/vfloat16_avx512.h771
-rw-r--r--thirdparty/embree-aarch64/common/simd/vfloat4_sse2.h925
-rw-r--r--thirdparty/embree-aarch64/common/simd/vfloat8_avx.h847
-rw-r--r--thirdparty/embree-aarch64/common/simd/vint16_avx512.h490
-rw-r--r--thirdparty/embree-aarch64/common/simd/vint4_sse2.h681
-rw-r--r--thirdparty/embree-aarch64/common/simd/vint8_avx.h464
-rw-r--r--thirdparty/embree-aarch64/common/simd/vint8_avx2.h512
-rw-r--r--thirdparty/embree-aarch64/common/simd/vllong4_avx2.h358
-rw-r--r--thirdparty/embree-aarch64/common/simd/vllong8_avx512.h381
-rw-r--r--thirdparty/embree-aarch64/common/simd/vuint16_avx512.h443
-rw-r--r--thirdparty/embree-aarch64/common/simd/vuint4_sse2.h499
-rw-r--r--thirdparty/embree-aarch64/common/simd/vuint8_avx.h379
-rw-r--r--thirdparty/embree-aarch64/common/simd/vuint8_avx2.h439
29 files changed, 9526 insertions, 0 deletions
diff --git a/thirdparty/embree-aarch64/common/simd/avx.h b/thirdparty/embree-aarch64/common/simd/avx.h
new file mode 100644
index 0000000000..c840e41805
--- /dev/null
+++ b/thirdparty/embree-aarch64/common/simd/avx.h
@@ -0,0 +1,34 @@
+// Copyright 2009-2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+#include "sse.h"
+
+#if defined(__AVX512VL__)
+#include "vboolf8_avx512.h"
+#include "vboold4_avx512.h"
+#else
+#include "vboolf8_avx.h"
+#include "vboold4_avx.h"
+#endif
+
+#if defined(__AVX2__)
+#include "vint8_avx2.h"
+#include "vuint8_avx2.h"
+#if defined(__X86_64__)
+#include "vllong4_avx2.h"
+#endif
+#else
+#include "vint8_avx.h"
+#include "vuint8_avx.h"
+#endif
+#include "vfloat8_avx.h"
+#if defined(__X86_64__)
+#include "vdouble4_avx.h"
+#endif
+
+#if defined(__AVX512F__)
+#include "avx512.h"
+#endif
+
diff --git a/thirdparty/embree-aarch64/common/simd/avx512.h b/thirdparty/embree-aarch64/common/simd/avx512.h
new file mode 100644
index 0000000000..25414ab5b1
--- /dev/null
+++ b/thirdparty/embree-aarch64/common/simd/avx512.h
@@ -0,0 +1,41 @@
+// Copyright 2009-2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+#include "../sys/platform.h"
+#include "../sys/intrinsics.h"
+#include "../math/constants.h"
+#include "../sys/alloc.h"
+#include "varying.h"
+
+#include "vboolf16_avx512.h"
+#include "vint16_avx512.h"
+#include "vuint16_avx512.h"
+#include "vfloat16_avx512.h"
+
+#include "vboold8_avx512.h"
+#include "vllong8_avx512.h"
+#include "vdouble8_avx512.h"
+
+namespace embree
+{
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Prefetching
+ ////////////////////////////////////////////////////////////////////////////////
+
+#define PFHINT_L1 0
+#define PFHINT_L2 1
+#define PFHINT_NT 2
+
+ template<const unsigned int mode>
+ __forceinline void prefetch(const void * __restrict__ const m)
+ {
+ if (mode == PFHINT_L1)
+ _mm_prefetch((const char*)m,_MM_HINT_T0);
+ else if (mode == PFHINT_L2)
+ _mm_prefetch((const char*)m,_MM_HINT_T1);
+ else if (mode == PFHINT_NT)
+ _mm_prefetch((const char*)m,_MM_HINT_NTA);
+ }
+}
diff --git a/thirdparty/embree-aarch64/common/simd/simd.h b/thirdparty/embree-aarch64/common/simd/simd.h
new file mode 100644
index 0000000000..647851110b
--- /dev/null
+++ b/thirdparty/embree-aarch64/common/simd/simd.h
@@ -0,0 +1,110 @@
+// Copyright 2009-2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+#include "../math/math.h"
+
+/* include SSE wrapper classes */
+#if defined(__SSE__) || defined(__ARM_NEON)
+# include "sse.h"
+#endif
+
+/* include AVX wrapper classes */
+#if defined(__AVX__)
+# include "avx.h"
+#endif
+
+/* include AVX512 wrapper classes */
+#if defined (__AVX512F__)
+# include "avx512.h"
+#endif
+
+namespace embree
+{
+ template <int N>
+ __forceinline vbool<N> isfinite(const vfloat<N>& v)
+ {
+ return (v >= vfloat<N>(-std::numeric_limits<float>::max()))
+ & (v <= vfloat<N>( std::numeric_limits<float>::max()));
+ }
+
+ /* foreach unique */
+ template<typename vbool, typename vint, typename Closure>
+ __forceinline void foreach_unique(const vbool& valid0, const vint& vi, const Closure& closure)
+ {
+ vbool valid1 = valid0;
+ while (any(valid1)) {
+ const int j = int(bsf(movemask(valid1)));
+ const int i = vi[j];
+ const vbool valid2 = valid1 & (i == vi);
+ valid1 = andn(valid1, valid2);
+ closure(valid2, i);
+ }
+ }
+
+ /* returns the next unique value i in vi and the corresponding valid_i mask */
+ template<typename vbool, typename vint>
+ __forceinline int next_unique(vbool& valid, const vint& vi, /*out*/ vbool& valid_i)
+ {
+ assert(any(valid));
+ const int j = int(bsf(movemask(valid)));
+ const int i = vi[j];
+ valid_i = valid & (i == vi);
+ valid = andn(valid, valid_i);
+ return i;
+ }
+
+ /* foreach unique index */
+ template<typename vbool, typename vint, typename Closure>
+ __forceinline void foreach_unique_index(const vbool& valid0, const vint& vi, const Closure& closure)
+ {
+ vbool valid1 = valid0;
+ while (any(valid1)) {
+ const int j = int(bsf(movemask(valid1)));
+ const int i = vi[j];
+ const vbool valid2 = valid1 & (i == vi);
+ valid1 = andn(valid1, valid2);
+ closure(valid2, i, j);
+ }
+ }
+
+ /* returns the index of the next unique value i in vi and the corresponding valid_i mask */
+ template<typename vbool, typename vint>
+ __forceinline int next_unique_index(vbool& valid, const vint& vi, /*out*/ vbool& valid_i)
+ {
+ assert(any(valid));
+ const int j = int(bsf(movemask(valid)));
+ const int i = vi[j];
+ valid_i = valid & (i == vi);
+ valid = andn(valid, valid_i);
+ return j;
+ }
+
+ template<typename Closure>
+ __forceinline void foreach2(int x0, int x1, int y0, int y1, const Closure& closure)
+ {
+ __aligned(64) int U[2*VSIZEX];
+ __aligned(64) int V[2*VSIZEX];
+ int index = 0;
+ for (int y=y0; y<y1; y++) {
+ const bool lasty = y+1>=y1;
+ const vintx vy = y;
+ for (int x=x0; x<x1; ) { //x+=VSIZEX) {
+ const bool lastx = x+VSIZEX >= x1;
+ vintx vx = x+vintx(step);
+ vintx::storeu(&U[index], vx);
+ vintx::storeu(&V[index], vy);
+ const int dx = min(x1-x,VSIZEX);
+ index += dx;
+ x += dx;
+ if (index >= VSIZEX || (lastx && lasty)) {
+ const vboolx valid = vintx(step) < vintx(index);
+ closure(valid, vintx::load(U), vintx::load(V));
+ x-= max(0, index-VSIZEX);
+ index = 0;
+ }
+ }
+ }
+ }
+}
diff --git a/thirdparty/embree-aarch64/common/simd/sse.cpp b/thirdparty/embree-aarch64/common/simd/sse.cpp
new file mode 100644
index 0000000000..1732cfa421
--- /dev/null
+++ b/thirdparty/embree-aarch64/common/simd/sse.cpp
@@ -0,0 +1,34 @@
+// Copyright 2009-2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+#include "sse.h"
+
+namespace embree
+{
+ const __m128 mm_lookupmask_ps[16] = {
+ _mm_castsi128_ps(_mm_set_epi32( 0, 0, 0, 0)),
+ _mm_castsi128_ps(_mm_set_epi32( 0, 0, 0,-1)),
+ _mm_castsi128_ps(_mm_set_epi32( 0, 0,-1, 0)),
+ _mm_castsi128_ps(_mm_set_epi32( 0, 0,-1,-1)),
+ _mm_castsi128_ps(_mm_set_epi32( 0,-1, 0, 0)),
+ _mm_castsi128_ps(_mm_set_epi32( 0,-1, 0,-1)),
+ _mm_castsi128_ps(_mm_set_epi32( 0,-1,-1, 0)),
+ _mm_castsi128_ps(_mm_set_epi32( 0,-1,-1,-1)),
+ _mm_castsi128_ps(_mm_set_epi32(-1, 0, 0, 0)),
+ _mm_castsi128_ps(_mm_set_epi32(-1, 0, 0,-1)),
+ _mm_castsi128_ps(_mm_set_epi32(-1, 0,-1, 0)),
+ _mm_castsi128_ps(_mm_set_epi32(-1, 0,-1,-1)),
+ _mm_castsi128_ps(_mm_set_epi32(-1,-1, 0, 0)),
+ _mm_castsi128_ps(_mm_set_epi32(-1,-1, 0,-1)),
+ _mm_castsi128_ps(_mm_set_epi32(-1,-1,-1, 0)),
+ _mm_castsi128_ps(_mm_set_epi32(-1,-1,-1,-1))
+ };
+
+ const __m128d mm_lookupmask_pd[4] = {
+ _mm_castsi128_pd(_mm_set_epi32( 0, 0, 0, 0)),
+ _mm_castsi128_pd(_mm_set_epi32( 0, 0,-1,-1)),
+ _mm_castsi128_pd(_mm_set_epi32(-1,-1, 0, 0)),
+ _mm_castsi128_pd(_mm_set_epi32(-1,-1,-1,-1))
+ };
+
+}
diff --git a/thirdparty/embree-aarch64/common/simd/sse.h b/thirdparty/embree-aarch64/common/simd/sse.h
new file mode 100644
index 0000000000..6bc818b55b
--- /dev/null
+++ b/thirdparty/embree-aarch64/common/simd/sse.h
@@ -0,0 +1,35 @@
+// Copyright 2009-2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+#include "../sys/platform.h"
+#include "../sys/intrinsics.h"
+#include "../sys/alloc.h"
+#include "../math/constants.h"
+#include "varying.h"
+
+namespace embree
+{
+#if (defined(__aarch64__) && defined(BUILD_IOS)) || defined(__SSE4_1__)
+ __forceinline __m128 blendv_ps(__m128 f, __m128 t, __m128 mask) {
+ return _mm_blendv_ps(f,t,mask);
+ }
+#else
+ __forceinline __m128 blendv_ps(__m128 f, __m128 t, __m128 mask) {
+ return _mm_or_ps(_mm_and_ps(mask, t), _mm_andnot_ps(mask, f));
+ }
+#endif
+
+ extern const __m128 mm_lookupmask_ps[16];
+ extern const __m128d mm_lookupmask_pd[4];
+}
+
+#if defined(__AVX512VL__)
+#include "vboolf4_avx512.h"
+#else
+#include "vboolf4_sse2.h"
+#endif
+#include "vint4_sse2.h"
+#include "vuint4_sse2.h"
+#include "vfloat4_sse2.h"
diff --git a/thirdparty/embree-aarch64/common/simd/varying.h b/thirdparty/embree-aarch64/common/simd/varying.h
new file mode 100644
index 0000000000..9a46817da9
--- /dev/null
+++ b/thirdparty/embree-aarch64/common/simd/varying.h
@@ -0,0 +1,132 @@
+// Copyright 2009-2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+#include "../sys/platform.h"
+
+namespace embree
+{
+ /* Varying numeric types */
+ template<int N>
+ struct vfloat
+ {
+ union { float f[N]; int i[N]; };
+ __forceinline const float& operator [](size_t index) const { assert(index < N); return f[index]; }
+ __forceinline float& operator [](size_t index) { assert(index < N); return f[index]; }
+ };
+
+ template<int N>
+ struct vdouble
+ {
+ union { double f[N]; long long i[N]; };
+ __forceinline const double& operator [](size_t index) const { assert(index < N); return f[index]; }
+ __forceinline double& operator [](size_t index) { assert(index < N); return f[index]; }
+ };
+
+ template<int N>
+ struct vint
+ {
+ int i[N];
+ __forceinline const int& operator [](size_t index) const { assert(index < N); return i[index]; }
+ __forceinline int& operator [](size_t index) { assert(index < N); return i[index]; }
+ };
+
+ template<int N>
+ struct vuint
+ {
+ unsigned int i[N];
+ __forceinline const unsigned int& operator [](size_t index) const { assert(index < N); return i[index]; }
+ __forceinline unsigned int& operator [](size_t index) { assert(index < N); return i[index]; }
+ };
+
+ template<int N>
+ struct vllong
+ {
+ long long i[N];
+ __forceinline const long long& operator [](size_t index) const { assert(index < N); return i[index]; }
+ __forceinline long long& operator [](size_t index) { assert(index < N); return i[index]; }
+ };
+
+ /* Varying bool types */
+ template<int N> struct vboolf { int i[N]; }; // for float/int
+ template<int N> struct vboold { long long i[N]; }; // for double/long long
+
+ /* Aliases to default types */
+ template<int N> using vreal = vfloat<N>;
+ template<int N> using vbool = vboolf<N>;
+
+ /* Varying size constants */
+#if defined(__AVX512VL__) // SKX
+ const int VSIZEX = 8; // default size
+ const int VSIZEL = 16; // large size
+#elif defined(__AVX512F__) // KNL
+ const int VSIZEX = 16;
+ const int VSIZEL = 16;
+#elif defined(__AVX__)
+ const int VSIZEX = 8;
+ const int VSIZEL = 8;
+#else
+ const int VSIZEX = 4;
+ const int VSIZEL = 4;
+#endif
+
+ /* Extends varying size N to optimal or up to max(N, N2) */
+ template<int N, int N2 = VSIZEX>
+ struct vextend
+ {
+#if defined(__AVX512F__) && !defined(__AVX512VL__) // KNL
+ /* use 16-wide SIMD calculations on KNL even for 4 and 8 wide SIMD */
+ static const int size = (N2 == VSIZEX) ? VSIZEX : N;
+ #define SIMD_MODE(N) N, 16
+#else
+ /* calculate with same SIMD width otherwise */
+ static const int size = N;
+ #define SIMD_MODE(N) N, N
+#endif
+ };
+
+ /* 4-wide shortcuts */
+ typedef vfloat<4> vfloat4;
+ typedef vdouble<4> vdouble4;
+ typedef vreal<4> vreal4;
+ typedef vint<4> vint4;
+ typedef vuint<4> vuint4;
+ typedef vllong<4> vllong4;
+ typedef vbool<4> vbool4;
+ typedef vboolf<4> vboolf4;
+ typedef vboold<4> vboold4;
+
+ /* 8-wide shortcuts */
+ typedef vfloat<8> vfloat8;
+ typedef vdouble<8> vdouble8;
+ typedef vreal<8> vreal8;
+ typedef vint<8> vint8;
+ typedef vuint<8> vuint8;
+ typedef vllong<8> vllong8;
+ typedef vbool<8> vbool8;
+ typedef vboolf<8> vboolf8;
+ typedef vboold<8> vboold8;
+
+ /* 16-wide shortcuts */
+ typedef vfloat<16> vfloat16;
+ typedef vdouble<16> vdouble16;
+ typedef vreal<16> vreal16;
+ typedef vint<16> vint16;
+ typedef vuint<16> vuint16;
+ typedef vllong<16> vllong16;
+ typedef vbool<16> vbool16;
+ typedef vboolf<16> vboolf16;
+ typedef vboold<16> vboold16;
+
+ /* Default shortcuts */
+ typedef vfloat<VSIZEX> vfloatx;
+ typedef vdouble<VSIZEX> vdoublex;
+ typedef vreal<VSIZEX> vrealx;
+ typedef vint<VSIZEX> vintx;
+ typedef vuint<VSIZEX> vuintx;
+ typedef vllong<VSIZEX> vllongx;
+ typedef vbool<VSIZEX> vboolx;
+ typedef vboolf<VSIZEX> vboolfx;
+ typedef vboold<VSIZEX> vbooldx;
+}
diff --git a/thirdparty/embree-aarch64/common/simd/vboold4_avx.h b/thirdparty/embree-aarch64/common/simd/vboold4_avx.h
new file mode 100644
index 0000000000..6505ee56f3
--- /dev/null
+++ b/thirdparty/embree-aarch64/common/simd/vboold4_avx.h
@@ -0,0 +1,160 @@
+// Copyright 2009-2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+namespace embree
+{
+ /* 4-wide AVX bool type for 64bit data types*/
+ template<>
+ struct vboold<4>
+ {
+ ALIGNED_STRUCT_(32);
+
+ typedef vboold4 Bool;
+
+ enum { size = 4 }; // number of SIMD elements
+ union { // data
+ __m256d v;
+ struct { __m128d vl,vh; };
+ long long i[4];
+ };
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constructors, Assignment & Cast Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboold() {}
+ __forceinline vboold(const vboold4& a) { v = a.v; }
+ __forceinline vboold4& operator =(const vboold4& a) { v = a.v; return *this; }
+
+ __forceinline vboold(__m256d a) : v(a) {}
+ __forceinline vboold(__m256i a) : v(_mm256_castsi256_pd(a)) {}
+
+ __forceinline operator const __m256() const { return _mm256_castpd_ps(v); }
+ __forceinline operator const __m256i() const { return _mm256_castpd_si256(v); }
+ __forceinline operator const __m256d() const { return v; }
+
+ __forceinline vboold(int a)
+ {
+ assert(a >= 0 && a <= 255);
+#if defined (__AVX2__)
+ const __m256i mask = _mm256_set_epi64x(0x8, 0x4, 0x2, 0x1);
+ const __m256i b = _mm256_set1_epi64x(a);
+ const __m256i c = _mm256_and_si256(b,mask);
+ v = _mm256_castsi256_pd(_mm256_cmpeq_epi64(c,mask));
+#else
+ vl = mm_lookupmask_pd[a & 0x3];
+ vh = mm_lookupmask_pd[a >> 2];
+#endif
+ }
+
+ __forceinline vboold(__m128d a, __m128d b) : vl(a), vh(b) {}
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constants
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboold(FalseTy) : v(_mm256_setzero_pd()) {}
+#if !defined(__aarch64__)
+ __forceinline vboold(TrueTy) : v(_mm256_cmp_pd(_mm256_setzero_pd(), _mm256_setzero_pd(), _CMP_EQ_OQ)) {}
+#else
+ __forceinline vboold(TrueTy) : v(_mm256_cmpeq_pd(_mm256_setzero_pd(), _mm256_setzero_pd())) {}
+#endif
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Array Access
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline bool operator [](size_t index) const { assert(index < 4); return (_mm256_movemask_pd(v) >> index) & 1; }
+ __forceinline long long& operator [](size_t index) { assert(index < 4); return i[index]; }
+ };
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Unary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboold4 operator !(const vboold4& a) { return _mm256_xor_pd(a, vboold4(embree::True)); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Binary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboold4 operator &(const vboold4& a, const vboold4& b) { return _mm256_and_pd(a, b); }
+ __forceinline vboold4 operator |(const vboold4& a, const vboold4& b) { return _mm256_or_pd (a, b); }
+ __forceinline vboold4 operator ^(const vboold4& a, const vboold4& b) { return _mm256_xor_pd(a, b); }
+
+ __forceinline vboold4 andn(const vboold4& a, const vboold4& b) { return _mm256_andnot_pd(b, a); }
+
+ __forceinline vboold4& operator &=(vboold4& a, const vboold4& b) { return a = a & b; }
+ __forceinline vboold4& operator |=(vboold4& a, const vboold4& b) { return a = a | b; }
+ __forceinline vboold4& operator ^=(vboold4& a, const vboold4& b) { return a = a ^ b; }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Comparison Operators + Select
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboold4 operator !=(const vboold4& a, const vboold4& b) { return _mm256_xor_pd(a, b); }
+ __forceinline vboold4 operator ==(const vboold4& a, const vboold4& b) { return _mm256_xor_pd(_mm256_xor_pd(a,b),vboold4(embree::True)); }
+
+ __forceinline vboold4 select(const vboold4& mask, const vboold4& t, const vboold4& f) {
+ return _mm256_blendv_pd(f, t, mask);
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Movement/Shifting/Shuffling Functions
+ ////////////////////////////////////////////////////////////////////////////////
+
+#if !defined(__aarch64__)
+ __forceinline vboold4 unpacklo(const vboold4& a, const vboold4& b) { return _mm256_unpacklo_pd(a, b); }
+ __forceinline vboold4 unpackhi(const vboold4& a, const vboold4& b) { return _mm256_unpackhi_pd(a, b); }
+#endif
+
+#if defined(__AVX2__)
+ template<int i0, int i1, int i2, int i3>
+ __forceinline vboold4 shuffle(const vboold4& v) {
+ return _mm256_permute4x64_pd(v, _MM_SHUFFLE(i3, i2, i1, i0));
+ }
+
+ template<int i>
+ __forceinline vboold4 shuffle(const vboold4& v) {
+ return _mm256_permute4x64_pd(v, _MM_SHUFFLE(i, i, i, i));
+ }
+#endif
+
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Reduction Operations
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline bool reduce_and(const vboold4& a) { return _mm256_movemask_pd(a) == (unsigned int)0xf; }
+ __forceinline bool reduce_or (const vboold4& a) { return !_mm256_testz_pd(a,a); }
+
+ __forceinline bool all (const vboold4& a) { return _mm256_movemask_pd(a) == (unsigned int)0xf; }
+ __forceinline bool any (const vboold4& a) { return !_mm256_testz_pd(a,a); }
+ __forceinline bool none(const vboold4& a) { return _mm256_testz_pd(a,a) != 0; }
+
+ __forceinline bool all (const vboold4& valid, const vboold4& b) { return all((!valid) | b); }
+ __forceinline bool any (const vboold4& valid, const vboold4& b) { return any(valid & b); }
+ __forceinline bool none(const vboold4& valid, const vboold4& b) { return none(valid & b); }
+
+ __forceinline unsigned int movemask(const vboold4& a) { return _mm256_movemask_pd(a); }
+ __forceinline size_t popcnt (const vboold4& a) { return popcnt((size_t)_mm256_movemask_pd(a)); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Get/Set Functions
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline bool get(const vboold4& a, size_t index) { return a[index]; }
+ __forceinline void set (vboold4& a, size_t index) { a[index] = -1; }
+ __forceinline void clear(vboold4& a, size_t index) { a[index] = 0; }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Output Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline embree_ostream operator <<(embree_ostream cout, const vboold4& a) {
+ return cout << "<" << a[0] << ", " << a[1] << ", " << a[2] << ", " << a[3] << ", "
+ << a[4] << ", " << a[5] << ", " << a[6] << ", " << a[7] << ">";
+ }
+}
diff --git a/thirdparty/embree-aarch64/common/simd/vboold4_avx512.h b/thirdparty/embree-aarch64/common/simd/vboold4_avx512.h
new file mode 100644
index 0000000000..4fe730d713
--- /dev/null
+++ b/thirdparty/embree-aarch64/common/simd/vboold4_avx512.h
@@ -0,0 +1,140 @@
+// Copyright 2009-2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+namespace embree
+{
+ /* 4-wide AVX-512 bool type */
+ template<>
+ struct vboold<4>
+ {
+ typedef vboold4 Bool;
+ typedef vint4 Int;
+
+ enum { size = 4 }; // number of SIMD elements
+ __mmask8 v; // data
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constructors, Assignment & Cast Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboold() {}
+ __forceinline vboold(const vboold4& t) { v = t.v; }
+ __forceinline vboold4& operator =(const vboold4& f) { v = f.v; return *this; }
+
+ __forceinline vboold(const __mmask8 &t) { v = t; }
+ __forceinline operator __mmask8() const { return v; }
+
+ __forceinline vboold(bool b) { v = b ? 0xf : 0x0; }
+ __forceinline vboold(int t) { v = (__mmask8)t; }
+ __forceinline vboold(unsigned int t) { v = (__mmask8)t; }
+
+ /* return int8 mask */
+ __forceinline __m128i mask8() const {
+ return _mm_movm_epi8(v);
+ }
+
+ /* return int32 mask */
+ __forceinline __m128i mask32() const {
+ return _mm_movm_epi32(v);
+ }
+
+ /* return int64 mask */
+ __forceinline __m256i mask64() const {
+ return _mm256_movm_epi64(v);
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constants
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboold(FalseTy) : v(0x0) {}
+ __forceinline vboold(TrueTy) : v(0xf) {}
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Array Access
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline bool operator [](size_t index) const {
+ assert(index < 4); return (mm512_mask2int(v) >> index) & 1;
+ }
+ };
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Unary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboold4 operator !(const vboold4& a) { return _mm512_kandn(a, 0xf); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Binary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboold4 operator &(const vboold4& a, const vboold4& b) { return _mm512_kand(a, b); }
+ __forceinline vboold4 operator |(const vboold4& a, const vboold4& b) { return _mm512_kor(a, b); }
+ __forceinline vboold4 operator ^(const vboold4& a, const vboold4& b) { return _mm512_kxor(a, b); }
+
+ __forceinline vboold4 andn(const vboold4& a, const vboold4& b) { return _mm512_kandn(b, a); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Assignment Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboold4& operator &=(vboold4& a, const vboold4& b) { return a = a & b; }
+ __forceinline vboold4& operator |=(vboold4& a, const vboold4& b) { return a = a | b; }
+ __forceinline vboold4& operator ^=(vboold4& a, const vboold4& b) { return a = a ^ b; }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Comparison Operators + Select
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboold4 operator !=(const vboold4& a, const vboold4& b) { return _mm512_kxor(a, b); }
+ __forceinline vboold4 operator ==(const vboold4& a, const vboold4& b) { return _mm512_kand(_mm512_kxnor(a, b), 0xf); }
+
+ __forceinline vboold4 select(const vboold4& s, const vboold4& a, const vboold4& b) {
+ return _mm512_kor(_mm512_kand(s, a), _mm512_kandn(s, b));
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Reduction Operations
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline int all (const vboold4& a) { return a.v == 0xf; }
+ __forceinline int any (const vboold4& a) { return _mm512_kortestz(a, a) == 0; }
+ __forceinline int none(const vboold4& a) { return _mm512_kortestz(a, a) != 0; }
+
+ __forceinline int all (const vboold4& valid, const vboold4& b) { return all((!valid) | b); }
+ __forceinline int any (const vboold4& valid, const vboold4& b) { return any(valid & b); }
+ __forceinline int none(const vboold4& valid, const vboold4& b) { return none(valid & b); }
+
+ __forceinline size_t movemask(const vboold4& a) { return _mm512_kmov(a); }
+ __forceinline size_t popcnt (const vboold4& a) { return popcnt(a.v); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Conversion Operations
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline unsigned int toInt(const vboold4& a) { return mm512_mask2int(a); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Get/Set Functions
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline bool get(const vboold4& a, size_t index) { assert(index < 4); return (toInt(a) >> index) & 1; }
+ __forceinline void set(vboold4& a, size_t index) { assert(index < 4); a |= 1 << index; }
+ __forceinline void clear(vboold4& a, size_t index) { assert(index < 4); a = andn(a, 1 << index); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Output Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline embree_ostream operator <<(embree_ostream cout, const vboold4& a)
+ {
+ cout << "<";
+ for (size_t i=0; i<4; i++) {
+ if ((a.v >> i) & 1) cout << "1"; else cout << "0";
+ }
+ return cout << ">";
+ }
+}
diff --git a/thirdparty/embree-aarch64/common/simd/vboold8_avx512.h b/thirdparty/embree-aarch64/common/simd/vboold8_avx512.h
new file mode 100644
index 0000000000..fdf3f00de5
--- /dev/null
+++ b/thirdparty/embree-aarch64/common/simd/vboold8_avx512.h
@@ -0,0 +1,148 @@
+// Copyright 2009-2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+namespace embree
+{
+ /* 8-wide AVX-512 bool type */
+ template<>
+ struct vboold<8>
+ {
+ typedef vboold8 Bool;
+ typedef vint8 Int;
+
+ enum { size = 8 }; // number of SIMD elements
+ __mmask8 v; // data
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constructors, Assignment & Cast Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboold() {}
+ __forceinline vboold(const vboold8& t) { v = t.v; }
+ __forceinline vboold8& operator =(const vboold8& f) { v = f.v; return *this; }
+
+ __forceinline vboold(const __mmask8& t) { v = t; }
+ __forceinline operator __mmask8() const { return v; }
+
+ __forceinline vboold(bool b) { v = b ? 0xff : 0x00; }
+ __forceinline vboold(int t) { v = (__mmask8)t; }
+ __forceinline vboold(unsigned int t) { v = (__mmask8)t; }
+
+ /* return int8 mask */
+ __forceinline __m128i mask8() const {
+#if defined(__AVX512BW__)
+ return _mm_movm_epi8(v);
+#else
+ const __m512i f = _mm512_set1_epi64(0);
+ const __m512i t = _mm512_set1_epi64(-1);
+ const __m512i m = _mm512_mask_or_epi64(f,v,t,t);
+ return _mm512_cvtepi64_epi8(m);
+#endif
+ }
+
+ /* return int64 mask */
+ __forceinline __m512i mask64() const {
+#if defined(__AVX512DQ__)
+ return _mm512_movm_epi64(v);
+#else
+ const __m512i f = _mm512_set1_epi64(0);
+ const __m512i t = _mm512_set1_epi64(-1);
+ return _mm512_mask_or_epi64(f,v,t,t);
+#endif
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constants
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboold(FalseTy) : v(0x00) {}
+ __forceinline vboold(TrueTy) : v(0xff) {}
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Array Access
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline bool operator [](size_t index) const {
+ assert(index < 8); return (mm512_mask2int(v) >> index) & 1;
+ }
+ };
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Unary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboold8 operator !(const vboold8& a) { return _mm512_knot(a); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Binary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboold8 operator &(const vboold8& a, const vboold8& b) { return _mm512_kand(a, b); }
+ __forceinline vboold8 operator |(const vboold8& a, const vboold8& b) { return _mm512_kor(a, b); }
+ __forceinline vboold8 operator ^(const vboold8& a, const vboold8& b) { return _mm512_kxor(a, b); }
+
+ __forceinline vboold8 andn(const vboold8& a, const vboold8& b) { return _mm512_kandn(b, a); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Assignment Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboold8& operator &=(vboold8& a, const vboold8& b) { return a = a & b; }
+ __forceinline vboold8& operator |=(vboold8& a, const vboold8& b) { return a = a | b; }
+ __forceinline vboold8& operator ^=(vboold8& a, const vboold8& b) { return a = a ^ b; }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Comparison Operators + Select
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboold8 operator !=(const vboold8& a, const vboold8& b) { return _mm512_kxor(a, b); }
+ __forceinline vboold8 operator ==(const vboold8& a, const vboold8& b) { return _mm512_kxnor(a, b); }
+
+ __forceinline vboold8 select(const vboold8& s, const vboold8& a, const vboold8& b) {
+ return _mm512_kor(_mm512_kand(s, a), _mm512_kandn(s, b));
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Reduction Operations
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline int all (const vboold8& a) { return a.v == 0xff; }
+ __forceinline int any (const vboold8& a) { return _mm512_kortestz(a, a) == 0; }
+ __forceinline int none(const vboold8& a) { return _mm512_kortestz(a, a) != 0; }
+
+ __forceinline int all (const vboold8& valid, const vboold8& b) { return all((!valid) | b); }
+ __forceinline int any (const vboold8& valid, const vboold8& b) { return any(valid & b); }
+ __forceinline int none(const vboold8& valid, const vboold8& b) { return none(valid & b); }
+
+ __forceinline size_t movemask(const vboold8& a) { return _mm512_kmov(a); }
+ __forceinline size_t popcnt (const vboold8& a) { return popcnt(a.v); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Conversion Operations
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline unsigned int toInt(const vboold8& a) { return mm512_mask2int(a); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Get/Set Functions
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline bool get(const vboold8& a, size_t index) { assert(index < 8); return (toInt(a) >> index) & 1; }
+ __forceinline void set(vboold8& a, size_t index) { assert(index < 8); a |= 1 << index; }
+ __forceinline void clear(vboold8& a, size_t index) { assert(index < 8); a = andn(a, 1 << index); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Output Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline embree_ostream operator <<(embree_ostream cout, const vboold8& a)
+ {
+ cout << "<";
+ for (size_t i=0; i<8; i++) {
+ if ((a.v >> i) & 1) cout << "1"; else cout << "0";
+ }
+ return cout << ">";
+ }
+}
diff --git a/thirdparty/embree-aarch64/common/simd/vboolf16_avx512.h b/thirdparty/embree-aarch64/common/simd/vboolf16_avx512.h
new file mode 100644
index 0000000000..238cdc8eb9
--- /dev/null
+++ b/thirdparty/embree-aarch64/common/simd/vboolf16_avx512.h
@@ -0,0 +1,150 @@
+// Copyright 2009-2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+namespace embree
+{
+ /* 16-wide AVX-512 bool type */
+ template<>
+ struct vboolf<16>
+ {
+ typedef vboolf16 Bool;
+ typedef vint16 Int;
+ typedef vfloat16 Float;
+
+ enum { size = 16 }; // number of SIMD elements
+ __mmask16 v; // data
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constructors, Assignment & Cast Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboolf() {}
+ __forceinline vboolf(const vboolf16& t) { v = t.v; }
+ __forceinline vboolf16& operator =(const vboolf16& f) { v = f.v; return *this; }
+
+ __forceinline vboolf(const __mmask16& t) { v = t; }
+ __forceinline operator __mmask16() const { return v; }
+
+ __forceinline vboolf(bool b) { v = b ? 0xFFFF : 0x0000; }
+ __forceinline vboolf(int t) { v = (__mmask16)t; }
+ __forceinline vboolf(unsigned int t) { v = (__mmask16)t; }
+
+ /* return int8 mask */
+ __forceinline __m128i mask8() const {
+#if defined(__AVX512BW__)
+ return _mm_movm_epi8(v);
+#else
+ const __m512i f = _mm512_set1_epi32(0);
+ const __m512i t = _mm512_set1_epi32(-1);
+ const __m512i m = _mm512_mask_or_epi32(f,v,t,t);
+ return _mm512_cvtepi32_epi8(m);
+#endif
+ }
+
+ /* return int32 mask */
+ __forceinline __m512i mask32() const {
+#if defined(__AVX512DQ__)
+ return _mm512_movm_epi32(v);
+#else
+ const __m512i f = _mm512_set1_epi32(0);
+ const __m512i t = _mm512_set1_epi32(-1);
+ return _mm512_mask_or_epi32(f,v,t,t);
+#endif
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constants
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboolf(FalseTy) : v(0x0000) {}
+ __forceinline vboolf(TrueTy) : v(0xffff) {}
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Array Access
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline bool operator [](size_t index) const {
+ assert(index < 16); return (mm512_mask2int(v) >> index) & 1;
+ }
+ };
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Unary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboolf16 operator !(const vboolf16& a) { return _mm512_knot(a); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Binary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboolf16 operator &(const vboolf16& a, const vboolf16& b) { return _mm512_kand(a,b); }
+ __forceinline vboolf16 operator |(const vboolf16& a, const vboolf16& b) { return _mm512_kor(a,b); }
+ __forceinline vboolf16 operator ^(const vboolf16& a, const vboolf16& b) { return _mm512_kxor(a,b); }
+
+ __forceinline vboolf16 andn(const vboolf16& a, const vboolf16& b) { return _mm512_kandn(b,a); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Assignment Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboolf16& operator &=(vboolf16& a, const vboolf16& b) { return a = a & b; }
+ __forceinline vboolf16& operator |=(vboolf16& a, const vboolf16& b) { return a = a | b; }
+ __forceinline vboolf16& operator ^=(vboolf16& a, const vboolf16& b) { return a = a ^ b; }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Comparison Operators + Select
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboolf16 operator !=(const vboolf16& a, const vboolf16& b) { return _mm512_kxor(a, b); }
+ __forceinline vboolf16 operator ==(const vboolf16& a, const vboolf16& b) { return _mm512_kxnor(a, b); }
+
+ __forceinline vboolf16 select(const vboolf16& s, const vboolf16& a, const vboolf16& b) {
+ return _mm512_kor(_mm512_kand(s,a),_mm512_kandn(s,b));
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Reduction Operations
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline int all (const vboolf16& a) { return _mm512_kortestc(a,a) != 0; }
+ __forceinline int any (const vboolf16& a) { return _mm512_kortestz(a,a) == 0; }
+ __forceinline int none(const vboolf16& a) { return _mm512_kortestz(a,a) != 0; }
+
+ __forceinline int all (const vboolf16& valid, const vboolf16& b) { return all((!valid) | b); }
+ __forceinline int any (const vboolf16& valid, const vboolf16& b) { return any(valid & b); }
+ __forceinline int none(const vboolf16& valid, const vboolf16& b) { return none(valid & b); }
+
+ __forceinline size_t movemask(const vboolf16& a) { return _mm512_kmov(a); }
+ __forceinline size_t popcnt (const vboolf16& a) { return popcnt(a.v); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Convertion Operations
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline unsigned int toInt (const vboolf16& a) { return mm512_mask2int(a); }
+ __forceinline vboolf16 toMask(const int& a) { return mm512_int2mask(a); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Get/Set Functions
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline bool get(const vboolf16& a, size_t index) { assert(index < 16); return (toInt(a) >> index) & 1; }
+ __forceinline void set(vboolf16& a, size_t index) { assert(index < 16); a |= 1 << index; }
+ __forceinline void clear(vboolf16& a, size_t index) { assert(index < 16); a = andn(a, 1 << index); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Output Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline embree_ostream operator <<(embree_ostream cout, const vboolf16& a)
+ {
+ cout << "<";
+ for (size_t i=0; i<16; i++) {
+ if ((a.v >> i) & 1) cout << "1"; else cout << "0";
+ }
+ return cout << ">";
+ }
+}
diff --git a/thirdparty/embree-aarch64/common/simd/vboolf4_avx512.h b/thirdparty/embree-aarch64/common/simd/vboolf4_avx512.h
new file mode 100644
index 0000000000..2ae4c4470e
--- /dev/null
+++ b/thirdparty/embree-aarch64/common/simd/vboolf4_avx512.h
@@ -0,0 +1,143 @@
+// Copyright 2009-2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+namespace embree
+{
+ /* 4-wide AVX-512 bool type */
+ template<>
+ struct vboolf<4>
+ {
+ typedef vboolf4 Bool;
+ typedef vint4 Int;
+
+ enum { size = 4 }; // number of SIMD elements
+ __mmask8 v; // data
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constructors, Assignment & Cast Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboolf() {}
+ __forceinline vboolf(const vboolf4& t) { v = t.v; }
+ __forceinline vboolf4& operator =(const vboolf4& f) { v = f.v; return *this; }
+
+ __forceinline vboolf(const __mmask8 &t) { v = t; }
+ __forceinline operator __mmask8() const { return v; }
+
+ __forceinline vboolf(bool b) { v = b ? 0xf : 0x0; }
+ __forceinline vboolf(int t) { v = (__mmask8)t; }
+ __forceinline vboolf(unsigned int t) { v = (__mmask8)t; }
+
+ __forceinline vboolf(bool a, bool b, bool c, bool d)
+ : v((__mmask8)((int(d) << 3) | (int(c) << 2) | (int(b) << 1) | int(a))) {}
+
+ /* return int8 mask */
+ __forceinline __m128i mask8() const {
+ return _mm_movm_epi8(v);
+ }
+
+ /* return int32 mask */
+ __forceinline __m128i mask32() const {
+ return _mm_movm_epi32(v);
+ }
+
+ /* return int64 mask */
+ __forceinline __m256i mask64() const {
+ return _mm256_movm_epi64(v);
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constants
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboolf(FalseTy) : v(0x0) {}
+ __forceinline vboolf(TrueTy) : v(0xf) {}
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Array Access
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline bool operator [](size_t index) const {
+ assert(index < 4); return (mm512_mask2int(v) >> index) & 1;
+ }
+ };
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Unary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboolf4 operator !(const vboolf4& a) { return _mm512_kandn(a, 0xf); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Binary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboolf4 operator &(const vboolf4& a, const vboolf4& b) { return _mm512_kand(a, b); }
+ __forceinline vboolf4 operator |(const vboolf4& a, const vboolf4& b) { return _mm512_kor(a, b); }
+ __forceinline vboolf4 operator ^(const vboolf4& a, const vboolf4& b) { return _mm512_kxor(a, b); }
+
+ __forceinline vboolf4 andn(const vboolf4& a, const vboolf4& b) { return _mm512_kandn(b, a); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Assignment Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboolf4& operator &=(vboolf4& a, const vboolf4& b) { return a = a & b; }
+ __forceinline vboolf4& operator |=(vboolf4& a, const vboolf4& b) { return a = a | b; }
+ __forceinline vboolf4& operator ^=(vboolf4& a, const vboolf4& b) { return a = a ^ b; }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Comparison Operators + Select
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboolf4 operator !=(const vboolf4& a, const vboolf4& b) { return _mm512_kxor(a, b); }
+ __forceinline vboolf4 operator ==(const vboolf4& a, const vboolf4& b) { return _mm512_kand(_mm512_kxnor(a, b), 0xf); }
+
+ __forceinline vboolf4 select(const vboolf4& s, const vboolf4& a, const vboolf4& b) {
+ return _mm512_kor(_mm512_kand(s, a), _mm512_kandn(s, b));
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Reduction Operations
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline int all (const vboolf4& a) { return a.v == 0xf; }
+ __forceinline int any (const vboolf4& a) { return _mm512_kortestz(a, a) == 0; }
+ __forceinline int none(const vboolf4& a) { return _mm512_kortestz(a, a) != 0; }
+
+ __forceinline int all (const vboolf4& valid, const vboolf4& b) { return all((!valid) | b); }
+ __forceinline int any (const vboolf4& valid, const vboolf4& b) { return any(valid & b); }
+ __forceinline int none(const vboolf4& valid, const vboolf4& b) { return none(valid & b); }
+
+ __forceinline size_t movemask(const vboolf4& a) { return _mm512_kmov(a); }
+ __forceinline size_t popcnt (const vboolf4& a) { return popcnt(a.v); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Conversion Operations
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline unsigned int toInt(const vboolf4& a) { return mm512_mask2int(a); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Get/Set Functions
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline bool get(const vboolf4& a, size_t index) { assert(index < 4); return (toInt(a) >> index) & 1; }
+ __forceinline void set(vboolf4& a, size_t index) { assert(index < 4); a |= 1 << index; }
+ __forceinline void clear(vboolf4& a, size_t index) { assert(index < 4); a = andn(a, 1 << index); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Output Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline embree_ostream operator <<(embree_ostream cout, const vboolf4& a)
+ {
+ cout << "<";
+ for (size_t i=0; i<4; i++) {
+ if ((a.v >> i) & 1) cout << "1"; else cout << "0";
+ }
+ return cout << ">";
+ }
+}
diff --git a/thirdparty/embree-aarch64/common/simd/vboolf4_sse2.h b/thirdparty/embree-aarch64/common/simd/vboolf4_sse2.h
new file mode 100644
index 0000000000..ed53b3c783
--- /dev/null
+++ b/thirdparty/embree-aarch64/common/simd/vboolf4_sse2.h
@@ -0,0 +1,198 @@
+// Copyright 2009-2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+namespace embree
+{
+ /* 4-wide SSE bool type */
+ template<>
+ struct vboolf<4>
+ {
+ ALIGNED_STRUCT_(16);
+
+ typedef vboolf4 Bool;
+ typedef vint4 Int;
+ typedef vfloat4 Float;
+
+ enum { size = 4 }; // number of SIMD elements
+ union { __m128 v; int i[4]; }; // data
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constructors, Assignment & Cast Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboolf() {}
+ __forceinline vboolf(const vboolf4& other) { v = other.v; }
+ __forceinline vboolf4& operator =(const vboolf4& other) { v = other.v; return *this; }
+
+ __forceinline vboolf(__m128 input) : v(input) {}
+ __forceinline operator const __m128&() const { return v; }
+ __forceinline operator const __m128i() const { return _mm_castps_si128(v); }
+ __forceinline operator const __m128d() const { return _mm_castps_pd(v); }
+
+ __forceinline vboolf(bool a)
+ : v(mm_lookupmask_ps[(size_t(a) << 3) | (size_t(a) << 2) | (size_t(a) << 1) | size_t(a)]) {}
+ __forceinline vboolf(bool a, bool b)
+ : v(mm_lookupmask_ps[(size_t(b) << 3) | (size_t(a) << 2) | (size_t(b) << 1) | size_t(a)]) {}
+ __forceinline vboolf(bool a, bool b, bool c, bool d)
+ : v(mm_lookupmask_ps[(size_t(d) << 3) | (size_t(c) << 2) | (size_t(b) << 1) | size_t(a)]) {}
+#if defined(__aarch64__) && defined(BUILD_IOS)
+ __forceinline vboolf(int mask) { v = mm_lookupmask_ps[mask]; }
+ __forceinline vboolf(unsigned int mask) { v = mm_lookupmask_ps[mask]; }
+#else
+ __forceinline vboolf(int mask) { assert(mask >= 0 && mask < 16); v = mm_lookupmask_ps[mask]; }
+ __forceinline vboolf(unsigned int mask) { assert(mask < 16); v = mm_lookupmask_ps[mask]; }
+#endif
+ /* return int32 mask */
+ __forceinline __m128i mask32() const {
+ return _mm_castps_si128(v);
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constants
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboolf(FalseTy) : v(_mm_setzero_ps()) {}
+ __forceinline vboolf(TrueTy) : v(_mm_castsi128_ps(_mm_cmpeq_epi32(_mm_setzero_si128(), _mm_setzero_si128()))) {}
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Array Access
+ ////////////////////////////////////////////////////////////////////////////////
+
+#if defined(__aarch64__) && defined(BUILD_IOS)
+ __forceinline bool operator [](size_t index) const { return (_mm_movemask_ps(v) >> index) & 1; }
+ __forceinline int& operator [](size_t index) { return i[index]; }
+#else
+ __forceinline bool operator [](size_t index) const { assert(index < 4); return (_mm_movemask_ps(v) >> index) & 1; }
+ __forceinline int& operator [](size_t index) { assert(index < 4); return i[index]; }
+#endif
+ };
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Unary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboolf4 operator !(const vboolf4& a) { return _mm_xor_ps(a, vboolf4(embree::True)); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Binary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboolf4 operator &(const vboolf4& a, const vboolf4& b) { return _mm_and_ps(a, b); }
+ __forceinline vboolf4 operator |(const vboolf4& a, const vboolf4& b) { return _mm_or_ps (a, b); }
+ __forceinline vboolf4 operator ^(const vboolf4& a, const vboolf4& b) { return _mm_xor_ps(a, b); }
+
+ __forceinline vboolf4 andn(const vboolf4& a, const vboolf4& b) { return _mm_andnot_ps(b, a); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Assignment Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboolf4& operator &=(vboolf4& a, const vboolf4& b) { return a = a & b; }
+ __forceinline vboolf4& operator |=(vboolf4& a, const vboolf4& b) { return a = a | b; }
+ __forceinline vboolf4& operator ^=(vboolf4& a, const vboolf4& b) { return a = a ^ b; }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Comparison Operators + Select
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboolf4 operator !=(const vboolf4& a, const vboolf4& b) { return _mm_xor_ps(a, b); }
+ __forceinline vboolf4 operator ==(const vboolf4& a, const vboolf4& b) { return _mm_castsi128_ps(_mm_cmpeq_epi32(a, b)); }
+
+ __forceinline vboolf4 select(const vboolf4& m, const vboolf4& t, const vboolf4& f) {
+#if (defined(__aarch64__) && defined(BUILD_IOS)) || defined(__SSE4_1__)
+ return _mm_blendv_ps(f, t, m);
+#else
+ return _mm_or_ps(_mm_and_ps(m, t), _mm_andnot_ps(m, f));
+#endif
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Movement/Shifting/Shuffling Functions
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboolf4 unpacklo(const vboolf4& a, const vboolf4& b) { return _mm_unpacklo_ps(a, b); }
+ __forceinline vboolf4 unpackhi(const vboolf4& a, const vboolf4& b) { return _mm_unpackhi_ps(a, b); }
+
+#if defined(__aarch64__)
+ template<int i0, int i1, int i2, int i3>
+ __forceinline vboolf4 shuffle(const vboolf4& v) {
+ return vreinterpretq_f32_u8(vqtbl1q_u8( vreinterpretq_u8_s32(v), _MN_SHUFFLE(i0, i1, i2, i3)));
+ }
+
+ template<int i0, int i1, int i2, int i3>
+ __forceinline vboolf4 shuffle(const vboolf4& a, const vboolf4& b) {
+ return vreinterpretq_f32_u8(vqtbl2q_u8( (uint8x16x2_t){(uint8x16_t)a.v, (uint8x16_t)b.v}, _MF_SHUFFLE(i0, i1, i2, i3)));
+ }
+#else
+ template<int i0, int i1, int i2, int i3>
+ __forceinline vboolf4 shuffle(const vboolf4& v) {
+ return _mm_castsi128_ps(_mm_shuffle_epi32(v, _MM_SHUFFLE(i3, i2, i1, i0)));
+ }
+
+ template<int i0, int i1, int i2, int i3>
+ __forceinline vboolf4 shuffle(const vboolf4& a, const vboolf4& b) {
+ return _mm_shuffle_ps(a, b, _MM_SHUFFLE(i3, i2, i1, i0));
+ }
+#endif
+
+ template<int i0>
+ __forceinline vboolf4 shuffle(const vboolf4& v) {
+ return shuffle<i0,i0,i0,i0>(v);
+ }
+
+#if defined(__SSE3__)
+ template<> __forceinline vboolf4 shuffle<0, 0, 2, 2>(const vboolf4& v) { return _mm_moveldup_ps(v); }
+ template<> __forceinline vboolf4 shuffle<1, 1, 3, 3>(const vboolf4& v) { return _mm_movehdup_ps(v); }
+ template<> __forceinline vboolf4 shuffle<0, 1, 0, 1>(const vboolf4& v) { return _mm_castpd_ps(_mm_movedup_pd(v)); }
+#endif
+
+#if defined(__SSE4_1__) && !defined(__aarch64__)
+ template<int dst, int src, int clr> __forceinline vboolf4 insert(const vboolf4& a, const vboolf4& b) { return _mm_insert_ps(a, b, (dst << 4) | (src << 6) | clr); }
+ template<int dst, int src> __forceinline vboolf4 insert(const vboolf4& a, const vboolf4& b) { return insert<dst, src, 0>(a, b); }
+ template<int dst> __forceinline vboolf4 insert(const vboolf4& a, const bool b) { return insert<dst, 0>(a, vboolf4(b)); }
+#endif
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Reduction Operations
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline bool reduce_and(const vboolf4& a) { return _mm_movemask_ps(a) == 0xf; }
+ __forceinline bool reduce_or (const vboolf4& a) { return _mm_movemask_ps(a) != 0x0; }
+
+ __forceinline bool all (const vboolf4& b) { return _mm_movemask_ps(b) == 0xf; }
+ __forceinline bool any (const vboolf4& b) { return _mm_movemask_ps(b) != 0x0; }
+ __forceinline bool none(const vboolf4& b) { return _mm_movemask_ps(b) == 0x0; }
+
+ __forceinline bool all (const vboolf4& valid, const vboolf4& b) { return all((!valid) | b); }
+ __forceinline bool any (const vboolf4& valid, const vboolf4& b) { return any(valid & b); }
+ __forceinline bool none(const vboolf4& valid, const vboolf4& b) { return none(valid & b); }
+
+ __forceinline size_t movemask(const vboolf4& a) { return _mm_movemask_ps(a); }
+#if defined(__aarch64__) && defined(BUILD_IOS)
+__forceinline size_t popcnt(const vboolf4& a) { return _mm_movemask_popcnt_ps(a); }
+#else
+#if defined(__SSE4_2__)
+ __forceinline size_t popcnt(const vboolf4& a) { return popcnt((size_t)_mm_movemask_ps(a)); }
+#else
+ __forceinline size_t popcnt(const vboolf4& a) { return bool(a[0])+bool(a[1])+bool(a[2])+bool(a[3]); }
+#endif
+#endif
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Get/Set Functions
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline bool get(const vboolf4& a, size_t index) { return a[index]; }
+ __forceinline void set(vboolf4& a, size_t index) { a[index] = -1; }
+ __forceinline void clear(vboolf4& a, size_t index) { a[index] = 0; }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Output Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline embree_ostream operator <<(embree_ostream cout, const vboolf4& a) {
+ return cout << "<" << a[0] << ", " << a[1] << ", " << a[2] << ", " << a[3] << ">";
+ }
+}
diff --git a/thirdparty/embree-aarch64/common/simd/vboolf8_avx.h b/thirdparty/embree-aarch64/common/simd/vboolf8_avx.h
new file mode 100644
index 0000000000..4f64741b55
--- /dev/null
+++ b/thirdparty/embree-aarch64/common/simd/vboolf8_avx.h
@@ -0,0 +1,189 @@
+// Copyright 2009-2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+namespace embree
+{
+ /* 8-wide AVX bool type */
+ template<>
+ struct vboolf<8>
+ {
+ ALIGNED_STRUCT_(32);
+
+ typedef vboolf8 Bool;
+ typedef vint8 Int;
+ typedef vfloat8 Float;
+
+ enum { size = 8 }; // number of SIMD elements
+ union { // data
+ __m256 v;
+ struct { __m128 vl,vh; };
+ int i[8];
+ };
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constructors, Assignment & Cast Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboolf() {}
+ __forceinline vboolf(const vboolf8& a) { v = a.v; }
+ __forceinline vboolf8& operator =(const vboolf8& a) { v = a.v; return *this; }
+
+ __forceinline vboolf(__m256 a) : v(a) {}
+ __forceinline operator const __m256&() const { return v; }
+ __forceinline operator const __m256i() const { return _mm256_castps_si256(v); }
+ __forceinline operator const __m256d() const { return _mm256_castps_pd(v); }
+
+ __forceinline vboolf(int a)
+ {
+ assert(a >= 0 && a <= 255);
+#if defined (__AVX2__)
+ const __m256i mask = _mm256_set_epi32(0x80, 0x40, 0x20, 0x10, 0x8, 0x4, 0x2, 0x1);
+ const __m256i b = _mm256_set1_epi32(a);
+ const __m256i c = _mm256_and_si256(b,mask);
+ v = _mm256_castsi256_ps(_mm256_cmpeq_epi32(c,mask));
+#else
+ vl = mm_lookupmask_ps[a & 0xF];
+ vh = mm_lookupmask_ps[a >> 4];
+#endif
+ }
+
+ __forceinline vboolf(const vboolf4& a) : v(_mm256_insertf128_ps(_mm256_castps128_ps256(a),a,1)) {}
+ __forceinline vboolf(const vboolf4& a, const vboolf4& b) : v(_mm256_insertf128_ps(_mm256_castps128_ps256(a),b,1)) {}
+ __forceinline vboolf(__m128 a, __m128 b) : vl(a), vh(b) {}
+
+ __forceinline vboolf(bool a) : v(vboolf8(vboolf4(a), vboolf4(a))) {}
+ __forceinline vboolf(bool a, bool b) : v(vboolf8(vboolf4(a), vboolf4(b))) {}
+ __forceinline vboolf(bool a, bool b, bool c, bool d) : v(vboolf8(vboolf4(a,b), vboolf4(c,d))) {}
+ __forceinline vboolf(bool a, bool b, bool c, bool d, bool e, bool f, bool g, bool h) : v(vboolf8(vboolf4(a,b,c,d), vboolf4(e,f,g,h))) {}
+
+ /* return int32 mask */
+ __forceinline __m256i mask32() const {
+ return _mm256_castps_si256(v);
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constants
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboolf(FalseTy) : v(_mm256_setzero_ps()) {}
+#if !defined(__aarch64__)
+ __forceinline vboolf(TrueTy) : v(_mm256_cmp_ps(_mm256_setzero_ps(), _mm256_setzero_ps(), _CMP_EQ_OQ)) {}
+#else
+ __forceinline vboolf(TrueTy) : v(_mm256_cmpeq_ps(_mm256_setzero_ps(), _mm256_setzero_ps())) {}
+#endif
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Array Access
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline bool operator [](size_t index) const { assert(index < 8); return (_mm256_movemask_ps(v) >> index) & 1; }
+ __forceinline int& operator [](size_t index) { assert(index < 8); return i[index]; }
+ };
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Unary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboolf8 operator !(const vboolf8& a) { return _mm256_xor_ps(a, vboolf8(embree::True)); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Binary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboolf8 operator &(const vboolf8& a, const vboolf8& b) { return _mm256_and_ps(a, b); }
+ __forceinline vboolf8 operator |(const vboolf8& a, const vboolf8& b) { return _mm256_or_ps (a, b); }
+ __forceinline vboolf8 operator ^(const vboolf8& a, const vboolf8& b) { return _mm256_xor_ps(a, b); }
+
+ __forceinline vboolf8 andn(const vboolf8& a, const vboolf8& b) { return _mm256_andnot_ps(b, a); }
+
+ __forceinline vboolf8& operator &=(vboolf8& a, const vboolf8& b) { return a = a & b; }
+ __forceinline vboolf8& operator |=(vboolf8& a, const vboolf8& b) { return a = a | b; }
+ __forceinline vboolf8& operator ^=(vboolf8& a, const vboolf8& b) { return a = a ^ b; }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Comparison Operators + Select
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboolf8 operator !=(const vboolf8& a, const vboolf8& b) { return _mm256_xor_ps(a, b); }
+ __forceinline vboolf8 operator ==(const vboolf8& a, const vboolf8& b) { return _mm256_xor_ps(_mm256_xor_ps(a,b),vboolf8(embree::True)); }
+
+ __forceinline vboolf8 select(const vboolf8& mask, const vboolf8& t, const vboolf8& f) {
+ return _mm256_blendv_ps(f, t, mask);
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Movement/Shifting/Shuffling Functions
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboolf8 unpacklo(const vboolf8& a, const vboolf8& b) { return _mm256_unpacklo_ps(a, b); }
+ __forceinline vboolf8 unpackhi(const vboolf8& a, const vboolf8& b) { return _mm256_unpackhi_ps(a, b); }
+
+ template<int i>
+ __forceinline vboolf8 shuffle(const vboolf8& v) {
+ return _mm256_permute_ps(v, _MM_SHUFFLE(i, i, i, i));
+ }
+
+ template<int i0, int i1>
+ __forceinline vboolf8 shuffle4(const vboolf8& v) {
+ return _mm256_permute2f128_ps(v, v, (i1 << 4) | (i0 << 0));
+ }
+
+ template<int i0, int i1>
+ __forceinline vboolf8 shuffle4(const vboolf8& a, const vboolf8& b) {
+ return _mm256_permute2f128_ps(a, b, (i1 << 4) | (i0 << 0));
+ }
+
+ template<int i0, int i1, int i2, int i3>
+ __forceinline vboolf8 shuffle(const vboolf8& v) {
+ return _mm256_permute_ps(v, _MM_SHUFFLE(i3, i2, i1, i0));
+ }
+
+ template<int i0, int i1, int i2, int i3>
+ __forceinline vboolf8 shuffle(const vboolf8& a, const vboolf8& b) {
+ return _mm256_shuffle_ps(a, b, _MM_SHUFFLE(i3, i2, i1, i0));
+ }
+
+ template<> __forceinline vboolf8 shuffle<0, 0, 2, 2>(const vboolf8& v) { return _mm256_moveldup_ps(v); }
+ template<> __forceinline vboolf8 shuffle<1, 1, 3, 3>(const vboolf8& v) { return _mm256_movehdup_ps(v); }
+ template<> __forceinline vboolf8 shuffle<0, 1, 0, 1>(const vboolf8& v) { return _mm256_castpd_ps(_mm256_movedup_pd(_mm256_castps_pd(v))); }
+
+ template<int i> __forceinline vboolf8 insert4(const vboolf8& a, const vboolf4& b) { return _mm256_insertf128_ps(a, b, i); }
+ template<int i> __forceinline vboolf4 extract4 (const vboolf8& a) { return _mm256_extractf128_ps(a, i); }
+ template<> __forceinline vboolf4 extract4<0>(const vboolf8& a) { return _mm256_castps256_ps128(a); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Reduction Operations
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline bool reduce_and(const vboolf8& a) { return _mm256_movemask_ps(a) == (unsigned int)0xff; }
+ __forceinline bool reduce_or (const vboolf8& a) { return !_mm256_testz_ps(a,a); }
+
+ __forceinline bool all (const vboolf8& a) { return _mm256_movemask_ps(a) == (unsigned int)0xff; }
+ __forceinline bool any (const vboolf8& a) { return !_mm256_testz_ps(a,a); }
+ __forceinline bool none(const vboolf8& a) { return _mm256_testz_ps(a,a) != 0; }
+
+ __forceinline bool all (const vboolf8& valid, const vboolf8& b) { return all((!valid) | b); }
+ __forceinline bool any (const vboolf8& valid, const vboolf8& b) { return any(valid & b); }
+ __forceinline bool none(const vboolf8& valid, const vboolf8& b) { return none(valid & b); }
+
+ __forceinline unsigned int movemask(const vboolf8& a) { return _mm256_movemask_ps(a); }
+ __forceinline size_t popcnt (const vboolf8& a) { return popcnt((size_t)_mm256_movemask_ps(a)); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Get/Set Functions
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline bool get(const vboolf8& a, size_t index) { return a[index]; }
+ __forceinline void set(vboolf8& a, size_t index) { a[index] = -1; }
+ __forceinline void clear(vboolf8& a, size_t index) { a[index] = 0; }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Output Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline embree_ostream operator <<(embree_ostream cout, const vboolf8& a) {
+ return cout << "<" << a[0] << ", " << a[1] << ", " << a[2] << ", " << a[3] << ", "
+ << a[4] << ", " << a[5] << ", " << a[6] << ", " << a[7] << ">";
+ }
+}
diff --git a/thirdparty/embree-aarch64/common/simd/vboolf8_avx512.h b/thirdparty/embree-aarch64/common/simd/vboolf8_avx512.h
new file mode 100644
index 0000000000..2a52b554c7
--- /dev/null
+++ b/thirdparty/embree-aarch64/common/simd/vboolf8_avx512.h
@@ -0,0 +1,143 @@
+// Copyright 2009-2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+namespace embree
+{
+ /* 8-wide AVX-512 bool type */
+ template<>
+ struct vboolf<8>
+ {
+ typedef vboolf8 Bool;
+ typedef vint8 Int;
+
+ enum { size = 8 }; // number of SIMD elements
+ __mmask8 v; // data
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constructors, Assignment & Cast Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboolf() {}
+ __forceinline vboolf(const vboolf8& t) { v = t.v; }
+ __forceinline vboolf8& operator =(const vboolf8& f) { v = f.v; return *this; }
+
+ __forceinline vboolf(const __mmask8 &t) { v = t; }
+ __forceinline operator __mmask8() const { return v; }
+
+ __forceinline vboolf(bool b) { v = b ? 0xff : 0x00; }
+ __forceinline vboolf(int t) { v = (__mmask8)t; }
+ __forceinline vboolf(unsigned int t) { v = (__mmask8)t; }
+
+ __forceinline vboolf(bool a, bool b, bool c, bool d, bool e, bool f, bool g, bool h)
+ : v((__mmask8)((int(h) << 7) | (int(g) << 6) | (int(f) << 5) | (int(e) << 4) | (int(d) << 3) | (int(c) << 2) | (int(b) << 1) | int(a))) {}
+
+ /* return int8 mask */
+ __forceinline __m128i mask8() const {
+ return _mm_movm_epi8(v);
+ }
+
+ /* return int32 mask */
+ __forceinline __m256i mask32() const {
+ return _mm256_movm_epi32(v);
+ }
+
+ /* return int64 mask */
+ __forceinline __m512i mask64() const {
+ return _mm512_movm_epi64(v);
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constants
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboolf(FalseTy) : v(0x00) {}
+ __forceinline vboolf(TrueTy) : v(0xff) {}
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Array Access
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline bool operator [](size_t index) const {
+ assert(index < 8); return (mm512_mask2int(v) >> index) & 1;
+ }
+ };
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Unary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboolf8 operator !(const vboolf8& a) { return _mm512_knot(a); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Binary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboolf8 operator &(const vboolf8& a, const vboolf8& b) { return _mm512_kand(a, b); }
+ __forceinline vboolf8 operator |(const vboolf8& a, const vboolf8& b) { return _mm512_kor(a, b); }
+ __forceinline vboolf8 operator ^(const vboolf8& a, const vboolf8& b) { return _mm512_kxor(a, b); }
+
+ __forceinline vboolf8 andn(const vboolf8& a, const vboolf8& b) { return _mm512_kandn(b, a); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Assignment Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboolf8& operator &=(vboolf8& a, const vboolf8& b) { return a = a & b; }
+ __forceinline vboolf8& operator |=(vboolf8& a, const vboolf8& b) { return a = a | b; }
+ __forceinline vboolf8& operator ^=(vboolf8& a, const vboolf8& b) { return a = a ^ b; }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Comparison Operators + Select
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboolf8 operator !=(const vboolf8& a, const vboolf8& b) { return _mm512_kxor(a, b); }
+ __forceinline vboolf8 operator ==(const vboolf8& a, const vboolf8& b) { return _mm512_kxnor(a, b); }
+
+ __forceinline vboolf8 select(const vboolf8& s, const vboolf8& a, const vboolf8& b) {
+ return _mm512_kor(_mm512_kand(s, a), _mm512_kandn(s, b));
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Reduction Operations
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline int all (const vboolf8& a) { return a.v == 0xff; }
+ __forceinline int any (const vboolf8& a) { return _mm512_kortestz(a, a) == 0; }
+ __forceinline int none(const vboolf8& a) { return _mm512_kortestz(a, a) != 0; }
+
+ __forceinline int all (const vboolf8& valid, const vboolf8& b) { return all((!valid) | b); }
+ __forceinline int any (const vboolf8& valid, const vboolf8& b) { return any(valid & b); }
+ __forceinline int none(const vboolf8& valid, const vboolf8& b) { return none(valid & b); }
+
+ __forceinline size_t movemask(const vboolf8& a) { return _mm512_kmov(a); }
+ __forceinline size_t popcnt (const vboolf8& a) { return popcnt(a.v); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Conversion Operations
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline unsigned int toInt(const vboolf8& a) { return mm512_mask2int(a); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Get/Set Functions
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline bool get(const vboolf8& a, size_t index) { assert(index < 8); return (toInt(a) >> index) & 1; }
+ __forceinline void set(vboolf8& a, size_t index) { assert(index < 8); a |= 1 << index; }
+ __forceinline void clear(vboolf8& a, size_t index) { assert(index < 8); a = andn(a, 1 << index); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Output Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline embree_ostream operator <<(embree_ostream cout, const vboolf8& a)
+ {
+ cout << "<";
+ for (size_t i=0; i<8; i++) {
+ if ((a.v >> i) & 1) cout << "1"; else cout << "0";
+ }
+ return cout << ">";
+ }
+}
diff --git a/thirdparty/embree-aarch64/common/simd/vdouble4_avx.h b/thirdparty/embree-aarch64/common/simd/vdouble4_avx.h
new file mode 100644
index 0000000000..1f65b45d7e
--- /dev/null
+++ b/thirdparty/embree-aarch64/common/simd/vdouble4_avx.h
@@ -0,0 +1,324 @@
+// Copyright 2009-2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+namespace embree
+{
+ /* 4-wide AVX 64-bit double type */
+ template<>
+ struct vdouble<4>
+ {
+ ALIGNED_STRUCT_(32);
+
+ typedef vboold4 Bool;
+
+ enum { size = 4 }; // number of SIMD elements
+ union { // data
+ __m256d v;
+ double i[4];
+ };
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constructors, Assignment & Cast Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vdouble() {}
+ __forceinline vdouble(const vdouble4& t) { v = t.v; }
+ __forceinline vdouble4& operator =(const vdouble4& f) { v = f.v; return *this; }
+
+ __forceinline vdouble(const __m256d& t) { v = t; }
+ __forceinline operator __m256d() const { return v; }
+
+ __forceinline vdouble(double i) {
+ v = _mm256_set1_pd(i);
+ }
+
+ __forceinline vdouble(double a, double b, double c, double d) {
+ v = _mm256_set_pd(d,c,b,a);
+ }
+
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constants
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vdouble(ZeroTy) : v(_mm256_setzero_pd()) {}
+ __forceinline vdouble(OneTy) : v(_mm256_set1_pd(1)) {}
+ __forceinline vdouble(StepTy) : v(_mm256_set_pd(3.0,2.0,1.0,0.0)) {}
+ __forceinline vdouble(ReverseStepTy) : v(_mm256_setr_pd(3.0,2.0,1.0,0.0)) {}
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Loads and Stores
+ ////////////////////////////////////////////////////////////////////////////////
+
+ static __forceinline void store_nt(double *__restrict__ ptr, const vdouble4& a) {
+ _mm256_stream_pd(ptr, a);
+ }
+
+ static __forceinline vdouble4 loadu(const double* addr) {
+ return _mm256_loadu_pd(addr);
+ }
+
+ static __forceinline vdouble4 load(const vdouble4* addr) {
+ return _mm256_load_pd((double*)addr);
+ }
+
+ static __forceinline vdouble4 load(const double* addr) {
+ return _mm256_load_pd(addr);
+ }
+
+ static __forceinline void store(double* ptr, const vdouble4& v) {
+ _mm256_store_pd(ptr, v);
+ }
+
+ static __forceinline void storeu(double* ptr, const vdouble4& v) {
+ _mm256_storeu_pd(ptr, v);
+ }
+
+ static __forceinline vdouble4 broadcast(const void* a) { return _mm256_set1_pd(*(double*)a); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Array Access
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline double& operator [](size_t index) { assert(index < 4); return i[index]; }
+ __forceinline const double& operator [](size_t index) const { assert(index < 4); return i[index]; }
+ };
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Unary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+#if defined(__AVX2__)
+ __forceinline vdouble4 asDouble(const vllong4& a) { return _mm256_castsi256_pd(a); }
+ __forceinline vllong4 asLLong (const vdouble4& a) { return _mm256_castpd_si256(a); }
+#endif
+
+ __forceinline vdouble4 operator +(const vdouble4& a) { return a; }
+ __forceinline vdouble4 operator -(const vdouble4& a) { return _mm256_sub_pd(_mm256_setzero_pd(), a); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Binary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vdouble4 operator +(const vdouble4& a, const vdouble4& b) { return _mm256_add_pd(a, b); }
+ __forceinline vdouble4 operator +(const vdouble4& a, double b) { return a + vdouble4(b); }
+ __forceinline vdouble4 operator +(double a, const vdouble4& b) { return vdouble4(a) + b; }
+
+ __forceinline vdouble4 operator -(const vdouble4& a, const vdouble4& b) { return _mm256_sub_pd(a, b); }
+ __forceinline vdouble4 operator -(const vdouble4& a, double b) { return a - vdouble4(b); }
+ __forceinline vdouble4 operator -(double a, const vdouble4& b) { return vdouble4(a) - b; }
+
+ __forceinline vdouble4 operator *(const vdouble4& a, const vdouble4& b) { return _mm256_mul_pd(a, b); }
+ __forceinline vdouble4 operator *(const vdouble4& a, double b) { return a * vdouble4(b); }
+ __forceinline vdouble4 operator *(double a, const vdouble4& b) { return vdouble4(a) * b; }
+
+ __forceinline vdouble4 operator &(const vdouble4& a, const vdouble4& b) { return _mm256_and_pd(a, b); }
+ __forceinline vdouble4 operator &(const vdouble4& a, double b) { return a & vdouble4(b); }
+ __forceinline vdouble4 operator &(double a, const vdouble4& b) { return vdouble4(a) & b; }
+
+ __forceinline vdouble4 operator |(const vdouble4& a, const vdouble4& b) { return _mm256_or_pd(a, b); }
+ __forceinline vdouble4 operator |(const vdouble4& a, double b) { return a | vdouble4(b); }
+ __forceinline vdouble4 operator |(double a, const vdouble4& b) { return vdouble4(a) | b; }
+
+ __forceinline vdouble4 operator ^(const vdouble4& a, const vdouble4& b) { return _mm256_xor_pd(a, b); }
+ __forceinline vdouble4 operator ^(const vdouble4& a, double b) { return a ^ vdouble4(b); }
+ __forceinline vdouble4 operator ^(double a, const vdouble4& b) { return vdouble4(a) ^ b; }
+
+ __forceinline vdouble4 min(const vdouble4& a, const vdouble4& b) { return _mm256_min_pd(a, b); }
+ __forceinline vdouble4 min(const vdouble4& a, double b) { return min(a,vdouble4(b)); }
+ __forceinline vdouble4 min(double a, const vdouble4& b) { return min(vdouble4(a),b); }
+
+ __forceinline vdouble4 max(const vdouble4& a, const vdouble4& b) { return _mm256_max_pd(a, b); }
+ __forceinline vdouble4 max(const vdouble4& a, double b) { return max(a,vdouble4(b)); }
+ __forceinline vdouble4 max(double a, const vdouble4& b) { return max(vdouble4(a),b); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Ternary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+#if defined(__FMA__)
+ __forceinline vdouble4 madd (const vdouble4& a, const vdouble4& b, const vdouble4& c) { return _mm256_fmadd_pd(a,b,c); }
+ __forceinline vdouble4 msub (const vdouble4& a, const vdouble4& b, const vdouble4& c) { return _mm256_fmsub_pd(a,b,c); }
+ __forceinline vdouble4 nmadd(const vdouble4& a, const vdouble4& b, const vdouble4& c) { return _mm256_fnmadd_pd(a,b,c); }
+ __forceinline vdouble4 nmsub(const vdouble4& a, const vdouble4& b, const vdouble4& c) { return _mm256_fnmsub_pd(a,b,c); }
+#else
+ __forceinline vdouble4 madd (const vdouble4& a, const vdouble4& b, const vdouble4& c) { return a*b+c; }
+ __forceinline vdouble4 msub (const vdouble4& a, const vdouble4& b, const vdouble4& c) { return a*b-c; }
+ __forceinline vdouble4 nmadd(const vdouble4& a, const vdouble4& b, const vdouble4& c) { return -a*b+c;}
+ __forceinline vdouble4 nmsub(const vdouble4& a, const vdouble4& b, const vdouble4& c) { return -a*b-c; }
+#endif
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Assignment Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vdouble4& operator +=(vdouble4& a, const vdouble4& b) { return a = a + b; }
+ __forceinline vdouble4& operator +=(vdouble4& a, double b) { return a = a + b; }
+
+ __forceinline vdouble4& operator -=(vdouble4& a, const vdouble4& b) { return a = a - b; }
+ __forceinline vdouble4& operator -=(vdouble4& a, double b) { return a = a - b; }
+
+ __forceinline vdouble4& operator *=(vdouble4& a, const vdouble4& b) { return a = a * b; }
+ __forceinline vdouble4& operator *=(vdouble4& a, double b) { return a = a * b; }
+
+ __forceinline vdouble4& operator &=(vdouble4& a, const vdouble4& b) { return a = a & b; }
+ __forceinline vdouble4& operator &=(vdouble4& a, double b) { return a = a & b; }
+
+ __forceinline vdouble4& operator |=(vdouble4& a, const vdouble4& b) { return a = a | b; }
+ __forceinline vdouble4& operator |=(vdouble4& a, double b) { return a = a | b; }
+
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Comparison Operators + Select
+ ////////////////////////////////////////////////////////////////////////////////
+
+#if defined(__AVX512VL__)
+ __forceinline vboold4 operator ==(const vdouble4& a, const vdouble4& b) { return _mm256_cmp_pd_mask(a, b, _MM_CMPINT_EQ); }
+ __forceinline vboold4 operator !=(const vdouble4& a, const vdouble4& b) { return _mm256_cmp_pd_mask(a, b, _MM_CMPINT_NE); }
+ __forceinline vboold4 operator < (const vdouble4& a, const vdouble4& b) { return _mm256_cmp_pd_mask(a, b, _MM_CMPINT_LT); }
+ __forceinline vboold4 operator >=(const vdouble4& a, const vdouble4& b) { return _mm256_cmp_pd_mask(a, b, _MM_CMPINT_GE); }
+ __forceinline vboold4 operator > (const vdouble4& a, const vdouble4& b) { return _mm256_cmp_pd_mask(a, b, _MM_CMPINT_GT); }
+ __forceinline vboold4 operator <=(const vdouble4& a, const vdouble4& b) { return _mm256_cmp_pd_mask(a, b, _MM_CMPINT_LE); }
+#elif !defined(__aarch64__)
+ __forceinline vboold4 operator ==(const vdouble4& a, const vdouble4& b) { return _mm256_cmp_pd(a, b, _CMP_EQ_OQ); }
+ __forceinline vboold4 operator !=(const vdouble4& a, const vdouble4& b) { return _mm256_cmp_pd(a, b, _CMP_NEQ_UQ); }
+ __forceinline vboold4 operator < (const vdouble4& a, const vdouble4& b) { return _mm256_cmp_pd(a, b, _CMP_LT_OS); }
+ __forceinline vboold4 operator >=(const vdouble4& a, const vdouble4& b) { return _mm256_cmp_pd(a, b, _CMP_NLT_US); }
+ __forceinline vboold4 operator > (const vdouble4& a, const vdouble4& b) { return _mm256_cmp_pd(a, b, _CMP_NLE_US); }
+ __forceinline vboold4 operator <=(const vdouble4& a, const vdouble4& b) { return _mm256_cmp_pd(a, b, _CMP_LE_OS); }
+#else
+ __forceinline vboold4 operator ==(const vdouble4& a, const vdouble4& b) { return _mm256_cmpeq_pd(a, b); }
+ __forceinline vboold4 operator !=(const vdouble4& a, const vdouble4& b) { return _mm256_cmpneq_pd(a, b); }
+ __forceinline vboold4 operator < (const vdouble4& a, const vdouble4& b) { return _mm256_cmplt_pd(a, b); }
+ __forceinline vboold4 operator >=(const vdouble4& a, const vdouble4& b) { return _mm256_cmpnlt_pd(a, b); }
+ __forceinline vboold4 operator > (const vdouble4& a, const vdouble4& b) { return _mm256_cmpnle_pd(a, b); }
+ __forceinline vboold4 operator <=(const vdouble4& a, const vdouble4& b) { return _mm256_cmple_pd(a, b); }
+#endif
+
+ __forceinline vboold4 operator ==(const vdouble4& a, double b) { return a == vdouble4(b); }
+ __forceinline vboold4 operator ==(double a, const vdouble4& b) { return vdouble4(a) == b; }
+
+ __forceinline vboold4 operator !=(const vdouble4& a, double b) { return a != vdouble4(b); }
+ __forceinline vboold4 operator !=(double a, const vdouble4& b) { return vdouble4(a) != b; }
+
+ __forceinline vboold4 operator < (const vdouble4& a, double b) { return a < vdouble4(b); }
+ __forceinline vboold4 operator < (double a, const vdouble4& b) { return vdouble4(a) < b; }
+
+ __forceinline vboold4 operator >=(const vdouble4& a, double b) { return a >= vdouble4(b); }
+ __forceinline vboold4 operator >=(double a, const vdouble4& b) { return vdouble4(a) >= b; }
+
+ __forceinline vboold4 operator > (const vdouble4& a, double b) { return a > vdouble4(b); }
+ __forceinline vboold4 operator > (double a, const vdouble4& b) { return vdouble4(a) > b; }
+
+ __forceinline vboold4 operator <=(const vdouble4& a, double b) { return a <= vdouble4(b); }
+ __forceinline vboold4 operator <=(double a, const vdouble4& b) { return vdouble4(a) <= b; }
+
+ __forceinline vboold4 eq(const vdouble4& a, const vdouble4& b) { return a == b; }
+ __forceinline vboold4 ne(const vdouble4& a, const vdouble4& b) { return a != b; }
+ __forceinline vboold4 lt(const vdouble4& a, const vdouble4& b) { return a < b; }
+ __forceinline vboold4 ge(const vdouble4& a, const vdouble4& b) { return a >= b; }
+ __forceinline vboold4 gt(const vdouble4& a, const vdouble4& b) { return a > b; }
+ __forceinline vboold4 le(const vdouble4& a, const vdouble4& b) { return a <= b; }
+
+#if defined(__AVX512VL__)
+ __forceinline vboold4 eq(const vboold4& mask, const vdouble4& a, const vdouble4& b) { return _mm256_mask_cmp_pd_mask(mask, a, b, _MM_CMPINT_EQ); }
+ __forceinline vboold4 ne(const vboold4& mask, const vdouble4& a, const vdouble4& b) { return _mm256_mask_cmp_pd_mask(mask, a, b, _MM_CMPINT_NE); }
+ __forceinline vboold4 lt(const vboold4& mask, const vdouble4& a, const vdouble4& b) { return _mm256_mask_cmp_pd_mask(mask, a, b, _MM_CMPINT_LT); }
+ __forceinline vboold4 ge(const vboold4& mask, const vdouble4& a, const vdouble4& b) { return _mm256_mask_cmp_pd_mask(mask, a, b, _MM_CMPINT_GE); }
+ __forceinline vboold4 gt(const vboold4& mask, const vdouble4& a, const vdouble4& b) { return _mm256_mask_cmp_pd_mask(mask, a, b, _MM_CMPINT_GT); }
+ __forceinline vboold4 le(const vboold4& mask, const vdouble4& a, const vdouble4& b) { return _mm256_mask_cmp_pd_mask(mask, a, b, _MM_CMPINT_LE); }
+#else
+ __forceinline vboold4 eq(const vboold4& mask, const vdouble4& a, const vdouble4& b) { return mask & (a == b); }
+ __forceinline vboold4 ne(const vboold4& mask, const vdouble4& a, const vdouble4& b) { return mask & (a != b); }
+ __forceinline vboold4 lt(const vboold4& mask, const vdouble4& a, const vdouble4& b) { return mask & (a < b); }
+ __forceinline vboold4 ge(const vboold4& mask, const vdouble4& a, const vdouble4& b) { return mask & (a >= b); }
+ __forceinline vboold4 gt(const vboold4& mask, const vdouble4& a, const vdouble4& b) { return mask & (a > b); }
+ __forceinline vboold4 le(const vboold4& mask, const vdouble4& a, const vdouble4& b) { return mask & (a <= b); }
+#endif
+
+ __forceinline vdouble4 select(const vboold4& m, const vdouble4& t, const vdouble4& f) {
+#if defined(__AVX512VL__)
+ return _mm256_mask_blend_pd(m, f, t);
+#else
+ return _mm256_blendv_pd(f, t, m);
+#endif
+ }
+
+ __forceinline void xchg(const vboold4& m, vdouble4& a, vdouble4& b) {
+ const vdouble4 c = a; a = select(m,b,a); b = select(m,c,b);
+ }
+
+ __forceinline vboold4 test(const vdouble4& a, const vdouble4& b) {
+#if defined(__AVX512VL__)
+ return _mm256_test_epi64_mask(_mm256_castpd_si256(a),_mm256_castpd_si256(b));
+#else
+ return _mm256_testz_si256(_mm256_castpd_si256(a),_mm256_castpd_si256(b));
+#endif
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ // Movement/Shifting/Shuffling Functions
+ ////////////////////////////////////////////////////////////////////////////////
+
+ template<int i0, int i1>
+ __forceinline vdouble4 shuffle(const vdouble4& v) {
+ return _mm256_permute_pd(v, (i1 << 3) | (i0 << 2) | (i1 << 1) | i0);
+ }
+
+ template<int i>
+ __forceinline vdouble4 shuffle(const vdouble4& v) {
+ return shuffle<i, i>(v);
+ }
+
+ template<int i0, int i1>
+ __forceinline vdouble4 shuffle2(const vdouble4& v) {
+ return _mm256_permute2f128_pd(v, v, (i1 << 4) | i0);
+ }
+
+ __forceinline double toScalar(const vdouble4& v) {
+ return _mm_cvtsd_f64(_mm256_castpd256_pd128(v));
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Reductions
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vdouble4 vreduce_min2(const vdouble4& x) { return min(x, shuffle<1,0>(x)); }
+ __forceinline vdouble4 vreduce_min (const vdouble4& y) { const vdouble4 x = vreduce_min2(y); return min(x, shuffle2<1,0>(x)); }
+
+ __forceinline vdouble4 vreduce_max2(const vdouble4& x) { return max(x,shuffle<1,0>(x)); }
+ __forceinline vdouble4 vreduce_max (const vdouble4& y) { const vdouble4 x = vreduce_max2(y); return max(x, shuffle2<1,0>(x)); }
+
+ __forceinline vdouble4 vreduce_and2(const vdouble4& x) { return x & shuffle<1,0>(x); }
+ __forceinline vdouble4 vreduce_and (const vdouble4& y) { const vdouble4 x = vreduce_and2(y); return x & shuffle2<1,0>(x); }
+
+ __forceinline vdouble4 vreduce_or2(const vdouble4& x) { return x | shuffle<1,0>(x); }
+ __forceinline vdouble4 vreduce_or (const vdouble4& y) { const vdouble4 x = vreduce_or2(y); return x | shuffle2<1,0>(x); }
+
+ __forceinline vdouble4 vreduce_add2(const vdouble4& x) { return x + shuffle<1,0>(x); }
+ __forceinline vdouble4 vreduce_add (const vdouble4& y) { const vdouble4 x = vreduce_add2(y); return x + shuffle2<1,0>(x); }
+
+ __forceinline double reduce_add(const vdouble4& a) { return toScalar(vreduce_add(a)); }
+ __forceinline double reduce_min(const vdouble4& a) { return toScalar(vreduce_min(a)); }
+ __forceinline double reduce_max(const vdouble4& a) { return toScalar(vreduce_max(a)); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Memory load and store operations
+ ////////////////////////////////////////////////////////////////////////////////
+
+
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Output Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline embree_ostream operator <<(embree_ostream cout, const vdouble4& v)
+ {
+ cout << "<" << v[0];
+ for (size_t i=1; i<4; i++) cout << ", " << v[i];
+ cout << ">";
+ return cout;
+ }
+}
diff --git a/thirdparty/embree-aarch64/common/simd/vdouble8_avx512.h b/thirdparty/embree-aarch64/common/simd/vdouble8_avx512.h
new file mode 100644
index 0000000000..4eec7d2f6a
--- /dev/null
+++ b/thirdparty/embree-aarch64/common/simd/vdouble8_avx512.h
@@ -0,0 +1,356 @@
+// Copyright 2009-2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+namespace embree
+{
+ /* 8-wide AVX-512 64-bit double type */
+ template<>
+ struct vdouble<8>
+ {
+ ALIGNED_STRUCT_(64);
+
+ typedef vboold8 Bool;
+
+ enum { size = 8 }; // number of SIMD elements
+ union { // data
+ __m512d v;
+ double i[8];
+ };
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constructors, Assignment & Cast Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vdouble() {}
+ __forceinline vdouble(const vdouble8& t) { v = t.v; }
+ __forceinline vdouble8& operator =(const vdouble8& f) { v = f.v; return *this; }
+
+ __forceinline vdouble(const __m512d& t) { v = t; }
+ __forceinline operator __m512d() const { return v; }
+ __forceinline operator __m256d() const { return _mm512_castpd512_pd256(v); }
+
+ __forceinline vdouble(double i) {
+ v = _mm512_set1_pd(i);
+ }
+
+ __forceinline vdouble(double a, double b, double c, double d) {
+ v = _mm512_set4_pd(d,c,b,a);
+ }
+
+ __forceinline vdouble(double a0, double a1, double a2, double a3,
+ double a4, double a5, double a6, double a7)
+ {
+ v = _mm512_set_pd(a7,a6,a5,a4,a3,a2,a1,a0);
+ }
+
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constants
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vdouble(ZeroTy) : v(_mm512_setzero_pd()) {}
+ __forceinline vdouble(OneTy) : v(_mm512_set1_pd(1)) {}
+ __forceinline vdouble(StepTy) : v(_mm512_set_pd(7.0,6.0,5.0,4.0,3.0,2.0,1.0,0.0)) {}
+ __forceinline vdouble(ReverseStepTy) : v(_mm512_setr_pd(7.0,6.0,5.0,4.0,3.0,2.0,1.0,0.0)) {}
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Loads and Stores
+ ////////////////////////////////////////////////////////////////////////////////
+
+ static __forceinline void store_nt(void *__restrict__ ptr, const vdouble8& a) {
+ _mm512_stream_pd((double*)ptr, a);
+ }
+
+ static __forceinline vdouble8 loadu(const void* addr) {
+ return _mm512_loadu_pd((double*)addr);
+ }
+
+ static __forceinline vdouble8 load(const vdouble8* addr) {
+ return _mm512_load_pd((double*)addr);
+ }
+
+ static __forceinline vdouble8 load(const double* addr) {
+ return _mm512_load_pd(addr);
+ }
+
+ static __forceinline void store(void* ptr, const vdouble8& v) {
+ _mm512_store_pd(ptr, v);
+ }
+
+ static __forceinline void storeu(void* ptr, const vdouble8& v) {
+ _mm512_storeu_pd(ptr, v);
+ }
+
+ static __forceinline void storeu(const vboold8& mask, double* ptr, const vdouble8& f) {
+ _mm512_mask_storeu_pd(ptr, mask, f);
+ }
+
+ static __forceinline void store(const vboold8& mask, void* addr, const vdouble8& v2) {
+ _mm512_mask_store_pd(addr, mask, v2);
+ }
+
+ /* pass by value to avoid compiler generating inefficient code */
+ static __forceinline void storeu_compact(const vboold8 mask,void * addr, const vdouble8& reg) {
+ _mm512_mask_compressstoreu_pd(addr, mask, reg);
+ }
+
+ static __forceinline vdouble8 compact64bit(const vboold8& mask, vdouble8& v) {
+ return _mm512_mask_compress_pd(v, mask, v);
+ }
+
+ static __forceinline vdouble8 compact(const vboold8& mask, vdouble8& v) {
+ return _mm512_mask_compress_pd(v, mask, v);
+ }
+
+ static __forceinline vdouble8 compact(const vboold8& mask, const vdouble8& a, vdouble8& b) {
+ return _mm512_mask_compress_pd(a, mask, b);
+ }
+
+ static __forceinline vdouble8 broadcast(const void* a) { return _mm512_set1_pd(*(double*)a); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Array Access
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline double& operator [](size_t index) { assert(index < 8); return i[index]; }
+ __forceinline const double& operator [](size_t index) const { assert(index < 8); return i[index]; }
+
+ };
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Unary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vdouble8 asDouble(const vllong8& a) { return _mm512_castsi512_pd(a); }
+ __forceinline vllong8 asLLong (const vdouble8& a) { return _mm512_castpd_si512(a); }
+
+ __forceinline vdouble8 operator +(const vdouble8& a) { return a; }
+ __forceinline vdouble8 operator -(const vdouble8& a) { return _mm512_sub_pd(_mm512_setzero_pd(), a); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Binary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vdouble8 operator +(const vdouble8& a, const vdouble8& b) { return _mm512_add_pd(a, b); }
+ __forceinline vdouble8 operator +(const vdouble8& a, double b) { return a + vdouble8(b); }
+ __forceinline vdouble8 operator +(double a, const vdouble8& b) { return vdouble8(a) + b; }
+
+ __forceinline vdouble8 operator -(const vdouble8& a, const vdouble8& b) { return _mm512_sub_pd(a, b); }
+ __forceinline vdouble8 operator -(const vdouble8& a, double b) { return a - vdouble8(b); }
+ __forceinline vdouble8 operator -(double a, const vdouble8& b) { return vdouble8(a) - b; }
+
+ __forceinline vdouble8 operator *(const vdouble8& a, const vdouble8& b) { return _mm512_mul_pd(a, b); }
+ __forceinline vdouble8 operator *(const vdouble8& a, double b) { return a * vdouble8(b); }
+ __forceinline vdouble8 operator *(double a, const vdouble8& b) { return vdouble8(a) * b; }
+
+ __forceinline vdouble8 operator &(const vdouble8& a, const vdouble8& b) { return _mm512_and_pd(a, b); }
+ __forceinline vdouble8 operator &(const vdouble8& a, double b) { return a & vdouble8(b); }
+ __forceinline vdouble8 operator &(double a, const vdouble8& b) { return vdouble8(a) & b; }
+
+ __forceinline vdouble8 operator |(const vdouble8& a, const vdouble8& b) { return _mm512_or_pd(a, b); }
+ __forceinline vdouble8 operator |(const vdouble8& a, double b) { return a | vdouble8(b); }
+ __forceinline vdouble8 operator |(double a, const vdouble8& b) { return vdouble8(a) | b; }
+
+ __forceinline vdouble8 operator ^(const vdouble8& a, const vdouble8& b) { return _mm512_xor_pd(a, b); }
+ __forceinline vdouble8 operator ^(const vdouble8& a, double b) { return a ^ vdouble8(b); }
+ __forceinline vdouble8 operator ^(double a, const vdouble8& b) { return vdouble8(a) ^ b; }
+
+ __forceinline vdouble8 operator <<(const vdouble8& a, const unsigned int n) { return _mm512_castsi512_pd(_mm512_slli_epi64(_mm512_castpd_si512(a), n)); }
+ __forceinline vdouble8 operator >>(const vdouble8& a, const unsigned int n) { return _mm512_castsi512_pd(_mm512_srai_epi64(_mm512_castpd_si512(a), n)); }
+
+ __forceinline vdouble8 operator <<(const vdouble8& a, const vllong8& n) { return _mm512_castsi512_pd(_mm512_sllv_epi64(_mm512_castpd_si512(a), n)); }
+ __forceinline vdouble8 operator >>(const vdouble8& a, const vllong8& n) { return _mm512_castsi512_pd(_mm512_srav_epi64(_mm512_castpd_si512(a), n)); }
+
+ __forceinline vdouble8 sll (const vdouble8& a, const unsigned int b) { return _mm512_castsi512_pd(_mm512_slli_epi64(_mm512_castpd_si512(a), b)); }
+ __forceinline vdouble8 sra (const vdouble8& a, const unsigned int b) { return _mm512_castsi512_pd(_mm512_srai_epi64(_mm512_castpd_si512(a), b)); }
+ __forceinline vdouble8 srl (const vdouble8& a, const unsigned int b) { return _mm512_castsi512_pd(_mm512_srli_epi64(_mm512_castpd_si512(a), b)); }
+
+ __forceinline vdouble8 min(const vdouble8& a, const vdouble8& b) { return _mm512_min_pd(a, b); }
+ __forceinline vdouble8 min(const vdouble8& a, double b) { return min(a,vdouble8(b)); }
+ __forceinline vdouble8 min(double a, const vdouble8& b) { return min(vdouble8(a),b); }
+
+ __forceinline vdouble8 max(const vdouble8& a, const vdouble8& b) { return _mm512_max_pd(a, b); }
+ __forceinline vdouble8 max(const vdouble8& a, double b) { return max(a,vdouble8(b)); }
+ __forceinline vdouble8 max(double a, const vdouble8& b) { return max(vdouble8(a),b); }
+
+ __forceinline vdouble8 mask_add(const vboold8& mask, vdouble8& c, const vdouble8& a, const vdouble8& b) { return _mm512_mask_add_pd(c,mask,a,b); }
+ __forceinline vdouble8 mask_sub(const vboold8& mask, vdouble8& c, const vdouble8& a, const vdouble8& b) { return _mm512_mask_sub_pd(c,mask,a,b); }
+
+ __forceinline vdouble8 mask_and(const vboold8& m,vdouble8& c, const vdouble8& a, const vdouble8& b) { return _mm512_mask_and_pd(c,m,a,b); }
+ __forceinline vdouble8 mask_or (const vboold8& m,vdouble8& c, const vdouble8& a, const vdouble8& b) { return _mm512_mask_or_pd(c,m,a,b); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Ternary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vdouble8 madd (const vdouble8& a, const vdouble8& b, const vdouble8& c) { return _mm512_fmadd_pd(a,b,c); }
+ __forceinline vdouble8 msub (const vdouble8& a, const vdouble8& b, const vdouble8& c) { return _mm512_fmsub_pd(a,b,c); }
+ __forceinline vdouble8 nmadd(const vdouble8& a, const vdouble8& b, const vdouble8& c) { return _mm512_fnmadd_pd(a,b,c); }
+ __forceinline vdouble8 nmsub(const vdouble8& a, const vdouble8& b, const vdouble8& c) { return _mm512_fnmsub_pd(a,b,c); }
+
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Assignment Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vdouble8& operator +=(vdouble8& a, const vdouble8& b) { return a = a + b; }
+ __forceinline vdouble8& operator +=(vdouble8& a, double b) { return a = a + b; }
+
+ __forceinline vdouble8& operator -=(vdouble8& a, const vdouble8& b) { return a = a - b; }
+ __forceinline vdouble8& operator -=(vdouble8& a, double b) { return a = a - b; }
+
+ __forceinline vdouble8& operator *=(vdouble8& a, const vdouble8& b) { return a = a * b; }
+ __forceinline vdouble8& operator *=(vdouble8& a, double b) { return a = a * b; }
+
+ __forceinline vdouble8& operator &=(vdouble8& a, const vdouble8& b) { return a = a & b; }
+ __forceinline vdouble8& operator &=(vdouble8& a, double b) { return a = a & b; }
+
+ __forceinline vdouble8& operator |=(vdouble8& a, const vdouble8& b) { return a = a | b; }
+ __forceinline vdouble8& operator |=(vdouble8& a, double b) { return a = a | b; }
+
+ __forceinline vdouble8& operator <<=(vdouble8& a, const double b) { return a = a << b; }
+ __forceinline vdouble8& operator >>=(vdouble8& a, const double b) { return a = a >> b; }
+
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Comparison Operators + Select
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboold8 operator ==(const vdouble8& a, const vdouble8& b) { return _mm512_cmp_pd_mask(a,b,_MM_CMPINT_EQ); }
+ __forceinline vboold8 operator ==(const vdouble8& a, double b) { return a == vdouble8(b); }
+ __forceinline vboold8 operator ==(double a, const vdouble8& b) { return vdouble8(a) == b; }
+
+ __forceinline vboold8 operator !=(const vdouble8& a, const vdouble8& b) { return _mm512_cmp_pd_mask(a,b,_MM_CMPINT_NE); }
+ __forceinline vboold8 operator !=(const vdouble8& a, double b) { return a != vdouble8(b); }
+ __forceinline vboold8 operator !=(double a, const vdouble8& b) { return vdouble8(a) != b; }
+
+ __forceinline vboold8 operator < (const vdouble8& a, const vdouble8& b) { return _mm512_cmp_pd_mask(a,b,_MM_CMPINT_LT); }
+ __forceinline vboold8 operator < (const vdouble8& a, double b) { return a < vdouble8(b); }
+ __forceinline vboold8 operator < (double a, const vdouble8& b) { return vdouble8(a) < b; }
+
+ __forceinline vboold8 operator >=(const vdouble8& a, const vdouble8& b) { return _mm512_cmp_pd_mask(a,b,_MM_CMPINT_GE); }
+ __forceinline vboold8 operator >=(const vdouble8& a, double b) { return a >= vdouble8(b); }
+ __forceinline vboold8 operator >=(double a, const vdouble8& b) { return vdouble8(a) >= b; }
+
+ __forceinline vboold8 operator > (const vdouble8& a, const vdouble8& b) { return _mm512_cmp_pd_mask(a,b,_MM_CMPINT_GT); }
+ __forceinline vboold8 operator > (const vdouble8& a, double b) { return a > vdouble8(b); }
+ __forceinline vboold8 operator > (double a, const vdouble8& b) { return vdouble8(a) > b; }
+
+ __forceinline vboold8 operator <=(const vdouble8& a, const vdouble8& b) { return _mm512_cmp_pd_mask(a,b,_MM_CMPINT_LE); }
+ __forceinline vboold8 operator <=(const vdouble8& a, double b) { return a <= vdouble8(b); }
+ __forceinline vboold8 operator <=(double a, const vdouble8& b) { return vdouble8(a) <= b; }
+
+ __forceinline vboold8 eq(const vdouble8& a, const vdouble8& b) { return _mm512_cmp_pd_mask(a,b,_MM_CMPINT_EQ); }
+ __forceinline vboold8 ne(const vdouble8& a, const vdouble8& b) { return _mm512_cmp_pd_mask(a,b,_MM_CMPINT_NE); }
+ __forceinline vboold8 lt(const vdouble8& a, const vdouble8& b) { return _mm512_cmp_pd_mask(a,b,_MM_CMPINT_LT); }
+ __forceinline vboold8 ge(const vdouble8& a, const vdouble8& b) { return _mm512_cmp_pd_mask(a,b,_MM_CMPINT_GE); }
+ __forceinline vboold8 gt(const vdouble8& a, const vdouble8& b) { return _mm512_cmp_pd_mask(a,b,_MM_CMPINT_GT); }
+ __forceinline vboold8 le(const vdouble8& a, const vdouble8& b) { return _mm512_cmp_pd_mask(a,b,_MM_CMPINT_LE); }
+
+ __forceinline vboold8 eq(const vboold8 mask, const vdouble8& a, const vdouble8& b) { return _mm512_mask_cmp_pd_mask(mask,a,b,_MM_CMPINT_EQ); }
+ __forceinline vboold8 ne(const vboold8 mask, const vdouble8& a, const vdouble8& b) { return _mm512_mask_cmp_pd_mask(mask,a,b,_MM_CMPINT_NE); }
+ __forceinline vboold8 lt(const vboold8 mask, const vdouble8& a, const vdouble8& b) { return _mm512_mask_cmp_pd_mask(mask,a,b,_MM_CMPINT_LT); }
+ __forceinline vboold8 ge(const vboold8 mask, const vdouble8& a, const vdouble8& b) { return _mm512_mask_cmp_pd_mask(mask,a,b,_MM_CMPINT_GE); }
+ __forceinline vboold8 gt(const vboold8 mask, const vdouble8& a, const vdouble8& b) { return _mm512_mask_cmp_pd_mask(mask,a,b,_MM_CMPINT_GT); }
+ __forceinline vboold8 le(const vboold8 mask, const vdouble8& a, const vdouble8& b) { return _mm512_mask_cmp_pd_mask(mask,a,b,_MM_CMPINT_LE); }
+
+ __forceinline vdouble8 select(const vboold8& m, const vdouble8& t, const vdouble8& f) {
+ return _mm512_mask_or_pd(f,m,t,t);
+ }
+
+ __forceinline void xchg(const vboold8& m, vdouble8& a, vdouble8& b) {
+ const vdouble8 c = a; a = select(m,b,a); b = select(m,c,b);
+ }
+
+ __forceinline vboold8 test(const vboold8& m, const vdouble8& a, const vdouble8& b) {
+ return _mm512_mask_test_epi64_mask(m,_mm512_castpd_si512(a),_mm512_castpd_si512(b));
+ }
+
+ __forceinline vboold8 test(const vdouble8& a, const vdouble8& b) {
+ return _mm512_test_epi64_mask(_mm512_castpd_si512(a),_mm512_castpd_si512(b));
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ // Movement/Shifting/Shuffling Functions
+ ////////////////////////////////////////////////////////////////////////////////
+
+ template<int i0, int i1>
+ __forceinline vdouble8 shuffle(const vdouble8& v) {
+ return _mm512_permute_pd(v, (i1 << 7) | (i0 << 6) | (i1 << 5) | (i0 << 4) | (i1 << 3) | (i0 << 2) | (i1 << 1) | i0);
+ }
+
+ template<int i>
+ __forceinline vdouble8 shuffle(const vdouble8& v) {
+ return shuffle<i, i>(v);
+ }
+
+ template<int i0, int i1, int i2, int i3>
+ __forceinline vdouble8 shuffle(const vdouble8& v) {
+ return _mm512_permutex_pd(v, _MM_SHUFFLE(i3, i2, i1, i0));
+ }
+
+ template<int i0, int i1>
+ __forceinline vdouble8 shuffle4(const vdouble8& v) {
+ return _mm512_shuffle_f64x2(v, v, _MM_SHUFFLE(i1*2+1, i1*2, i0*2+1, i0*2));
+ }
+
+ template<int i>
+ __forceinline vdouble8 shuffle4(const vdouble8& v) {
+ return shuffle4<i, i>(v);
+ }
+
+ template<int i>
+ __forceinline vdouble8 align_shift_right(const vdouble8& a, const vdouble8& b) {
+ return _mm512_castsi512_pd(_mm512_alignr_epi64(_mm512_castpd_si512(a), _mm512_castpd_si512(b), i));
+ }
+
+ __forceinline double toScalar(const vdouble8& v) {
+ return _mm_cvtsd_f64(_mm512_castpd512_pd128(v));
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Reductions
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vdouble8 vreduce_add2(vdouble8 x) { return x + shuffle<1,0,3,2>(x); }
+ __forceinline vdouble8 vreduce_add4(vdouble8 x) { x = vreduce_add2(x); return x + shuffle<2,3,0,1>(x); }
+ __forceinline vdouble8 vreduce_add (vdouble8 x) { x = vreduce_add4(x); return x + shuffle4<1,0>(x); }
+
+ __forceinline vdouble8 vreduce_min2(vdouble8 x) { return min(x, shuffle<1,0,3,2>(x)); }
+ __forceinline vdouble8 vreduce_min4(vdouble8 x) { x = vreduce_min2(x); return min(x, shuffle<2,3,0,1>(x)); }
+ __forceinline vdouble8 vreduce_min (vdouble8 x) { x = vreduce_min4(x); return min(x, shuffle4<1,0>(x)); }
+
+ __forceinline vdouble8 vreduce_max2(vdouble8 x) { return max(x, shuffle<1,0,3,2>(x)); }
+ __forceinline vdouble8 vreduce_max4(vdouble8 x) { x = vreduce_max2(x); return max(x, shuffle<2,3,0,1>(x)); }
+ __forceinline vdouble8 vreduce_max (vdouble8 x) { x = vreduce_max4(x); return max(x, shuffle4<1,0>(x)); }
+
+ __forceinline double reduce_add(const vdouble8& v) { return toScalar(vreduce_add(v)); }
+ __forceinline double reduce_min(const vdouble8& v) { return toScalar(vreduce_min(v)); }
+ __forceinline double reduce_max(const vdouble8& v) { return toScalar(vreduce_max(v)); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Memory load and store operations
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vdouble8 permute(const vdouble8& v, const vllong8& index) {
+ return _mm512_permutexvar_pd(index, v);
+ }
+
+ __forceinline vdouble8 reverse(const vdouble8& a) {
+ return permute(a, vllong8(reverse_step));
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Output Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline embree_ostream operator <<(embree_ostream cout, const vdouble8& v)
+ {
+ cout << "<" << v[0];
+ for (size_t i=1; i<8; i++) cout << ", " << v[i];
+ cout << ">";
+ return cout;
+ }
+}
diff --git a/thirdparty/embree-aarch64/common/simd/vfloat16_avx512.h b/thirdparty/embree-aarch64/common/simd/vfloat16_avx512.h
new file mode 100644
index 0000000000..aed2419b77
--- /dev/null
+++ b/thirdparty/embree-aarch64/common/simd/vfloat16_avx512.h
@@ -0,0 +1,771 @@
+// Copyright 2009-2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+namespace embree
+{
+ /* 16-wide AVX-512 float type */
+ template<>
+ struct vfloat<16>
+ {
+ ALIGNED_STRUCT_(64);
+
+ typedef vboolf16 Bool;
+ typedef vint16 Int;
+ typedef vfloat16 Float;
+
+ enum { size = 16 }; // number of SIMD elements
+ union { // data
+ __m512 v;
+ float f[16];
+ int i[16];
+ };
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constructors, Assignment & Cast Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vfloat() {}
+ __forceinline vfloat(const vfloat16& t) { v = t; }
+ __forceinline vfloat16& operator =(const vfloat16& f) { v = f.v; return *this; }
+
+ __forceinline vfloat(const __m512& t) { v = t; }
+ __forceinline operator __m512() const { return v; }
+ __forceinline operator __m256() const { return _mm512_castps512_ps256(v); }
+ __forceinline operator __m128() const { return _mm512_castps512_ps128(v); }
+
+ __forceinline vfloat(float f) {
+ v = _mm512_set1_ps(f);
+ }
+
+ __forceinline vfloat(float a, float b, float c, float d) {
+ v = _mm512_set4_ps(a, b, c, d);
+ }
+
+ __forceinline vfloat(const vfloat4& i) {
+ v = _mm512_broadcast_f32x4(i);
+ }
+
+ __forceinline vfloat(const vfloat4& a, const vfloat4& b, const vfloat4& c, const vfloat4& d) {
+ v = _mm512_castps128_ps512(a);
+ v = _mm512_insertf32x4(v, b, 1);
+ v = _mm512_insertf32x4(v, c, 2);
+ v = _mm512_insertf32x4(v, d, 3);
+ }
+
+ __forceinline vfloat(const vboolf16& mask, const vfloat4& a, const vfloat4& b) {
+ v = _mm512_broadcast_f32x4(a);
+ v = _mm512_mask_broadcast_f32x4(v,mask,b);
+ }
+
+ __forceinline vfloat(const vfloat8& i) {
+ v = _mm512_castpd_ps(_mm512_broadcast_f64x4(_mm256_castps_pd(i)));
+ }
+
+ __forceinline vfloat(const vfloat8& a, const vfloat8& b) {
+ v = _mm512_castps256_ps512(a);
+#if defined(__AVX512DQ__)
+ v = _mm512_insertf32x8(v, b, 1);
+#else
+ v = _mm512_castpd_ps(_mm512_insertf64x4(_mm512_castps_pd(v), _mm256_castps_pd(b), 1));
+#endif
+ }
+
+ /* WARNING: due to f64x4 the mask is considered as an 8bit mask */
+ __forceinline vfloat(const vboolf16& mask, const vfloat8& a, const vfloat8& b) {
+ __m512d aa = _mm512_broadcast_f64x4(_mm256_castps_pd(a));
+ aa = _mm512_mask_broadcast_f64x4(aa,mask,_mm256_castps_pd(b));
+ v = _mm512_castpd_ps(aa);
+ }
+
+ __forceinline explicit vfloat(const vint16& a) {
+ v = _mm512_cvtepi32_ps(a);
+ }
+
+ __forceinline explicit vfloat(const vuint16& a) {
+ v = _mm512_cvtepu32_ps(a);
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constants
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vfloat(ZeroTy) : v(_mm512_setzero_ps()) {}
+ __forceinline vfloat(OneTy) : v(_mm512_set1_ps(1.0f)) {}
+ __forceinline vfloat(PosInfTy) : v(_mm512_set1_ps(pos_inf)) {}
+ __forceinline vfloat(NegInfTy) : v(_mm512_set1_ps(neg_inf)) {}
+ __forceinline vfloat(StepTy) : v(_mm512_set_ps(15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0)) {}
+ __forceinline vfloat(NaNTy) : v(_mm512_set1_ps(nan)) {}
+ __forceinline vfloat(UndefinedTy) : v(_mm512_undefined_ps()) {}
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Loads and Stores
+ ////////////////////////////////////////////////////////////////////////////////
+
+ static __forceinline vfloat16 load (const void* ptr) { return _mm512_load_ps((float*)ptr); }
+ static __forceinline vfloat16 loadu(const void* ptr) { return _mm512_loadu_ps((float*)ptr); }
+
+ static __forceinline vfloat16 load (const vboolf16& mask, const void* ptr) { return _mm512_mask_load_ps (_mm512_setzero_ps(),mask,(float*)ptr); }
+ static __forceinline vfloat16 loadu(const vboolf16& mask, const void* ptr) { return _mm512_mask_loadu_ps(_mm512_setzero_ps(),mask,(float*)ptr); }
+
+ static __forceinline void store (void* ptr, const vfloat16& v) { _mm512_store_ps ((float*)ptr,v); }
+ static __forceinline void storeu(void* ptr, const vfloat16& v) { _mm512_storeu_ps((float*)ptr,v); }
+
+ static __forceinline void store (const vboolf16& mask, void* ptr, const vfloat16& v) { _mm512_mask_store_ps ((float*)ptr,mask,v); }
+ static __forceinline void storeu(const vboolf16& mask, void* ptr, const vfloat16& v) { _mm512_mask_storeu_ps((float*)ptr,mask,v); }
+
+ static __forceinline void store_nt(void* __restrict__ ptr, const vfloat16& a) {
+ _mm512_stream_ps((float*)ptr,a);
+ }
+
+ static __forceinline vfloat16 broadcast(const float* f) {
+ return _mm512_set1_ps(*f);
+ }
+
+ static __forceinline vfloat16 compact(const vboolf16& mask, vfloat16 &v) {
+ return _mm512_mask_compress_ps(v, mask, v);
+ }
+ static __forceinline vfloat16 compact(const vboolf16& mask, vfloat16 &a, const vfloat16& b) {
+ return _mm512_mask_compress_ps(a, mask, b);
+ }
+
+ static __forceinline vfloat16 expand(const vboolf16& mask, const vfloat16& a, vfloat16& b) {
+ return _mm512_mask_expand_ps(b, mask, a);
+ }
+
+ static __forceinline vfloat16 loadu_compact(const vboolf16& mask, const void* ptr) {
+ return _mm512_mask_expandloadu_ps(_mm512_setzero_ps(), mask, (float*)ptr);
+ }
+
+ static __forceinline void storeu_compact(const vboolf16& mask, float *addr, const vfloat16 reg) {
+ _mm512_mask_compressstoreu_ps(addr, mask, reg);
+ }
+
+ static __forceinline void storeu_compact_single(const vboolf16& mask, float * addr, const vfloat16& reg) {
+ //_mm512_mask_compressstoreu_ps(addr,mask,reg);
+ *addr = mm512_cvtss_f32(_mm512_mask_compress_ps(reg, mask, reg));
+ }
+
+ template<int scale = 4>
+ static __forceinline vfloat16 gather(const float* ptr, const vint16& index) {
+ return _mm512_i32gather_ps(index, ptr, scale);
+ }
+
+ template<int scale = 4>
+ static __forceinline vfloat16 gather(const vboolf16& mask, const float* ptr, const vint16& index) {
+ vfloat16 r = zero;
+ return _mm512_mask_i32gather_ps(r, mask, index, ptr, scale);
+ }
+
+ template<int scale = 4>
+ static __forceinline void scatter(float* ptr, const vint16& index, const vfloat16& v) {
+ _mm512_i32scatter_ps(ptr, index, v, scale);
+ }
+
+ template<int scale = 4>
+ static __forceinline void scatter(const vboolf16& mask, float* ptr, const vint16& index, const vfloat16& v) {
+ _mm512_mask_i32scatter_ps(ptr, mask, index, v, scale);
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Array Access
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline float& operator [](size_t index) { assert(index < 16); return f[index]; }
+ __forceinline const float& operator [](size_t index) const { assert(index < 16); return f[index]; }
+ };
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Unary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vfloat16 asFloat(const vint16& a) { return _mm512_castsi512_ps(a); }
+ __forceinline vint16 asInt (const vfloat16& a) { return _mm512_castps_si512(a); }
+ __forceinline vuint16 asUInt (const vfloat16& a) { return _mm512_castps_si512(a); }
+
+ __forceinline vint16 toInt (const vfloat16& a) { return vint16(a); }
+ __forceinline vfloat16 toFloat(const vint16& a) { return vfloat16(a); }
+
+ __forceinline vfloat16 operator +(const vfloat16& a) { return a; }
+ __forceinline vfloat16 operator -(const vfloat16& a) { return _mm512_mul_ps(a,vfloat16(-1)); }
+
+ __forceinline vfloat16 abs (const vfloat16& a) { return _mm512_castsi512_ps(_mm512_and_epi32(_mm512_castps_si512(a),_mm512_set1_epi32(0x7FFFFFFF))); }
+ __forceinline vfloat16 signmsk(const vfloat16& a) { return _mm512_castsi512_ps(_mm512_and_epi32(_mm512_castps_si512(a),_mm512_set1_epi32(0x80000000))); }
+
+ __forceinline vfloat16 rcp(const vfloat16& a) {
+#if defined(__AVX512ER__)
+ return _mm512_rcp28_ps(a);
+#else
+ const vfloat16 r = _mm512_rcp14_ps(a);
+ return _mm512_mul_ps(r, _mm512_fnmadd_ps(r, a, vfloat16(2.0f)));
+#endif
+ }
+
+ __forceinline vfloat16 sqr (const vfloat16& a) { return _mm512_mul_ps(a,a); }
+ __forceinline vfloat16 sqrt(const vfloat16& a) { return _mm512_sqrt_ps(a); }
+
+ __forceinline vfloat16 rsqrt(const vfloat16& a)
+ {
+#if defined(__AVX512VL__)
+ const vfloat16 r = _mm512_rsqrt14_ps(a);
+ return _mm512_fmadd_ps(_mm512_set1_ps(1.5f), r,
+ _mm512_mul_ps(_mm512_mul_ps(_mm512_mul_ps(a, _mm512_set1_ps(-0.5f)), r), _mm512_mul_ps(r, r)));
+#else
+ return _mm512_rsqrt28_ps(a);
+#endif
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Binary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vfloat16 operator +(const vfloat16& a, const vfloat16& b) { return _mm512_add_ps(a, b); }
+ __forceinline vfloat16 operator +(const vfloat16& a, float b) { return a + vfloat16(b); }
+ __forceinline vfloat16 operator +(float a, const vfloat16& b) { return vfloat16(a) + b; }
+
+ __forceinline vfloat16 operator -(const vfloat16& a, const vfloat16& b) { return _mm512_sub_ps(a, b); }
+ __forceinline vfloat16 operator -(const vfloat16& a, float b) { return a - vfloat16(b); }
+ __forceinline vfloat16 operator -(float a, const vfloat16& b) { return vfloat16(a) - b; }
+
+ __forceinline vfloat16 operator *(const vfloat16& a, const vfloat16& b) { return _mm512_mul_ps(a, b); }
+ __forceinline vfloat16 operator *(const vfloat16& a, float b) { return a * vfloat16(b); }
+ __forceinline vfloat16 operator *(float a, const vfloat16& b) { return vfloat16(a) * b; }
+
+ __forceinline vfloat16 operator /(const vfloat16& a, const vfloat16& b) { return _mm512_div_ps(a,b); }
+ __forceinline vfloat16 operator /(const vfloat16& a, float b) { return a/vfloat16(b); }
+ __forceinline vfloat16 operator /(float a, const vfloat16& b) { return vfloat16(a)/b; }
+
+ __forceinline vfloat16 operator &(const vfloat16& a, const vfloat16& b) { return _mm512_and_ps(a,b); }
+ __forceinline vfloat16 operator |(const vfloat16& a, const vfloat16& b) { return _mm512_or_ps(a,b); }
+ __forceinline vfloat16 operator ^(const vfloat16& a, const vfloat16& b) {
+ return _mm512_castsi512_ps(_mm512_xor_epi32(_mm512_castps_si512(a),_mm512_castps_si512(b)));
+ }
+
+ __forceinline vfloat16 min(const vfloat16& a, const vfloat16& b) {
+ return _mm512_min_ps(a,b);
+ }
+ __forceinline vfloat16 min(const vfloat16& a, float b) {
+ return _mm512_min_ps(a,vfloat16(b));
+ }
+ __forceinline vfloat16 min(const float& a, const vfloat16& b) {
+ return _mm512_min_ps(vfloat16(a),b);
+ }
+
+ __forceinline vfloat16 max(const vfloat16& a, const vfloat16& b) {
+ return _mm512_max_ps(a,b);
+ }
+ __forceinline vfloat16 max(const vfloat16& a, float b) {
+ return _mm512_max_ps(a,vfloat16(b));
+ }
+ __forceinline vfloat16 max(const float& a, const vfloat16& b) {
+ return _mm512_max_ps(vfloat16(a),b);
+ }
+
+ __forceinline vfloat16 mask_add(const vboolf16& mask, const vfloat16& c, const vfloat16& a, const vfloat16& b) { return _mm512_mask_add_ps (c,mask,a,b); }
+ __forceinline vfloat16 mask_min(const vboolf16& mask, const vfloat16& c, const vfloat16& a, const vfloat16& b) {
+ return _mm512_mask_min_ps(c,mask,a,b);
+ };
+ __forceinline vfloat16 mask_max(const vboolf16& mask, const vfloat16& c, const vfloat16& a, const vfloat16& b) {
+ return _mm512_mask_max_ps(c,mask,a,b);
+ };
+
+ __forceinline vfloat16 mini(const vfloat16& a, const vfloat16& b) {
+#if !defined(__AVX512ER__) // SKX
+ const vint16 ai = _mm512_castps_si512(a);
+ const vint16 bi = _mm512_castps_si512(b);
+ const vint16 ci = _mm512_min_epi32(ai,bi);
+ return _mm512_castsi512_ps(ci);
+#else // KNL
+ return min(a,b);
+#endif
+ }
+
+ __forceinline vfloat16 maxi(const vfloat16& a, const vfloat16& b) {
+#if !defined(__AVX512ER__) // SKX
+ const vint16 ai = _mm512_castps_si512(a);
+ const vint16 bi = _mm512_castps_si512(b);
+ const vint16 ci = _mm512_max_epi32(ai,bi);
+ return _mm512_castsi512_ps(ci);
+#else // KNL
+ return max(a,b);
+#endif
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Ternary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vfloat16 madd (const vfloat16& a, const vfloat16& b, const vfloat16& c) { return _mm512_fmadd_ps(a,b,c); }
+ __forceinline vfloat16 msub (const vfloat16& a, const vfloat16& b, const vfloat16& c) { return _mm512_fmsub_ps(a,b,c); }
+ __forceinline vfloat16 nmadd(const vfloat16& a, const vfloat16& b, const vfloat16& c) { return _mm512_fnmadd_ps(a,b,c); }
+ __forceinline vfloat16 nmsub(const vfloat16& a, const vfloat16& b, const vfloat16& c) { return _mm512_fnmsub_ps(a,b,c); }
+
+ __forceinline vfloat16 mask_msub(const vboolf16& mask,const vfloat16& a, const vfloat16& b, const vfloat16& c) { return _mm512_mask_fmsub_ps(a,mask,b,c); }
+
+ __forceinline vfloat16 madd231 (const vfloat16& a, const vfloat16& b, const vfloat16& c) { return _mm512_fmadd_ps(c,b,a); }
+ __forceinline vfloat16 msub213 (const vfloat16& a, const vfloat16& b, const vfloat16& c) { return _mm512_fmsub_ps(a,b,c); }
+ __forceinline vfloat16 msub231 (const vfloat16& a, const vfloat16& b, const vfloat16& c) { return _mm512_fmsub_ps(c,b,a); }
+ __forceinline vfloat16 msubr231(const vfloat16& a, const vfloat16& b, const vfloat16& c) { return _mm512_fnmadd_ps(c,b,a); }
+
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Operators with rounding
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vfloat16 madd_round_down(const vfloat16& a, const vfloat16& b, const vfloat16& c) { return _mm512_fmadd_round_ps(a,b,c,_MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC); }
+ __forceinline vfloat16 madd_round_up (const vfloat16& a, const vfloat16& b, const vfloat16& c) { return _mm512_fmadd_round_ps(a,b,c,_MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC); }
+
+ __forceinline vfloat16 mul_round_down(const vfloat16& a, const vfloat16& b) { return _mm512_mul_round_ps(a,b,_MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC); }
+ __forceinline vfloat16 mul_round_up (const vfloat16& a, const vfloat16& b) { return _mm512_mul_round_ps(a,b,_MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC); }
+
+ __forceinline vfloat16 add_round_down(const vfloat16& a, const vfloat16& b) { return _mm512_add_round_ps(a,b,_MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC); }
+ __forceinline vfloat16 add_round_up (const vfloat16& a, const vfloat16& b) { return _mm512_add_round_ps(a,b,_MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC); }
+
+ __forceinline vfloat16 sub_round_down(const vfloat16& a, const vfloat16& b) { return _mm512_sub_round_ps(a,b,_MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC); }
+ __forceinline vfloat16 sub_round_up (const vfloat16& a, const vfloat16& b) { return _mm512_sub_round_ps(a,b,_MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC); }
+
+ __forceinline vfloat16 div_round_down(const vfloat16& a, const vfloat16& b) { return _mm512_div_round_ps(a,b,_MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC); }
+ __forceinline vfloat16 div_round_up (const vfloat16& a, const vfloat16& b) { return _mm512_div_round_ps(a,b,_MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC); }
+
+ __forceinline vfloat16 mask_msub_round_down(const vboolf16& mask,const vfloat16& a, const vfloat16& b, const vfloat16& c) { return _mm512_mask_fmsub_round_ps(a,mask,b,c,_MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC); }
+ __forceinline vfloat16 mask_msub_round_up (const vboolf16& mask,const vfloat16& a, const vfloat16& b, const vfloat16& c) { return _mm512_mask_fmsub_round_ps(a,mask,b,c,_MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC); }
+
+ __forceinline vfloat16 mask_mul_round_down(const vboolf16& mask,const vfloat16& a, const vfloat16& b, const vfloat16& c) { return _mm512_mask_mul_round_ps(a,mask,b,c,_MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC); }
+ __forceinline vfloat16 mask_mul_round_up (const vboolf16& mask,const vfloat16& a, const vfloat16& b, const vfloat16& c) { return _mm512_mask_mul_round_ps(a,mask,b,c,_MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC); }
+
+ __forceinline vfloat16 mask_sub_round_down(const vboolf16& mask,const vfloat16& a, const vfloat16& b, const vfloat16& c) { return _mm512_mask_sub_round_ps(a,mask,b,c,_MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC); }
+ __forceinline vfloat16 mask_sub_round_up (const vboolf16& mask,const vfloat16& a, const vfloat16& b, const vfloat16& c) { return _mm512_mask_sub_round_ps(a,mask,b,c,_MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC); }
+
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Assignment Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vfloat16& operator +=(vfloat16& a, const vfloat16& b) { return a = a + b; }
+ __forceinline vfloat16& operator +=(vfloat16& a, float b) { return a = a + b; }
+
+ __forceinline vfloat16& operator -=(vfloat16& a, const vfloat16& b) { return a = a - b; }
+ __forceinline vfloat16& operator -=(vfloat16& a, float b) { return a = a - b; }
+
+ __forceinline vfloat16& operator *=(vfloat16& a, const vfloat16& b) { return a = a * b; }
+ __forceinline vfloat16& operator *=(vfloat16& a, float b) { return a = a * b; }
+
+ __forceinline vfloat16& operator /=(vfloat16& a, const vfloat16& b) { return a = a / b; }
+ __forceinline vfloat16& operator /=(vfloat16& a, float b) { return a = a / b; }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Comparison Operators + Select
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboolf16 operator ==(const vfloat16& a, const vfloat16& b) { return _mm512_cmp_ps_mask(a,b,_MM_CMPINT_EQ); }
+ __forceinline vboolf16 operator ==(const vfloat16& a, float b) { return a == vfloat16(b); }
+ __forceinline vboolf16 operator ==(float a, const vfloat16& b) { return vfloat16(a) == b; }
+
+ __forceinline vboolf16 operator !=(const vfloat16& a, const vfloat16& b) { return _mm512_cmp_ps_mask(a,b,_MM_CMPINT_NE); }
+ __forceinline vboolf16 operator !=(const vfloat16& a, float b) { return a != vfloat16(b); }
+ __forceinline vboolf16 operator !=(float a, const vfloat16& b) { return vfloat16(a) != b; }
+
+ __forceinline vboolf16 operator < (const vfloat16& a, const vfloat16& b) { return _mm512_cmp_ps_mask(a,b,_MM_CMPINT_LT); }
+ __forceinline vboolf16 operator < (const vfloat16& a, float b) { return a < vfloat16(b); }
+ __forceinline vboolf16 operator < (float a, const vfloat16& b) { return vfloat16(a) < b; }
+
+ __forceinline vboolf16 operator >=(const vfloat16& a, const vfloat16& b) { return _mm512_cmp_ps_mask(a,b,_MM_CMPINT_GE); }
+ __forceinline vboolf16 operator >=(const vfloat16& a, float b) { return a >= vfloat16(b); }
+ __forceinline vboolf16 operator >=(float a, const vfloat16& b) { return vfloat16(a) >= b; }
+
+ __forceinline vboolf16 operator > (const vfloat16& a, const vfloat16& b) { return _mm512_cmp_ps_mask(a,b,_MM_CMPINT_GT); }
+ __forceinline vboolf16 operator > (const vfloat16& a, float b) { return a > vfloat16(b); }
+ __forceinline vboolf16 operator > (float a, const vfloat16& b) { return vfloat16(a) > b; }
+
+ __forceinline vboolf16 operator <=(const vfloat16& a, const vfloat16& b) { return _mm512_cmp_ps_mask(a,b,_MM_CMPINT_LE); }
+ __forceinline vboolf16 operator <=(const vfloat16& a, float b) { return a <= vfloat16(b); }
+ __forceinline vboolf16 operator <=(float a, const vfloat16& b) { return vfloat16(a) <= b; }
+
+ __forceinline vboolf16 eq(const vfloat16& a, const vfloat16& b) { return _mm512_cmp_ps_mask(a,b,_MM_CMPINT_EQ); }
+ __forceinline vboolf16 ne(const vfloat16& a, const vfloat16& b) { return _mm512_cmp_ps_mask(a,b,_MM_CMPINT_NE); }
+ __forceinline vboolf16 lt(const vfloat16& a, const vfloat16& b) { return _mm512_cmp_ps_mask(a,b,_MM_CMPINT_LT); }
+ __forceinline vboolf16 ge(const vfloat16& a, const vfloat16& b) { return _mm512_cmp_ps_mask(a,b,_MM_CMPINT_GE); }
+ __forceinline vboolf16 gt(const vfloat16& a, const vfloat16& b) { return _mm512_cmp_ps_mask(a,b,_MM_CMPINT_GT); }
+ __forceinline vboolf16 le(const vfloat16& a, const vfloat16& b) { return _mm512_cmp_ps_mask(a,b,_MM_CMPINT_LE); }
+
+ __forceinline vboolf16 eq(const vboolf16& mask, const vfloat16& a, const vfloat16& b) { return _mm512_mask_cmp_ps_mask(mask,a,b,_MM_CMPINT_EQ); }
+ __forceinline vboolf16 ne(const vboolf16& mask, const vfloat16& a, const vfloat16& b) { return _mm512_mask_cmp_ps_mask(mask,a,b,_MM_CMPINT_NE); }
+ __forceinline vboolf16 lt(const vboolf16& mask, const vfloat16& a, const vfloat16& b) { return _mm512_mask_cmp_ps_mask(mask,a,b,_MM_CMPINT_LT); }
+ __forceinline vboolf16 ge(const vboolf16& mask, const vfloat16& a, const vfloat16& b) { return _mm512_mask_cmp_ps_mask(mask,a,b,_MM_CMPINT_GE); }
+ __forceinline vboolf16 gt(const vboolf16& mask, const vfloat16& a, const vfloat16& b) { return _mm512_mask_cmp_ps_mask(mask,a,b,_MM_CMPINT_GT); }
+ __forceinline vboolf16 le(const vboolf16& mask, const vfloat16& a, const vfloat16& b) { return _mm512_mask_cmp_ps_mask(mask,a,b,_MM_CMPINT_LE); }
+
+ __forceinline vfloat16 select(const vboolf16& s, const vfloat16& t, const vfloat16& f) {
+ return _mm512_mask_blend_ps(s, f, t);
+ }
+
+ __forceinline vfloat16 lerp(const vfloat16& a, const vfloat16& b, const vfloat16& t) {
+ return madd(t,b-a,a);
+ }
+
+ __forceinline void xchg(vboolf16 m, vfloat16& a, vfloat16& b)
+ {
+ vfloat16 c = a;
+ a = select(m,b,a);
+ b = select(m,c,b);
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Rounding Functions
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vfloat16 floor(const vfloat16& a) {
+ return _mm512_floor_ps(a);
+ }
+ __forceinline vfloat16 ceil (const vfloat16& a) {
+ return _mm512_ceil_ps(a);
+ }
+ __forceinline vfloat16 round (const vfloat16& a) {
+ return _mm512_roundscale_ps(a, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
+ }
+ __forceinline vint16 floori (const vfloat16& a) {
+ return _mm512_cvt_roundps_epi32(a, _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC);
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Movement/Shifting/Shuffling Functions
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vfloat16 unpacklo(const vfloat16& a, const vfloat16& b) { return _mm512_unpacklo_ps(a, b); }
+ __forceinline vfloat16 unpackhi(const vfloat16& a, const vfloat16& b) { return _mm512_unpackhi_ps(a, b); }
+
+ template<int i>
+ __forceinline vfloat16 shuffle(const vfloat16& v) {
+ return _mm512_permute_ps(v, _MM_SHUFFLE(i, i, i, i));
+ }
+
+ template<int i0, int i1, int i2, int i3>
+ __forceinline vfloat16 shuffle(const vfloat16& v) {
+ return _mm512_permute_ps(v, _MM_SHUFFLE(i3, i2, i1, i0));
+ }
+
+ template<int i>
+ __forceinline vfloat16 shuffle4(const vfloat16& v) {
+ return _mm512_shuffle_f32x4(v, v ,_MM_SHUFFLE(i, i, i, i));
+ }
+
+ template<int i0, int i1, int i2, int i3>
+ __forceinline vfloat16 shuffle4(const vfloat16& v) {
+ return _mm512_shuffle_f32x4(v, v, _MM_SHUFFLE(i3, i2, i1, i0));
+ }
+
+ __forceinline vfloat16 interleave_even(const vfloat16& a, const vfloat16& b) {
+ return _mm512_castsi512_ps(_mm512_mask_shuffle_epi32(_mm512_castps_si512(a), mm512_int2mask(0xaaaa), _mm512_castps_si512(b), (_MM_PERM_ENUM)0xb1));
+ }
+
+ __forceinline vfloat16 interleave_odd(const vfloat16& a, const vfloat16& b) {
+ return _mm512_castsi512_ps(_mm512_mask_shuffle_epi32(_mm512_castps_si512(b), mm512_int2mask(0x5555), _mm512_castps_si512(a), (_MM_PERM_ENUM)0xb1));
+ }
+
+ __forceinline vfloat16 interleave2_even(const vfloat16& a, const vfloat16& b) {
+ /* mask should be 8-bit but is 16-bit to reuse for interleave_even */
+ return _mm512_castsi512_ps(_mm512_mask_permutex_epi64(_mm512_castps_si512(a), mm512_int2mask(0xaaaa), _mm512_castps_si512(b), (_MM_PERM_ENUM)0xb1));
+ }
+
+ __forceinline vfloat16 interleave2_odd(const vfloat16& a, const vfloat16& b) {
+ /* mask should be 8-bit but is 16-bit to reuse for interleave_odd */
+ return _mm512_castsi512_ps(_mm512_mask_permutex_epi64(_mm512_castps_si512(b), mm512_int2mask(0x5555), _mm512_castps_si512(a), (_MM_PERM_ENUM)0xb1));
+ }
+
+ __forceinline vfloat16 interleave4_even(const vfloat16& a, const vfloat16& b) {
+ return _mm512_castsi512_ps(_mm512_mask_permutex_epi64(_mm512_castps_si512(a), mm512_int2mask(0xcc), _mm512_castps_si512(b), (_MM_PERM_ENUM)0x4e));
+ }
+
+ __forceinline vfloat16 interleave4_odd(const vfloat16& a, const vfloat16& b) {
+ return _mm512_castsi512_ps(_mm512_mask_permutex_epi64(_mm512_castps_si512(b), mm512_int2mask(0x33), _mm512_castps_si512(a), (_MM_PERM_ENUM)0x4e));
+ }
+
+ __forceinline vfloat16 permute(vfloat16 v, __m512i index) {
+ return _mm512_castsi512_ps(_mm512_permutexvar_epi32(index, _mm512_castps_si512(v)));
+ }
+
+ __forceinline vfloat16 reverse(const vfloat16& v) {
+ return permute(v,_mm512_setr_epi32(15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0));
+ }
+
+ template<int i>
+ __forceinline vfloat16 align_shift_right(const vfloat16& a, const vfloat16& b) {
+ return _mm512_castsi512_ps(_mm512_alignr_epi32(_mm512_castps_si512(a),_mm512_castps_si512(b),i));
+ };
+
+ template<int i>
+ __forceinline vfloat16 mask_align_shift_right(const vboolf16& mask, vfloat16& c, const vfloat16& a, const vfloat16& b) {
+ return _mm512_castsi512_ps(_mm512_mask_alignr_epi32(_mm512_castps_si512(c),mask,_mm512_castps_si512(a),_mm512_castps_si512(b),i));
+ };
+
+ __forceinline vfloat16 shift_left_1(const vfloat16& a) {
+ vfloat16 z = zero;
+ return mask_align_shift_right<15>(0xfffe,z,a,a);
+ }
+
+ __forceinline vfloat16 shift_right_1(const vfloat16& x) {
+ return align_shift_right<1>(zero,x);
+ }
+
+ __forceinline float toScalar(const vfloat16& v) { return mm512_cvtss_f32(v); }
+
+
+ template<int i> __forceinline vfloat16 insert4(const vfloat16& a, const vfloat4& b) { return _mm512_insertf32x4(a, b, i); }
+
+ template<int N, int i>
+ vfloat<N> extractN(const vfloat16& v);
+
+ template<> __forceinline vfloat4 extractN<4,0>(const vfloat16& v) { return _mm512_castps512_ps128(v); }
+ template<> __forceinline vfloat4 extractN<4,1>(const vfloat16& v) { return _mm512_extractf32x4_ps(v, 1); }
+ template<> __forceinline vfloat4 extractN<4,2>(const vfloat16& v) { return _mm512_extractf32x4_ps(v, 2); }
+ template<> __forceinline vfloat4 extractN<4,3>(const vfloat16& v) { return _mm512_extractf32x4_ps(v, 3); }
+
+ template<> __forceinline vfloat8 extractN<8,0>(const vfloat16& v) { return _mm512_castps512_ps256(v); }
+ template<> __forceinline vfloat8 extractN<8,1>(const vfloat16& v) { return _mm512_extractf32x8_ps(v, 1); }
+
+ template<int i> __forceinline vfloat4 extract4 (const vfloat16& v) { return _mm512_extractf32x4_ps(v, i); }
+ template<> __forceinline vfloat4 extract4<0>(const vfloat16& v) { return _mm512_castps512_ps128(v); }
+
+ template<int i> __forceinline vfloat8 extract8 (const vfloat16& v) { return _mm512_extractf32x8_ps(v, i); }
+ template<> __forceinline vfloat8 extract8<0>(const vfloat16& v) { return _mm512_castps512_ps256(v); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Transpose
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline void transpose(const vfloat16& r0, const vfloat16& r1, const vfloat16& r2, const vfloat16& r3,
+ vfloat16& c0, vfloat16& c1, vfloat16& c2, vfloat16& c3)
+ {
+#if defined(__AVX512F__) && !defined(__AVX512VL__) // KNL
+ vfloat16 a0a1_c0c1 = interleave_even(r0, r1);
+ vfloat16 a2a3_c2c3 = interleave_even(r2, r3);
+ vfloat16 b0b1_d0d1 = interleave_odd (r0, r1);
+ vfloat16 b2b3_d2d3 = interleave_odd (r2, r3);
+
+ c0 = interleave2_even(a0a1_c0c1, a2a3_c2c3);
+ c1 = interleave2_even(b0b1_d0d1, b2b3_d2d3);
+ c2 = interleave2_odd (a0a1_c0c1, a2a3_c2c3);
+ c3 = interleave2_odd (b0b1_d0d1, b2b3_d2d3);
+#else
+ vfloat16 a0a2_b0b2 = unpacklo(r0, r2);
+ vfloat16 c0c2_d0d2 = unpackhi(r0, r2);
+ vfloat16 a1a3_b1b3 = unpacklo(r1, r3);
+ vfloat16 c1c3_d1d3 = unpackhi(r1, r3);
+
+ c0 = unpacklo(a0a2_b0b2, a1a3_b1b3);
+ c1 = unpackhi(a0a2_b0b2, a1a3_b1b3);
+ c2 = unpacklo(c0c2_d0d2, c1c3_d1d3);
+ c3 = unpackhi(c0c2_d0d2, c1c3_d1d3);
+#endif
+ }
+
+ __forceinline void transpose(const vfloat4& r0, const vfloat4& r1, const vfloat4& r2, const vfloat4& r3,
+ const vfloat4& r4, const vfloat4& r5, const vfloat4& r6, const vfloat4& r7,
+ const vfloat4& r8, const vfloat4& r9, const vfloat4& r10, const vfloat4& r11,
+ const vfloat4& r12, const vfloat4& r13, const vfloat4& r14, const vfloat4& r15,
+ vfloat16& c0, vfloat16& c1, vfloat16& c2, vfloat16& c3)
+ {
+ return transpose(vfloat16(r0, r4, r8, r12), vfloat16(r1, r5, r9, r13), vfloat16(r2, r6, r10, r14), vfloat16(r3, r7, r11, r15),
+ c0, c1, c2, c3);
+ }
+
+ __forceinline void transpose(const vfloat16& r0, const vfloat16& r1, const vfloat16& r2, const vfloat16& r3,
+ const vfloat16& r4, const vfloat16& r5, const vfloat16& r6, const vfloat16& r7,
+ vfloat16& c0, vfloat16& c1, vfloat16& c2, vfloat16& c3,
+ vfloat16& c4, vfloat16& c5, vfloat16& c6, vfloat16& c7)
+ {
+ vfloat16 a0a1a2a3_e0e1e2e3, b0b1b2b3_f0f1f2f3, c0c1c2c3_g0g1g2g3, d0d1d2d3_h0h1h2h3;
+ transpose(r0, r1, r2, r3, a0a1a2a3_e0e1e2e3, b0b1b2b3_f0f1f2f3, c0c1c2c3_g0g1g2g3, d0d1d2d3_h0h1h2h3);
+
+ vfloat16 a4a5a6a7_e4e5e6e7, b4b5b6b7_f4f5f6f7, c4c5c6c7_g4g5g6g7, d4d5d6d7_h4h5h6h7;
+ transpose(r4, r5, r6, r7, a4a5a6a7_e4e5e6e7, b4b5b6b7_f4f5f6f7, c4c5c6c7_g4g5g6g7, d4d5d6d7_h4h5h6h7);
+
+ c0 = interleave4_even(a0a1a2a3_e0e1e2e3, a4a5a6a7_e4e5e6e7);
+ c1 = interleave4_even(b0b1b2b3_f0f1f2f3, b4b5b6b7_f4f5f6f7);
+ c2 = interleave4_even(c0c1c2c3_g0g1g2g3, c4c5c6c7_g4g5g6g7);
+ c3 = interleave4_even(d0d1d2d3_h0h1h2h3, d4d5d6d7_h4h5h6h7);
+ c4 = interleave4_odd (a0a1a2a3_e0e1e2e3, a4a5a6a7_e4e5e6e7);
+ c5 = interleave4_odd (b0b1b2b3_f0f1f2f3, b4b5b6b7_f4f5f6f7);
+ c6 = interleave4_odd (c0c1c2c3_g0g1g2g3, c4c5c6c7_g4g5g6g7);
+ c7 = interleave4_odd (d0d1d2d3_h0h1h2h3, d4d5d6d7_h4h5h6h7);
+ }
+
+ __forceinline void transpose(const vfloat8& r0, const vfloat8& r1, const vfloat8& r2, const vfloat8& r3,
+ const vfloat8& r4, const vfloat8& r5, const vfloat8& r6, const vfloat8& r7,
+ const vfloat8& r8, const vfloat8& r9, const vfloat8& r10, const vfloat8& r11,
+ const vfloat8& r12, const vfloat8& r13, const vfloat8& r14, const vfloat8& r15,
+ vfloat16& c0, vfloat16& c1, vfloat16& c2, vfloat16& c3,
+ vfloat16& c4, vfloat16& c5, vfloat16& c6, vfloat16& c7)
+ {
+ return transpose(vfloat16(r0, r8), vfloat16(r1, r9), vfloat16(r2, r10), vfloat16(r3, r11),
+ vfloat16(r4, r12), vfloat16(r5, r13), vfloat16(r6, r14), vfloat16(r7, r15),
+ c0, c1, c2, c3, c4, c5, c6, c7);
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Reductions
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vfloat16 vreduce_add2(vfloat16 x) { return x + shuffle<1,0,3,2>(x); }
+ __forceinline vfloat16 vreduce_add4(vfloat16 x) { x = vreduce_add2(x); return x + shuffle<2,3,0,1>(x); }
+ __forceinline vfloat16 vreduce_add8(vfloat16 x) { x = vreduce_add4(x); return x + shuffle4<1,0,3,2>(x); }
+ __forceinline vfloat16 vreduce_add (vfloat16 x) { x = vreduce_add8(x); return x + shuffle4<2,3,0,1>(x); }
+
+ __forceinline vfloat16 vreduce_min2(vfloat16 x) { return min(x, shuffle<1,0,3,2>(x)); }
+ __forceinline vfloat16 vreduce_min4(vfloat16 x) { x = vreduce_min2(x); return min(x, shuffle<2,3,0,1>(x)); }
+ __forceinline vfloat16 vreduce_min8(vfloat16 x) { x = vreduce_min4(x); return min(x, shuffle4<1,0,3,2>(x)); }
+ __forceinline vfloat16 vreduce_min (vfloat16 x) { x = vreduce_min8(x); return min(x, shuffle4<2,3,0,1>(x)); }
+
+ __forceinline vfloat16 vreduce_max2(vfloat16 x) { return max(x, shuffle<1,0,3,2>(x)); }
+ __forceinline vfloat16 vreduce_max4(vfloat16 x) { x = vreduce_max2(x); return max(x, shuffle<2,3,0,1>(x)); }
+ __forceinline vfloat16 vreduce_max8(vfloat16 x) { x = vreduce_max4(x); return max(x, shuffle4<1,0,3,2>(x)); }
+ __forceinline vfloat16 vreduce_max (vfloat16 x) { x = vreduce_max8(x); return max(x, shuffle4<2,3,0,1>(x)); }
+
+ __forceinline float reduce_add(const vfloat16& v) { return toScalar(vreduce_add(v)); }
+ __forceinline float reduce_min(const vfloat16& v) { return toScalar(vreduce_min(v)); }
+ __forceinline float reduce_max(const vfloat16& v) { return toScalar(vreduce_max(v)); }
+
+ __forceinline size_t select_min(const vfloat16& v) {
+ return bsf(_mm512_kmov(_mm512_cmp_epi32_mask(_mm512_castps_si512(v),_mm512_castps_si512(vreduce_min(v)),_MM_CMPINT_EQ)));
+ }
+
+ __forceinline size_t select_max(const vfloat16& v) {
+ return bsf(_mm512_kmov(_mm512_cmp_epi32_mask(_mm512_castps_si512(v),_mm512_castps_si512(vreduce_max(v)),_MM_CMPINT_EQ)));
+ }
+
+ __forceinline size_t select_min(const vboolf16& valid, const vfloat16& v)
+ {
+ const vfloat16 a = select(valid,v,vfloat16(pos_inf));
+ const vbool16 valid_min = valid & (a == vreduce_min(a));
+ return bsf(movemask(any(valid_min) ? valid_min : valid));
+ }
+
+ __forceinline size_t select_max(const vboolf16& valid, const vfloat16& v)
+ {
+ const vfloat16 a = select(valid,v,vfloat16(neg_inf));
+ const vbool16 valid_max = valid & (a == vreduce_max(a));
+ return bsf(movemask(any(valid_max) ? valid_max : valid));
+ }
+
+ __forceinline vfloat16 prefix_sum(const vfloat16& a)
+ {
+ const vfloat16 z(zero);
+ vfloat16 v = a;
+ v = v + align_shift_right<16-1>(v,z);
+ v = v + align_shift_right<16-2>(v,z);
+ v = v + align_shift_right<16-4>(v,z);
+ v = v + align_shift_right<16-8>(v,z);
+ return v;
+ }
+
+ __forceinline vfloat16 reverse_prefix_sum(const vfloat16& a)
+ {
+ const vfloat16 z(zero);
+ vfloat16 v = a;
+ v = v + align_shift_right<1>(z,v);
+ v = v + align_shift_right<2>(z,v);
+ v = v + align_shift_right<4>(z,v);
+ v = v + align_shift_right<8>(z,v);
+ return v;
+ }
+
+ __forceinline vfloat16 prefix_min(const vfloat16& a)
+ {
+ const vfloat16 z(pos_inf);
+ vfloat16 v = a;
+ v = min(v,align_shift_right<16-1>(v,z));
+ v = min(v,align_shift_right<16-2>(v,z));
+ v = min(v,align_shift_right<16-4>(v,z));
+ v = min(v,align_shift_right<16-8>(v,z));
+ return v;
+ }
+
+ __forceinline vfloat16 prefix_max(const vfloat16& a)
+ {
+ const vfloat16 z(neg_inf);
+ vfloat16 v = a;
+ v = max(v,align_shift_right<16-1>(v,z));
+ v = max(v,align_shift_right<16-2>(v,z));
+ v = max(v,align_shift_right<16-4>(v,z));
+ v = max(v,align_shift_right<16-8>(v,z));
+ return v;
+ }
+
+
+ __forceinline vfloat16 reverse_prefix_min(const vfloat16& a)
+ {
+ const vfloat16 z(pos_inf);
+ vfloat16 v = a;
+ v = min(v,align_shift_right<1>(z,v));
+ v = min(v,align_shift_right<2>(z,v));
+ v = min(v,align_shift_right<4>(z,v));
+ v = min(v,align_shift_right<8>(z,v));
+ return v;
+ }
+
+ __forceinline vfloat16 reverse_prefix_max(const vfloat16& a)
+ {
+ const vfloat16 z(neg_inf);
+ vfloat16 v = a;
+ v = max(v,align_shift_right<1>(z,v));
+ v = max(v,align_shift_right<2>(z,v));
+ v = max(v,align_shift_right<4>(z,v));
+ v = max(v,align_shift_right<8>(z,v));
+ return v;
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Memory load and store operations
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vfloat16 loadAOS4to16f(const float& x, const float& y, const float& z)
+ {
+ vfloat16 f = zero;
+ f = select(0x1111,vfloat16::broadcast(&x),f);
+ f = select(0x2222,vfloat16::broadcast(&y),f);
+ f = select(0x4444,vfloat16::broadcast(&z),f);
+ return f;
+ }
+
+ __forceinline vfloat16 loadAOS4to16f(unsigned int index,
+ const vfloat16& x,
+ const vfloat16& y,
+ const vfloat16& z)
+ {
+ vfloat16 f = zero;
+ f = select(0x1111,vfloat16::broadcast((float*)&x + index),f);
+ f = select(0x2222,vfloat16::broadcast((float*)&y + index),f);
+ f = select(0x4444,vfloat16::broadcast((float*)&z + index),f);
+ return f;
+ }
+
+ __forceinline vfloat16 loadAOS4to16f(unsigned int index,
+ const vfloat16& x,
+ const vfloat16& y,
+ const vfloat16& z,
+ const vfloat16& fill)
+ {
+ vfloat16 f = fill;
+ f = select(0x1111,vfloat16::broadcast((float*)&x + index),f);
+ f = select(0x2222,vfloat16::broadcast((float*)&y + index),f);
+ f = select(0x4444,vfloat16::broadcast((float*)&z + index),f);
+ return f;
+ }
+
+ __forceinline vfloat16 rcp_safe(const vfloat16& a) {
+ return rcp(select(a != vfloat16(zero), a, vfloat16(min_rcp_input)));
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Output Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline embree_ostream operator <<(embree_ostream cout, const vfloat16& v)
+ {
+ cout << "<" << v[0];
+ for (int i=1; i<16; i++) cout << ", " << v[i];
+ cout << ">";
+ return cout;
+ }
+}
diff --git a/thirdparty/embree-aarch64/common/simd/vfloat4_sse2.h b/thirdparty/embree-aarch64/common/simd/vfloat4_sse2.h
new file mode 100644
index 0000000000..5732c0fbc8
--- /dev/null
+++ b/thirdparty/embree-aarch64/common/simd/vfloat4_sse2.h
@@ -0,0 +1,925 @@
+// Copyright 2009-2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+namespace embree
+{
+ /* 4-wide SSE float type */
+ template<>
+ struct vfloat<4>
+ {
+ ALIGNED_STRUCT_(16);
+
+ typedef vboolf4 Bool;
+ typedef vint4 Int;
+ typedef vfloat4 Float;
+
+ enum { size = 4 }; // number of SIMD elements
+ union { __m128 v; float f[4]; int i[4]; }; // data
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constructors, Assignment & Cast Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vfloat() {}
+ __forceinline vfloat(const vfloat4& other) { v = other.v; }
+ __forceinline vfloat4& operator =(const vfloat4& other) { v = other.v; return *this; }
+
+ __forceinline vfloat(__m128 a) : v(a) {}
+ __forceinline operator const __m128&() const { return v; }
+ __forceinline operator __m128&() { return v; }
+
+ __forceinline vfloat(float a) : v(_mm_set1_ps(a)) {}
+ __forceinline vfloat(float a, float b, float c, float d) : v(_mm_set_ps(d, c, b, a)) {}
+
+ __forceinline explicit vfloat(const vint4& a) : v(_mm_cvtepi32_ps(a)) {}
+#if defined(__aarch64__)
+ __forceinline explicit vfloat(const vuint4& x) {
+ v = vcvtq_f32_u32(vreinterpretq_u32_s32(x.v));
+ }
+#else
+ __forceinline explicit vfloat(const vuint4& x) {
+ const __m128i a = _mm_and_si128(x,_mm_set1_epi32(0x7FFFFFFF));
+ const __m128i b = _mm_and_si128(_mm_srai_epi32(x,31),_mm_set1_epi32(0x4F000000)); //0x4F000000 = 2^31
+ const __m128 af = _mm_cvtepi32_ps(a);
+ const __m128 bf = _mm_castsi128_ps(b);
+ v = _mm_add_ps(af,bf);
+ }
+#endif
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constants
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vfloat(ZeroTy) : v(_mm_setzero_ps()) {}
+ __forceinline vfloat(OneTy) : v(_mm_set1_ps(1.0f)) {}
+ __forceinline vfloat(PosInfTy) : v(_mm_set1_ps(pos_inf)) {}
+ __forceinline vfloat(NegInfTy) : v(_mm_set1_ps(neg_inf)) {}
+ __forceinline vfloat(StepTy) : v(_mm_set_ps(3.0f, 2.0f, 1.0f, 0.0f)) {}
+ __forceinline vfloat(NaNTy) : v(_mm_set1_ps(nan)) {}
+ __forceinline vfloat(UndefinedTy) : v(_mm_undefined_ps()) {}
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Loads and Stores
+ ////////////////////////////////////////////////////////////////////////////////
+
+ static __forceinline vfloat4 load (const void* a) { return _mm_load_ps((float*)a); }
+ static __forceinline vfloat4 loadu(const void* a) { return _mm_loadu_ps((float*)a); }
+
+ static __forceinline void store (void* ptr, const vfloat4& v) { _mm_store_ps((float*)ptr,v); }
+ static __forceinline void storeu(void* ptr, const vfloat4& v) { _mm_storeu_ps((float*)ptr,v); }
+
+#if defined(__AVX512VL__)
+
+ static __forceinline vfloat4 compact(const vboolf4& mask, vfloat4 &v) {
+ return _mm_mask_compress_ps(v, mask, v);
+ }
+ static __forceinline vfloat4 compact(const vboolf4& mask, vfloat4 &a, const vfloat4& b) {
+ return _mm_mask_compress_ps(a, mask, b);
+ }
+
+ static __forceinline vfloat4 load (const vboolf4& mask, const void* ptr) { return _mm_mask_load_ps (_mm_setzero_ps(),mask,(float*)ptr); }
+ static __forceinline vfloat4 loadu(const vboolf4& mask, const void* ptr) { return _mm_mask_loadu_ps(_mm_setzero_ps(),mask,(float*)ptr); }
+
+ static __forceinline void store (const vboolf4& mask, void* ptr, const vfloat4& v) { _mm_mask_store_ps ((float*)ptr,mask,v); }
+ static __forceinline void storeu(const vboolf4& mask, void* ptr, const vfloat4& v) { _mm_mask_storeu_ps((float*)ptr,mask,v); }
+#elif defined(__AVX__)
+ static __forceinline vfloat4 load (const vboolf4& mask, const void* ptr) { return _mm_maskload_ps((float*)ptr,mask); }
+ static __forceinline vfloat4 loadu(const vboolf4& mask, const void* ptr) { return _mm_maskload_ps((float*)ptr,mask); }
+
+ static __forceinline void store (const vboolf4& mask, void* ptr, const vfloat4& v) { _mm_maskstore_ps((float*)ptr,(__m128i)mask,v); }
+ static __forceinline void storeu(const vboolf4& mask, void* ptr, const vfloat4& v) { _mm_maskstore_ps((float*)ptr,(__m128i)mask,v); }
+#else
+ static __forceinline vfloat4 load (const vboolf4& mask, const void* ptr) { return _mm_and_ps(_mm_load_ps ((float*)ptr),mask); }
+ static __forceinline vfloat4 loadu(const vboolf4& mask, const void* ptr) { return _mm_and_ps(_mm_loadu_ps((float*)ptr),mask); }
+
+ static __forceinline void store (const vboolf4& mask, void* ptr, const vfloat4& v) { store (ptr,select(mask,v,load (ptr))); }
+ static __forceinline void storeu(const vboolf4& mask, void* ptr, const vfloat4& v) { storeu(ptr,select(mask,v,loadu(ptr))); }
+#endif
+
+#if defined(__AVX__)
+ static __forceinline vfloat4 broadcast(const void* a) { return _mm_broadcast_ss((float*)a); }
+#else
+ static __forceinline vfloat4 broadcast(const void* a) { return _mm_set1_ps(*(float*)a); }
+#endif
+
+ static __forceinline vfloat4 load_nt (const float* ptr) {
+#if defined (__SSE4_1__)
+ return _mm_castsi128_ps(_mm_stream_load_si128((__m128i*)ptr));
+#else
+ return _mm_load_ps(ptr);
+#endif
+ }
+
+#if defined(__aarch64__)
+ static __forceinline vfloat4 load(const int8_t* ptr) {
+ return __m128(_mm_load4epi8_f32(((__m128i*)ptr)));
+ }
+#elif defined(__SSE4_1__)
+ static __forceinline vfloat4 load(const int8_t* ptr) {
+ return _mm_cvtepi32_ps(_mm_cvtepi8_epi32(_mm_loadu_si128((__m128i*)ptr)));
+ }
+#else
+ static __forceinline vfloat4 load(const int8_t* ptr) {
+ return vfloat4(ptr[0],ptr[1],ptr[2],ptr[3]);
+ }
+#endif
+
+#if defined(__aarch64__)
+ static __forceinline vfloat4 load(const uint8_t* ptr) {
+ return __m128(_mm_load4epu8_f32(((__m128i*)ptr)));
+ }
+#elif defined(__SSE4_1__)
+ static __forceinline vfloat4 load(const uint8_t* ptr) {
+ return _mm_cvtepi32_ps(_mm_cvtepu8_epi32(_mm_loadu_si128((__m128i*)ptr)));
+ }
+#else
+ static __forceinline vfloat4 load(const uint8_t* ptr) {
+ //return _mm_cvtpu8_ps(*(__m64*)ptr); // don't enable, will use MMX instructions
+ return vfloat4(ptr[0],ptr[1],ptr[2],ptr[3]);
+ }
+#endif
+
+#if defined(__aarch64__)
+ static __forceinline vfloat4 load(const short* ptr) {
+ return __m128(_mm_load4epi16_f32(((__m128i*)ptr)));
+ }
+#elif defined(__SSE4_1__)
+ static __forceinline vfloat4 load(const short* ptr) {
+ return _mm_cvtepi32_ps(_mm_cvtepi16_epi32(_mm_loadu_si128((__m128i*)ptr)));
+ }
+#else
+ static __forceinline vfloat4 load(const short* ptr) {
+ return vfloat4(ptr[0],ptr[1],ptr[2],ptr[3]);
+ }
+#endif
+
+ static __forceinline vfloat4 load(const unsigned short* ptr) {
+ return _mm_mul_ps(vfloat4(vint4::load(ptr)),vfloat4(1.0f/65535.0f));
+ }
+
+ static __forceinline void store_nt(void* ptr, const vfloat4& v)
+ {
+#if defined (__SSE4_1__)
+#if defined(__aarch64__)
+ _mm_stream_ps((float*)ptr,vreinterpretq_s32_f32(v.v));
+#else
+ _mm_stream_ps((float*)ptr,v);
+#endif
+#else
+ _mm_store_ps((float*)ptr,v);
+#endif
+ }
+
+ template<int scale = 4>
+ static __forceinline vfloat4 gather(const float* ptr, const vint4& index) {
+#if defined(__AVX2__) && !defined(__aarch64__)
+ return _mm_i32gather_ps(ptr, index, scale);
+#else
+ return vfloat4(
+ *(float*)(((int8_t*)ptr)+scale*index[0]),
+ *(float*)(((int8_t*)ptr)+scale*index[1]),
+ *(float*)(((int8_t*)ptr)+scale*index[2]),
+ *(float*)(((int8_t*)ptr)+scale*index[3]));
+#endif
+ }
+
+ template<int scale = 4>
+ static __forceinline vfloat4 gather(const vboolf4& mask, const float* ptr, const vint4& index) {
+ vfloat4 r = zero;
+#if defined(__AVX512VL__)
+ return _mm_mmask_i32gather_ps(r, mask, index, ptr, scale);
+#elif defined(__AVX2__) && !defined(__aarch64__)
+ return _mm_mask_i32gather_ps(r, ptr, index, mask, scale);
+#else
+ if (likely(mask[0])) r[0] = *(float*)(((int8_t*)ptr)+scale*index[0]);
+ if (likely(mask[1])) r[1] = *(float*)(((int8_t*)ptr)+scale*index[1]);
+ if (likely(mask[2])) r[2] = *(float*)(((int8_t*)ptr)+scale*index[2]);
+ if (likely(mask[3])) r[3] = *(float*)(((int8_t*)ptr)+scale*index[3]);
+ return r;
+#endif
+ }
+
+ template<int scale = 4>
+ static __forceinline void scatter(void* ptr, const vint4& index, const vfloat4& v)
+ {
+#if defined(__AVX512VL__)
+ _mm_i32scatter_ps((float*)ptr, index, v, scale);
+#else
+ *(float*)(((int8_t*)ptr)+scale*index[0]) = v[0];
+ *(float*)(((int8_t*)ptr)+scale*index[1]) = v[1];
+ *(float*)(((int8_t*)ptr)+scale*index[2]) = v[2];
+ *(float*)(((int8_t*)ptr)+scale*index[3]) = v[3];
+#endif
+ }
+
+ template<int scale = 4>
+ static __forceinline void scatter(const vboolf4& mask, void* ptr, const vint4& index, const vfloat4& v)
+ {
+#if defined(__AVX512VL__)
+ _mm_mask_i32scatter_ps((float*)ptr ,mask, index, v, scale);
+#else
+ if (likely(mask[0])) *(float*)(((int8_t*)ptr)+scale*index[0]) = v[0];
+ if (likely(mask[1])) *(float*)(((int8_t*)ptr)+scale*index[1]) = v[1];
+ if (likely(mask[2])) *(float*)(((int8_t*)ptr)+scale*index[2]) = v[2];
+ if (likely(mask[3])) *(float*)(((int8_t*)ptr)+scale*index[3]) = v[3];
+#endif
+ }
+
+ static __forceinline void store(const vboolf4& mask, int8_t* ptr, const vint4& ofs, const vfloat4& v) {
+ scatter<1>(mask,ptr,ofs,v);
+ }
+ static __forceinline void store(const vboolf4& mask, float* ptr, const vint4& ofs, const vfloat4& v) {
+ scatter<4>(mask,ptr,ofs,v);
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Array Access
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline const float& operator [](size_t index) const { assert(index < 4); return f[index]; }
+ __forceinline float& operator [](size_t index) { assert(index < 4); return f[index]; }
+
+ friend __forceinline vfloat4 select(const vboolf4& m, const vfloat4& t, const vfloat4& f) {
+#if defined(__AVX512VL__)
+ return _mm_mask_blend_ps(m, f, t);
+#elif defined(__SSE4_1__) || (defined(__aarch64__))
+ return _mm_blendv_ps(f, t, m);
+#else
+ return _mm_or_ps(_mm_and_ps(m, t), _mm_andnot_ps(m, f));
+#endif
+ }
+ };
+
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Unary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vfloat4 asFloat(const vint4& a) { return _mm_castsi128_ps(a); }
+ __forceinline vint4 asInt (const vfloat4& a) { return _mm_castps_si128(a); }
+ __forceinline vuint4 asUInt (const vfloat4& a) { return _mm_castps_si128(a); }
+
+ __forceinline vint4 toInt (const vfloat4& a) { return vint4(a); }
+ __forceinline vfloat4 toFloat(const vint4& a) { return vfloat4(a); }
+
+ __forceinline vfloat4 operator +(const vfloat4& a) { return a; }
+#if defined(__aarch64__)
+ __forceinline vfloat4 operator -(const vfloat4& a) {
+ return vnegq_f32(a);
+ }
+#else
+ __forceinline vfloat4 operator -(const vfloat4& a) { return _mm_xor_ps(a, _mm_castsi128_ps(_mm_set1_epi32(0x80000000))); }
+#endif
+
+#if defined(__aarch64__)
+ __forceinline vfloat4 abs(const vfloat4& a) { return _mm_abs_ps(a); }
+#else
+ __forceinline vfloat4 abs(const vfloat4& a) { return _mm_and_ps(a, _mm_castsi128_ps(_mm_set1_epi32(0x7fffffff))); }
+#endif
+
+#if defined(__AVX512VL__)
+ __forceinline vfloat4 sign(const vfloat4& a) { return _mm_mask_blend_ps(_mm_cmp_ps_mask(a, vfloat4(zero), _CMP_LT_OQ), vfloat4(one), -vfloat4(one)); }
+#else
+ __forceinline vfloat4 sign(const vfloat4& a) { return blendv_ps(vfloat4(one), -vfloat4(one), _mm_cmplt_ps(a, vfloat4(zero))); }
+#endif
+
+#if defined(__aarch64__)
+ __forceinline vfloat4 signmsk(const vfloat4& a) { return _mm_and_ps(a, vreinterpretq_f32_u32(v0x80000000)); }
+#else
+ __forceinline vfloat4 signmsk(const vfloat4& a) { return _mm_and_ps(a,_mm_castsi128_ps(_mm_set1_epi32(0x80000000))); }
+#endif
+
+ __forceinline vfloat4 rcp(const vfloat4& a)
+ {
+#if defined(__aarch64__)
+#if defined(BUILD_IOS)
+ return vfloat4(vdivq_f32(vdupq_n_f32(1.0f),a.v));
+#else //BUILD_IOS
+ __m128 reciprocal = _mm_rcp_ps(a);
+ reciprocal = vmulq_f32(vrecpsq_f32(a, reciprocal), reciprocal);
+ reciprocal = vmulq_f32(vrecpsq_f32(a, reciprocal), reciprocal);
+ // +1 round since NEON's reciprocal estimate instruction has less accuracy than SSE2's rcp.
+ reciprocal = vmulq_f32(vrecpsq_f32(a, reciprocal), reciprocal);
+ return (const vfloat4)reciprocal;
+#endif // BUILD_IOS
+#else
+
+#if defined(__AVX512VL__)
+ const vfloat4 r = _mm_rcp14_ps(a);
+#else
+ const vfloat4 r = _mm_rcp_ps(a);
+#endif
+
+#if defined(__AVX2__)
+ return _mm_mul_ps(r,_mm_fnmadd_ps(r, a, vfloat4(2.0f)));
+#else
+ return _mm_mul_ps(r,_mm_sub_ps(vfloat4(2.0f), _mm_mul_ps(r, a)));
+#endif
+
+#endif //defined(__aarch64__)
+ }
+ __forceinline vfloat4 sqr (const vfloat4& a) { return _mm_mul_ps(a,a); }
+ __forceinline vfloat4 sqrt(const vfloat4& a) { return _mm_sqrt_ps(a); }
+
+ __forceinline vfloat4 rsqrt(const vfloat4& a)
+ {
+#if defined(__aarch64__)
+ vfloat4 r = _mm_rsqrt_ps(a);
+ r = vmulq_f32(r, vrsqrtsq_f32(vmulq_f32(a, r), r));
+ r = vmulq_f32(r, vrsqrtsq_f32(vmulq_f32(a, r), r));
+ r = vmulq_f32(r, vrsqrtsq_f32(vmulq_f32(a, r), r));
+ return r;
+#else
+
+#if defined(__AVX512VL__)
+ const vfloat4 r = _mm_rsqrt14_ps(a);
+#else
+ const vfloat4 r = _mm_rsqrt_ps(a);
+#endif
+
+#if defined(__AVX2__)
+ return _mm_fmadd_ps(_mm_set1_ps(1.5f), r,
+ _mm_mul_ps(_mm_mul_ps(_mm_mul_ps(a, _mm_set1_ps(-0.5f)), r), _mm_mul_ps(r, r)));
+#else
+ return _mm_add_ps(_mm_mul_ps(_mm_set1_ps(1.5f), r),
+ _mm_mul_ps(_mm_mul_ps(_mm_mul_ps(a, _mm_set1_ps(-0.5f)), r), _mm_mul_ps(r, r)));
+#endif
+
+#endif
+ }
+
+ __forceinline vboolf4 isnan(const vfloat4& a) {
+#if defined(__aarch64__)
+ const vfloat4 b = _mm_and_ps(a, vreinterpretq_f32_u32(v0x7fffffff));
+#else
+ const vfloat4 b = _mm_and_ps(a, _mm_castsi128_ps(_mm_set1_epi32(0x7fffffff)));
+#endif
+#if defined(__AVX512VL__)
+ return _mm_cmp_epi32_mask(_mm_castps_si128(b), _mm_set1_epi32(0x7f800000), _MM_CMPINT_GT);
+#else
+ return _mm_castsi128_ps(_mm_cmpgt_epi32(_mm_castps_si128(b), _mm_set1_epi32(0x7f800000)));
+#endif
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Binary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vfloat4 operator +(const vfloat4& a, const vfloat4& b) { return _mm_add_ps(a, b); }
+ __forceinline vfloat4 operator +(const vfloat4& a, float b) { return a + vfloat4(b); }
+ __forceinline vfloat4 operator +(float a, const vfloat4& b) { return vfloat4(a) + b; }
+
+ __forceinline vfloat4 operator -(const vfloat4& a, const vfloat4& b) { return _mm_sub_ps(a, b); }
+ __forceinline vfloat4 operator -(const vfloat4& a, float b) { return a - vfloat4(b); }
+ __forceinline vfloat4 operator -(float a, const vfloat4& b) { return vfloat4(a) - b; }
+
+ __forceinline vfloat4 operator *(const vfloat4& a, const vfloat4& b) { return _mm_mul_ps(a, b); }
+ __forceinline vfloat4 operator *(const vfloat4& a, float b) { return a * vfloat4(b); }
+ __forceinline vfloat4 operator *(float a, const vfloat4& b) { return vfloat4(a) * b; }
+
+ __forceinline vfloat4 operator /(const vfloat4& a, const vfloat4& b) { return _mm_div_ps(a,b); }
+ __forceinline vfloat4 operator /(const vfloat4& a, float b) { return a/vfloat4(b); }
+ __forceinline vfloat4 operator /(float a, const vfloat4& b) { return vfloat4(a)/b; }
+
+ __forceinline vfloat4 operator &(const vfloat4& a, const vfloat4& b) { return _mm_and_ps(a,b); }
+ __forceinline vfloat4 operator |(const vfloat4& a, const vfloat4& b) { return _mm_or_ps(a,b); }
+ __forceinline vfloat4 operator ^(const vfloat4& a, const vfloat4& b) { return _mm_xor_ps(a,b); }
+ __forceinline vfloat4 operator ^(const vfloat4& a, const vint4& b) { return _mm_xor_ps(a,_mm_castsi128_ps(b)); }
+
+ __forceinline vfloat4 min(const vfloat4& a, const vfloat4& b) { return _mm_min_ps(a,b); }
+ __forceinline vfloat4 min(const vfloat4& a, float b) { return _mm_min_ps(a,vfloat4(b)); }
+ __forceinline vfloat4 min(float a, const vfloat4& b) { return _mm_min_ps(vfloat4(a),b); }
+
+ __forceinline vfloat4 max(const vfloat4& a, const vfloat4& b) { return _mm_max_ps(a,b); }
+ __forceinline vfloat4 max(const vfloat4& a, float b) { return _mm_max_ps(a,vfloat4(b)); }
+ __forceinline vfloat4 max(float a, const vfloat4& b) { return _mm_max_ps(vfloat4(a),b); }
+
+#if defined(__SSE4_1__) || defined(__aarch64__)
+
+ __forceinline vfloat4 mini(const vfloat4& a, const vfloat4& b) {
+ const vint4 ai = _mm_castps_si128(a);
+ const vint4 bi = _mm_castps_si128(b);
+ const vint4 ci = _mm_min_epi32(ai,bi);
+ return _mm_castsi128_ps(ci);
+ }
+
+ __forceinline vfloat4 maxi(const vfloat4& a, const vfloat4& b) {
+ const vint4 ai = _mm_castps_si128(a);
+ const vint4 bi = _mm_castps_si128(b);
+ const vint4 ci = _mm_max_epi32(ai,bi);
+ return _mm_castsi128_ps(ci);
+ }
+
+ __forceinline vfloat4 minui(const vfloat4& a, const vfloat4& b) {
+ const vint4 ai = _mm_castps_si128(a);
+ const vint4 bi = _mm_castps_si128(b);
+ const vint4 ci = _mm_min_epu32(ai,bi);
+ return _mm_castsi128_ps(ci);
+ }
+
+ __forceinline vfloat4 maxui(const vfloat4& a, const vfloat4& b) {
+ const vint4 ai = _mm_castps_si128(a);
+ const vint4 bi = _mm_castps_si128(b);
+ const vint4 ci = _mm_max_epu32(ai,bi);
+ return _mm_castsi128_ps(ci);
+ }
+#else
+ __forceinline vfloat4 mini(const vfloat4& a, const vfloat4& b) {
+ return min(a,b);
+ }
+
+ __forceinline vfloat4 maxi(const vfloat4& a, const vfloat4& b) {
+ return max(a,b);
+ }
+#endif
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Ternary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+#if defined(__AVX2__)
+ __forceinline vfloat4 madd (const vfloat4& a, const vfloat4& b, const vfloat4& c) { return _mm_fmadd_ps(a,b,c); }
+ __forceinline vfloat4 msub (const vfloat4& a, const vfloat4& b, const vfloat4& c) { return _mm_fmsub_ps(a,b,c); }
+ __forceinline vfloat4 nmadd(const vfloat4& a, const vfloat4& b, const vfloat4& c) { return _mm_fnmadd_ps(a,b,c); }
+ __forceinline vfloat4 nmsub(const vfloat4& a, const vfloat4& b, const vfloat4& c) { return _mm_fnmsub_ps(a,b,c); }
+#else
+
+#if defined(__aarch64__)
+ __forceinline vfloat4 madd (const vfloat4& a, const vfloat4& b, const vfloat4& c) {
+ return _mm_madd_ps(a, b, c); //a*b+c;
+ }
+ __forceinline vfloat4 nmadd(const vfloat4& a, const vfloat4& b, const vfloat4& c) {
+ return _mm_msub_ps(a, b, c); //-a*b+c;
+ }
+ __forceinline vfloat4 nmsub(const vfloat4& a, const vfloat4& b, const vfloat4& c) {
+ return vnegq_f32(vfmaq_f32(c,a, b));
+ }
+#else
+ __forceinline vfloat4 madd (const vfloat4& a, const vfloat4& b, const vfloat4& c) { return a*b+c; }
+ __forceinline vfloat4 nmadd(const vfloat4& a, const vfloat4& b, const vfloat4& c) { return -a*b+c;}
+ __forceinline vfloat4 nmsub(const vfloat4& a, const vfloat4& b, const vfloat4& c) { return -a*b-c; }
+#endif
+ __forceinline vfloat4 msub (const vfloat4& a, const vfloat4& b, const vfloat4& c) { return a*b-c; }
+
+#endif
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Assignment Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vfloat4& operator +=(vfloat4& a, const vfloat4& b) { return a = a + b; }
+ __forceinline vfloat4& operator +=(vfloat4& a, float b) { return a = a + b; }
+
+ __forceinline vfloat4& operator -=(vfloat4& a, const vfloat4& b) { return a = a - b; }
+ __forceinline vfloat4& operator -=(vfloat4& a, float b) { return a = a - b; }
+
+ __forceinline vfloat4& operator *=(vfloat4& a, const vfloat4& b) { return a = a * b; }
+ __forceinline vfloat4& operator *=(vfloat4& a, float b) { return a = a * b; }
+
+ __forceinline vfloat4& operator /=(vfloat4& a, const vfloat4& b) { return a = a / b; }
+ __forceinline vfloat4& operator /=(vfloat4& a, float b) { return a = a / b; }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Comparison Operators + Select
+ ////////////////////////////////////////////////////////////////////////////////
+
+#if defined(__AVX512VL__)
+ __forceinline vboolf4 operator ==(const vfloat4& a, const vfloat4& b) { return _mm_cmp_ps_mask(a, b, _MM_CMPINT_EQ); }
+ __forceinline vboolf4 operator !=(const vfloat4& a, const vfloat4& b) { return _mm_cmp_ps_mask(a, b, _MM_CMPINT_NE); }
+ __forceinline vboolf4 operator < (const vfloat4& a, const vfloat4& b) { return _mm_cmp_ps_mask(a, b, _MM_CMPINT_LT); }
+ __forceinline vboolf4 operator >=(const vfloat4& a, const vfloat4& b) { return _mm_cmp_ps_mask(a, b, _MM_CMPINT_GE); }
+ __forceinline vboolf4 operator > (const vfloat4& a, const vfloat4& b) { return _mm_cmp_ps_mask(a, b, _MM_CMPINT_GT); }
+ __forceinline vboolf4 operator <=(const vfloat4& a, const vfloat4& b) { return _mm_cmp_ps_mask(a, b, _MM_CMPINT_LE); }
+#else
+ __forceinline vboolf4 operator ==(const vfloat4& a, const vfloat4& b) { return _mm_cmpeq_ps (a, b); }
+ __forceinline vboolf4 operator !=(const vfloat4& a, const vfloat4& b) { return _mm_cmpneq_ps(a, b); }
+ __forceinline vboolf4 operator < (const vfloat4& a, const vfloat4& b) { return _mm_cmplt_ps (a, b); }
+#if defined(__aarch64__)
+ __forceinline vboolf4 operator >=(const vfloat4& a, const vfloat4& b) { return _mm_cmpge_ps (a, b); }
+ __forceinline vboolf4 operator > (const vfloat4& a, const vfloat4& b) { return _mm_cmpgt_ps (a, b); }
+#else
+ __forceinline vboolf4 operator >=(const vfloat4& a, const vfloat4& b) { return _mm_cmpnlt_ps(a, b); }
+ __forceinline vboolf4 operator > (const vfloat4& a, const vfloat4& b) { return _mm_cmpnle_ps(a, b); }
+#endif
+ __forceinline vboolf4 operator <=(const vfloat4& a, const vfloat4& b) { return _mm_cmple_ps (a, b); }
+#endif
+
+ __forceinline vboolf4 operator ==(const vfloat4& a, float b) { return a == vfloat4(b); }
+ __forceinline vboolf4 operator ==(float a, const vfloat4& b) { return vfloat4(a) == b; }
+
+ __forceinline vboolf4 operator !=(const vfloat4& a, float b) { return a != vfloat4(b); }
+ __forceinline vboolf4 operator !=(float a, const vfloat4& b) { return vfloat4(a) != b; }
+
+ __forceinline vboolf4 operator < (const vfloat4& a, float b) { return a < vfloat4(b); }
+ __forceinline vboolf4 operator < (float a, const vfloat4& b) { return vfloat4(a) < b; }
+
+ __forceinline vboolf4 operator >=(const vfloat4& a, float b) { return a >= vfloat4(b); }
+ __forceinline vboolf4 operator >=(float a, const vfloat4& b) { return vfloat4(a) >= b; }
+
+ __forceinline vboolf4 operator > (const vfloat4& a, float b) { return a > vfloat4(b); }
+ __forceinline vboolf4 operator > (float a, const vfloat4& b) { return vfloat4(a) > b; }
+
+ __forceinline vboolf4 operator <=(const vfloat4& a, float b) { return a <= vfloat4(b); }
+ __forceinline vboolf4 operator <=(float a, const vfloat4& b) { return vfloat4(a) <= b; }
+
+ __forceinline vboolf4 eq(const vfloat4& a, const vfloat4& b) { return a == b; }
+ __forceinline vboolf4 ne(const vfloat4& a, const vfloat4& b) { return a != b; }
+ __forceinline vboolf4 lt(const vfloat4& a, const vfloat4& b) { return a < b; }
+ __forceinline vboolf4 ge(const vfloat4& a, const vfloat4& b) { return a >= b; }
+ __forceinline vboolf4 gt(const vfloat4& a, const vfloat4& b) { return a > b; }
+ __forceinline vboolf4 le(const vfloat4& a, const vfloat4& b) { return a <= b; }
+
+#if defined(__AVX512VL__)
+ __forceinline vboolf4 eq(const vboolf4& mask, const vfloat4& a, const vfloat4& b) { return _mm_mask_cmp_ps_mask(mask, a, b, _MM_CMPINT_EQ); }
+ __forceinline vboolf4 ne(const vboolf4& mask, const vfloat4& a, const vfloat4& b) { return _mm_mask_cmp_ps_mask(mask, a, b, _MM_CMPINT_NE); }
+ __forceinline vboolf4 lt(const vboolf4& mask, const vfloat4& a, const vfloat4& b) { return _mm_mask_cmp_ps_mask(mask, a, b, _MM_CMPINT_LT); }
+ __forceinline vboolf4 ge(const vboolf4& mask, const vfloat4& a, const vfloat4& b) { return _mm_mask_cmp_ps_mask(mask, a, b, _MM_CMPINT_GE); }
+ __forceinline vboolf4 gt(const vboolf4& mask, const vfloat4& a, const vfloat4& b) { return _mm_mask_cmp_ps_mask(mask, a, b, _MM_CMPINT_GT); }
+ __forceinline vboolf4 le(const vboolf4& mask, const vfloat4& a, const vfloat4& b) { return _mm_mask_cmp_ps_mask(mask, a, b, _MM_CMPINT_LE); }
+#else
+ __forceinline vboolf4 eq(const vboolf4& mask, const vfloat4& a, const vfloat4& b) { return mask & (a == b); }
+ __forceinline vboolf4 ne(const vboolf4& mask, const vfloat4& a, const vfloat4& b) { return mask & (a != b); }
+ __forceinline vboolf4 lt(const vboolf4& mask, const vfloat4& a, const vfloat4& b) { return mask & (a < b); }
+ __forceinline vboolf4 ge(const vboolf4& mask, const vfloat4& a, const vfloat4& b) { return mask & (a >= b); }
+ __forceinline vboolf4 gt(const vboolf4& mask, const vfloat4& a, const vfloat4& b) { return mask & (a > b); }
+ __forceinline vboolf4 le(const vboolf4& mask, const vfloat4& a, const vfloat4& b) { return mask & (a <= b); }
+#endif
+
+ template<int mask>
+ __forceinline vfloat4 select(const vfloat4& t, const vfloat4& f)
+ {
+#if defined(__SSE4_1__)
+ return _mm_blend_ps(f, t, mask);
+#else
+ return select(vboolf4(mask), t, f);
+#endif
+ }
+
+#if defined(__aarch64__)
+ template<> __forceinline vfloat4 select<0>(const vfloat4& t, const vfloat4& f) {
+ return _mm_blendv_ps(f, t, vreinterpretq_f32_u32(vzero));
+ }
+ template<> __forceinline vfloat4 select<1>(const vfloat4& t, const vfloat4& f) {
+ return _mm_blendv_ps(f, t, vreinterpretq_f32_u32(v000F));
+ }
+ template<> __forceinline vfloat4 select<2>(const vfloat4& t, const vfloat4& f) {
+ return _mm_blendv_ps(f, t, vreinterpretq_f32_u32(v00F0));
+ }
+ template<> __forceinline vfloat4 select<3>(const vfloat4& t, const vfloat4& f) {
+ return _mm_blendv_ps(f, t, vreinterpretq_f32_u32(v00FF));
+ }
+ template<> __forceinline vfloat4 select<4>(const vfloat4& t, const vfloat4& f) {
+ return _mm_blendv_ps(f, t, vreinterpretq_f32_u32(v0F00));
+ }
+ template<> __forceinline vfloat4 select<5>(const vfloat4& t, const vfloat4& f) {
+ return _mm_blendv_ps(f, t, vreinterpretq_f32_u32(v0F0F));
+ }
+ template<> __forceinline vfloat4 select<6>(const vfloat4& t, const vfloat4& f) {
+ return _mm_blendv_ps(f, t, vreinterpretq_f32_u32(v0FF0));
+ }
+ template<> __forceinline vfloat4 select<7>(const vfloat4& t, const vfloat4& f) {
+ return _mm_blendv_ps(f, t, vreinterpretq_f32_u32(v0FFF));
+ }
+ template<> __forceinline vfloat4 select<8>(const vfloat4& t, const vfloat4& f) {
+ return _mm_blendv_ps(f, t, vreinterpretq_f32_u32(vF000));
+ }
+ template<> __forceinline vfloat4 select<9>(const vfloat4& t, const vfloat4& f) {
+ return _mm_blendv_ps(f, t, vreinterpretq_f32_u32(vF00F));
+ }
+ template<> __forceinline vfloat4 select<10>(const vfloat4& t, const vfloat4& f) {
+ return _mm_blendv_ps(f, t, vreinterpretq_f32_u32(vF0F0));
+ }
+ template<> __forceinline vfloat4 select<11>(const vfloat4& t, const vfloat4& f) {
+ return _mm_blendv_ps(f, t, vreinterpretq_f32_u32(vF0FF));
+ }
+ template<> __forceinline vfloat4 select<12>(const vfloat4& t, const vfloat4& f) {
+ return _mm_blendv_ps(f, t, vreinterpretq_f32_u32(vFF00));
+ }
+ template<> __forceinline vfloat4 select<13>(const vfloat4& t, const vfloat4& f) {
+ return _mm_blendv_ps(f, t, vreinterpretq_f32_u32(vFF0F));
+ }
+ template<> __forceinline vfloat4 select<14>(const vfloat4& t, const vfloat4& f) {
+ return _mm_blendv_ps(f, t, vreinterpretq_f32_u32(vFFF0));
+ }
+ template<> __forceinline vfloat4 select<15>(const vfloat4& t, const vfloat4& f) {
+ return _mm_blendv_ps(f, t, vreinterpretq_f32_u32(vFFFF));
+ }
+#endif
+
+ __forceinline vfloat4 lerp(const vfloat4& a, const vfloat4& b, const vfloat4& t) {
+ return madd(t,b-a,a);
+ }
+
+ __forceinline bool isvalid(const vfloat4& v) {
+ return all((v > vfloat4(-FLT_LARGE)) & (v < vfloat4(+FLT_LARGE)));
+ }
+
+ __forceinline bool is_finite(const vfloat4& a) {
+ return all((a >= vfloat4(-FLT_MAX)) & (a <= vfloat4(+FLT_MAX)));
+ }
+
+ __forceinline bool is_finite(const vboolf4& valid, const vfloat4& a) {
+ return all(valid, (a >= vfloat4(-FLT_MAX)) & (a <= vfloat4(+FLT_MAX)));
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Rounding Functions
+ ////////////////////////////////////////////////////////////////////////////////
+
+#if defined(__aarch64__)
+ __forceinline vfloat4 floor(const vfloat4& a) { return vrndmq_f32(a.v); } // towards -inf
+ __forceinline vfloat4 ceil (const vfloat4& a) { return vrndpq_f32(a.v); } // toward +inf
+ __forceinline vfloat4 trunc(const vfloat4& a) { return vrndq_f32(a.v); } // towards 0
+ __forceinline vfloat4 round(const vfloat4& a) { return vrndnq_f32(a.v); } // to nearest, ties to even. NOTE(LTE): arm clang uses vrndnq, old gcc uses vrndqn?
+#elif defined (__SSE4_1__)
+ __forceinline vfloat4 floor(const vfloat4& a) { return _mm_round_ps(a, _MM_FROUND_TO_NEG_INF ); }
+ __forceinline vfloat4 ceil (const vfloat4& a) { return _mm_round_ps(a, _MM_FROUND_TO_POS_INF ); }
+ __forceinline vfloat4 trunc(const vfloat4& a) { return _mm_round_ps(a, _MM_FROUND_TO_ZERO ); }
+ __forceinline vfloat4 round(const vfloat4& a) { return _mm_round_ps(a, _MM_FROUND_TO_NEAREST_INT); } // (even) https://www.felixcloutier.com/x86/roundpd
+#else
+ __forceinline vfloat4 floor(const vfloat4& a) { return vfloat4(floorf(a[0]),floorf(a[1]),floorf(a[2]),floorf(a[3])); }
+ __forceinline vfloat4 ceil (const vfloat4& a) { return vfloat4(ceilf (a[0]),ceilf (a[1]),ceilf (a[2]),ceilf (a[3])); }
+ __forceinline vfloat4 trunc(const vfloat4& a) { return vfloat4(truncf(a[0]),truncf(a[1]),truncf(a[2]),truncf(a[3])); }
+ __forceinline vfloat4 round(const vfloat4& a) { return vfloat4(roundf(a[0]),roundf(a[1]),roundf(a[2]),roundf(a[3])); }
+#endif
+ __forceinline vfloat4 frac(const vfloat4& a) { return a-floor(a); }
+
+ __forceinline vint4 floori(const vfloat4& a) {
+#if defined(__aarch64__)
+ return vcvtq_s32_f32(floor(a));
+#elif defined(__SSE4_1__)
+ return vint4(floor(a));
+#else
+ return vint4(a-vfloat4(0.5f));
+#endif
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Movement/Shifting/Shuffling Functions
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vfloat4 unpacklo(const vfloat4& a, const vfloat4& b) { return _mm_unpacklo_ps(a, b); }
+ __forceinline vfloat4 unpackhi(const vfloat4& a, const vfloat4& b) { return _mm_unpackhi_ps(a, b); }
+
+#if defined(__aarch64__)
+ template<int i0, int i1, int i2, int i3>
+ __forceinline vfloat4 shuffle(const vfloat4& v) {
+ return vreinterpretq_f32_u8(vqtbl1q_u8( (uint8x16_t)v.v, _MN_SHUFFLE(i0, i1, i2, i3)));
+ }
+ template<int i0, int i1, int i2, int i3>
+ __forceinline vfloat4 shuffle(const vfloat4& a, const vfloat4& b) {
+ return vreinterpretq_f32_u8(vqtbl2q_u8( (uint8x16x2_t){(uint8x16_t)a.v, (uint8x16_t)b.v}, _MF_SHUFFLE(i0, i1, i2, i3)));
+ }
+#else
+ template<int i0, int i1, int i2, int i3>
+ __forceinline vfloat4 shuffle(const vfloat4& v) {
+ return _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v), _MM_SHUFFLE(i3, i2, i1, i0)));
+ }
+
+ template<int i0, int i1, int i2, int i3>
+ __forceinline vfloat4 shuffle(const vfloat4& a, const vfloat4& b) {
+ return _mm_shuffle_ps(a, b, _MM_SHUFFLE(i3, i2, i1, i0));
+ }
+#endif
+
+#if defined (__SSSE3__)
+ __forceinline vfloat4 shuffle8(const vfloat4& a, const vint4& shuf) {
+ return _mm_castsi128_ps(_mm_shuffle_epi8(_mm_castps_si128(a), shuf));
+ }
+#endif
+
+#if defined(__aarch64__)
+ template<> __forceinline vfloat4 shuffle<0, 0, 2, 2>(const vfloat4& v) { return __m128(vqtbl1q_u8( uint8x16_t(v.v), v0022 )); }
+ template<> __forceinline vfloat4 shuffle<1, 1, 3, 3>(const vfloat4& v) { return __m128(vqtbl1q_u8( uint8x16_t(v.v), v1133)); }
+ template<> __forceinline vfloat4 shuffle<0, 1, 0, 1>(const vfloat4& v) { return __m128(vqtbl1q_u8( uint8x16_t(v.v), v0101)); }
+#elif defined(__SSE3__)
+ template<> __forceinline vfloat4 shuffle<0, 0, 2, 2>(const vfloat4& v) { return _mm_moveldup_ps(v); }
+ template<> __forceinline vfloat4 shuffle<1, 1, 3, 3>(const vfloat4& v) { return _mm_movehdup_ps(v); }
+ template<> __forceinline vfloat4 shuffle<0, 1, 0, 1>(const vfloat4& v) { return _mm_castpd_ps(_mm_movedup_pd(_mm_castps_pd(v))); }
+#endif
+
+ template<int i>
+ __forceinline vfloat4 shuffle(const vfloat4& v) {
+ return shuffle<i,i,i,i>(v);
+ }
+
+#if defined(__aarch64__)
+ template<int i> __forceinline float extract(const vfloat4& a);
+ template<> __forceinline float extract<0>(const vfloat4& b) {
+ return b[0];
+ }
+ template<> __forceinline float extract<1>(const vfloat4& b) {
+ return b[1];
+ }
+ template<> __forceinline float extract<2>(const vfloat4& b) {
+ return b[2];
+ }
+ template<> __forceinline float extract<3>(const vfloat4& b) {
+ return b[3];
+ }
+#elif defined (__SSE4_1__) && !defined(__GNUC__)
+ template<int i> __forceinline float extract(const vfloat4& a) { return _mm_cvtss_f32(_mm_extract_ps(a,i)); }
+ template<> __forceinline float extract<0>(const vfloat4& a) { return _mm_cvtss_f32(a); }
+#else
+ template<int i> __forceinline float extract(const vfloat4& a) { return _mm_cvtss_f32(shuffle<i,i,i,i>(a)); }
+ template<> __forceinline float extract<0>(const vfloat4& a) { return _mm_cvtss_f32(a); }
+#endif
+
+
+#if defined(__aarch64__)
+ template<int dst> __forceinline vfloat4 insert(const vfloat4& a, float b);
+ template<> __forceinline vfloat4 insert<0>(const vfloat4& a, float b)
+ {
+ vfloat4 c = a;
+ c[0] = b;
+ return c;
+ }
+ template<> __forceinline vfloat4 insert<1>(const vfloat4& a, float b)
+ {
+ vfloat4 c = a;
+ c[1] = b;
+ return c;
+ }
+ template<> __forceinline vfloat4 insert<2>(const vfloat4& a, float b)
+ {
+ vfloat4 c = a;
+ c[2] = b;
+ return c;
+ }
+ template<> __forceinline vfloat4 insert<3>(const vfloat4& a, float b)
+ {
+ vfloat4 c = a;
+ c[3] = b;
+ return c;
+ }
+#elif defined (__SSE4_1__)
+ template<int dst, int src, int clr> __forceinline vfloat4 insert(const vfloat4& a, const vfloat4& b) { return _mm_insert_ps(a, b, (dst << 4) | (src << 6) | clr); }
+ template<int dst, int src> __forceinline vfloat4 insert(const vfloat4& a, const vfloat4& b) { return insert<dst, src, 0>(a, b); }
+ template<int dst> __forceinline vfloat4 insert(const vfloat4& a, const float b) { return insert<dst, 0>(a, _mm_set_ss(b)); }
+#else
+ template<int dst, int src> __forceinline vfloat4 insert(const vfloat4& a, const vfloat4& b) { vfloat4 c = a; c[dst&3] = b[src&3]; return c; }
+ template<int dst> __forceinline vfloat4 insert(const vfloat4& a, float b) { vfloat4 c = a; c[dst&3] = b; return c; }
+#endif
+
+#if defined(__aarch64__)
+ __forceinline float toScalar(const vfloat4& v) {
+ return v[0];
+ }
+#else
+ __forceinline float toScalar(const vfloat4& v) { return _mm_cvtss_f32(v); }
+#endif
+ __forceinline vfloat4 broadcast4f(const vfloat4& a, size_t k) {
+ return vfloat4::broadcast(&a[k]);
+ }
+
+ __forceinline vfloat4 shift_right_1(const vfloat4& x) {
+ return _mm_castsi128_ps(_mm_srli_si128(_mm_castps_si128(x), 4));
+ }
+
+#if defined (__AVX2__)
+ __forceinline vfloat4 permute(const vfloat4 &a, const __m128i &index) {
+ return _mm_permutevar_ps(a,index);
+ }
+
+ __forceinline vfloat4 broadcast1f(const void* a) { return _mm_broadcast_ss((float*)a); }
+
+#endif
+
+#if defined(__AVX512VL__)
+ template<int i>
+ __forceinline vfloat4 align_shift_right(const vfloat4& a, const vfloat4& b) {
+ return _mm_castsi128_ps(_mm_alignr_epi32(_mm_castps_si128(a), _mm_castps_si128(b), i));
+ }
+#endif
+
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Sorting Network
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vfloat4 sort_ascending(const vfloat4& v)
+ {
+ const vfloat4 a0 = v;
+ const vfloat4 b0 = shuffle<1,0,3,2>(a0);
+ const vfloat4 c0 = min(a0,b0);
+ const vfloat4 d0 = max(a0,b0);
+ const vfloat4 a1 = select<0x5 /* 0b0101 */>(c0,d0);
+ const vfloat4 b1 = shuffle<2,3,0,1>(a1);
+ const vfloat4 c1 = min(a1,b1);
+ const vfloat4 d1 = max(a1,b1);
+ const vfloat4 a2 = select<0x3 /* 0b0011 */>(c1,d1);
+ const vfloat4 b2 = shuffle<0,2,1,3>(a2);
+ const vfloat4 c2 = min(a2,b2);
+ const vfloat4 d2 = max(a2,b2);
+ const vfloat4 a3 = select<0x2 /* 0b0010 */>(c2,d2);
+ return a3;
+ }
+
+ __forceinline vfloat4 sort_descending(const vfloat4& v)
+ {
+ const vfloat4 a0 = v;
+ const vfloat4 b0 = shuffle<1,0,3,2>(a0);
+ const vfloat4 c0 = max(a0,b0);
+ const vfloat4 d0 = min(a0,b0);
+ const vfloat4 a1 = select<0x5 /* 0b0101 */>(c0,d0);
+ const vfloat4 b1 = shuffle<2,3,0,1>(a1);
+ const vfloat4 c1 = max(a1,b1);
+ const vfloat4 d1 = min(a1,b1);
+ const vfloat4 a2 = select<0x3 /* 0b0011 */>(c1,d1);
+ const vfloat4 b2 = shuffle<0,2,1,3>(a2);
+ const vfloat4 c2 = max(a2,b2);
+ const vfloat4 d2 = min(a2,b2);
+ const vfloat4 a3 = select<0x2 /* 0b0010 */>(c2,d2);
+ return a3;
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Transpose
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline void transpose(const vfloat4& r0, const vfloat4& r1, const vfloat4& r2, const vfloat4& r3, vfloat4& c0, vfloat4& c1, vfloat4& c2, vfloat4& c3)
+ {
+ vfloat4 l02 = unpacklo(r0,r2);
+ vfloat4 h02 = unpackhi(r0,r2);
+ vfloat4 l13 = unpacklo(r1,r3);
+ vfloat4 h13 = unpackhi(r1,r3);
+ c0 = unpacklo(l02,l13);
+ c1 = unpackhi(l02,l13);
+ c2 = unpacklo(h02,h13);
+ c3 = unpackhi(h02,h13);
+ }
+
+ __forceinline void transpose(const vfloat4& r0, const vfloat4& r1, const vfloat4& r2, const vfloat4& r3, vfloat4& c0, vfloat4& c1, vfloat4& c2)
+ {
+ vfloat4 l02 = unpacklo(r0,r2);
+ vfloat4 h02 = unpackhi(r0,r2);
+ vfloat4 l13 = unpacklo(r1,r3);
+ vfloat4 h13 = unpackhi(r1,r3);
+ c0 = unpacklo(l02,l13);
+ c1 = unpackhi(l02,l13);
+ c2 = unpacklo(h02,h13);
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Reductions
+ ////////////////////////////////////////////////////////////////////////////////
+#if defined(__aarch64__)
+ __forceinline vfloat4 vreduce_min(const vfloat4& v) { float h = vminvq_f32(v); return vdupq_n_f32(h); }
+ __forceinline vfloat4 vreduce_max(const vfloat4& v) { float h = vmaxvq_f32(v); return vdupq_n_f32(h); }
+ __forceinline vfloat4 vreduce_add(const vfloat4& v) { float h = vaddvq_f32(v); return vdupq_n_f32(h); }
+#else
+ __forceinline vfloat4 vreduce_min(const vfloat4& v) { vfloat4 h = min(shuffle<1,0,3,2>(v),v); return min(shuffle<2,3,0,1>(h),h); }
+ __forceinline vfloat4 vreduce_max(const vfloat4& v) { vfloat4 h = max(shuffle<1,0,3,2>(v),v); return max(shuffle<2,3,0,1>(h),h); }
+ __forceinline vfloat4 vreduce_add(const vfloat4& v) { vfloat4 h = shuffle<1,0,3,2>(v) + v ; return shuffle<2,3,0,1>(h) + h ; }
+#endif
+
+#if defined(__aarch64__)
+ __forceinline float reduce_min(const vfloat4& v) { return vminvq_f32(v); }
+ __forceinline float reduce_max(const vfloat4& v) { return vmaxvq_f32(v); }
+ __forceinline float reduce_add(const vfloat4& v) { return vaddvq_f32(v); }
+#else
+ __forceinline float reduce_min(const vfloat4& v) { return _mm_cvtss_f32(vreduce_min(v)); }
+ __forceinline float reduce_max(const vfloat4& v) { return _mm_cvtss_f32(vreduce_max(v)); }
+ __forceinline float reduce_add(const vfloat4& v) { return _mm_cvtss_f32(vreduce_add(v)); }
+#endif
+
+ __forceinline size_t select_min(const vboolf4& valid, const vfloat4& v)
+ {
+ const vfloat4 a = select(valid,v,vfloat4(pos_inf));
+ const vbool4 valid_min = valid & (a == vreduce_min(a));
+ return bsf(movemask(any(valid_min) ? valid_min : valid));
+ }
+ __forceinline size_t select_max(const vboolf4& valid, const vfloat4& v)
+ {
+ const vfloat4 a = select(valid,v,vfloat4(neg_inf));
+ const vbool4 valid_max = valid & (a == vreduce_max(a));
+ return bsf(movemask(any(valid_max) ? valid_max : valid));
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Euclidian Space Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline float dot(const vfloat4& a, const vfloat4& b) {
+ return reduce_add(a*b);
+ }
+
+ __forceinline vfloat4 cross(const vfloat4& a, const vfloat4& b)
+ {
+ const vfloat4 a0 = a;
+ const vfloat4 b0 = shuffle<1,2,0,3>(b);
+ const vfloat4 a1 = shuffle<1,2,0,3>(a);
+ const vfloat4 b1 = b;
+ return shuffle<1,2,0,3>(prod_diff(a0,b0,a1,b1));
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Output Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline embree_ostream operator <<(embree_ostream cout, const vfloat4& a) {
+ return cout << "<" << a[0] << ", " << a[1] << ", " << a[2] << ", " << a[3] << ">";
+ }
+
+}
diff --git a/thirdparty/embree-aarch64/common/simd/vfloat8_avx.h b/thirdparty/embree-aarch64/common/simd/vfloat8_avx.h
new file mode 100644
index 0000000000..3c7e4a8cdc
--- /dev/null
+++ b/thirdparty/embree-aarch64/common/simd/vfloat8_avx.h
@@ -0,0 +1,847 @@
+// Copyright 2009-2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+namespace embree
+{
+ /* 8-wide AVX float type */
+ template<>
+ struct vfloat<8>
+ {
+ ALIGNED_STRUCT_(32);
+
+ typedef vboolf8 Bool;
+ typedef vint8 Int;
+ typedef vfloat8 Float;
+
+ enum { size = 8 }; // number of SIMD elements
+ union { __m256 v; float f[8]; int i[8]; }; // data
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constructors, Assignment & Cast Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vfloat() {}
+ __forceinline vfloat(const vfloat8& other) { v = other.v; }
+ __forceinline vfloat8& operator =(const vfloat8& other) { v = other.v; return *this; }
+
+ __forceinline vfloat(__m256 a) : v(a) {}
+ __forceinline operator const __m256&() const { return v; }
+ __forceinline operator __m256&() { return v; }
+
+ __forceinline explicit vfloat(const vfloat4& a) : v(_mm256_insertf128_ps(_mm256_castps128_ps256(a),a,1)) {}
+ __forceinline vfloat(const vfloat4& a, const vfloat4& b) : v(_mm256_insertf128_ps(_mm256_castps128_ps256(a),b,1)) {}
+
+ __forceinline explicit vfloat(const int8_t* a) : v(_mm256_loadu_ps((const float*)a)) {}
+ __forceinline vfloat(float a) : v(_mm256_set1_ps(a)) {}
+ __forceinline vfloat(float a, float b) : v(_mm256_set_ps(b, a, b, a, b, a, b, a)) {}
+ __forceinline vfloat(float a, float b, float c, float d) : v(_mm256_set_ps(d, c, b, a, d, c, b, a)) {}
+ __forceinline vfloat(float a, float b, float c, float d, float e, float f, float g, float h) : v(_mm256_set_ps(h, g, f, e, d, c, b, a)) {}
+
+ __forceinline explicit vfloat(__m256i a) : v(_mm256_cvtepi32_ps(a)) {}
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constants
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vfloat(ZeroTy) : v(_mm256_setzero_ps()) {}
+ __forceinline vfloat(OneTy) : v(_mm256_set1_ps(1.0f)) {}
+ __forceinline vfloat(PosInfTy) : v(_mm256_set1_ps(pos_inf)) {}
+ __forceinline vfloat(NegInfTy) : v(_mm256_set1_ps(neg_inf)) {}
+ __forceinline vfloat(StepTy) : v(_mm256_set_ps(7.0f, 6.0f, 5.0f, 4.0f, 3.0f, 2.0f, 1.0f, 0.0f)) {}
+ __forceinline vfloat(NaNTy) : v(_mm256_set1_ps(nan)) {}
+ __forceinline vfloat(UndefinedTy) : v(_mm256_undefined_ps()) {}
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Loads and Stores
+ ////////////////////////////////////////////////////////////////////////////////
+
+ static __forceinline vfloat8 broadcast(const void* a) {
+ return _mm256_broadcast_ss((float*)a);
+ }
+
+ static __forceinline vfloat8 broadcast2(const float* a, const float* b) {
+#if defined(__INTEL_COMPILER)
+ const vfloat8 v0 = _mm256_broadcast_ss(a);
+ const vfloat8 v1 = _mm256_broadcast_ss(b);
+ return _mm256_blend_ps(v1, v0, 0xf);
+#else
+ return _mm256_set_ps(*b,*b,*b,*b,*a,*a,*a,*a);
+#endif
+ }
+
+ static __forceinline vfloat8 broadcast4f(const vfloat4* ptr) {
+ return _mm256_broadcast_ps((__m128*)ptr);
+ }
+
+ static __forceinline vfloat8 load(const int8_t* ptr) {
+#if defined(__AVX2__)
+ return _mm256_cvtepi32_ps(_mm256_cvtepi8_epi32(_mm_loadu_si128((__m128i*)ptr)));
+#else
+ return vfloat8(vfloat4::load(ptr),vfloat4::load(ptr+4));
+#endif
+ }
+
+ static __forceinline vfloat8 load(const uint8_t* ptr) {
+#if defined(__AVX2__)
+ return _mm256_cvtepi32_ps(_mm256_cvtepu8_epi32(_mm_loadu_si128((__m128i*)ptr)));
+#else
+ return vfloat8(vfloat4::load(ptr),vfloat4::load(ptr+4));
+#endif
+ }
+
+ static __forceinline vfloat8 load(const short* ptr) {
+#if defined(__AVX2__)
+ return _mm256_cvtepi32_ps(_mm256_cvtepi16_epi32(_mm_loadu_si128((__m128i*)ptr)));
+#else
+ return vfloat8(vfloat4::load(ptr),vfloat4::load(ptr+4));
+#endif
+ }
+
+ static __forceinline vfloat8 load (const void* ptr) { return _mm256_load_ps((float*)ptr); }
+ static __forceinline vfloat8 loadu(const void* ptr) { return _mm256_loadu_ps((float*)ptr); }
+
+ static __forceinline void store (void* ptr, const vfloat8& v) { return _mm256_store_ps((float*)ptr,v); }
+ static __forceinline void storeu(void* ptr, const vfloat8& v) { return _mm256_storeu_ps((float*)ptr,v); }
+
+#if defined(__AVX512VL__)
+
+ static __forceinline vfloat8 compact(const vboolf8& mask, vfloat8 &v) {
+ return _mm256_mask_compress_ps(v, mask, v);
+ }
+ static __forceinline vfloat8 compact(const vboolf8& mask, vfloat8 &a, const vfloat8& b) {
+ return _mm256_mask_compress_ps(a, mask, b);
+ }
+
+ static __forceinline vfloat8 load (const vboolf8& mask, const void* ptr) { return _mm256_mask_load_ps (_mm256_setzero_ps(),mask,(float*)ptr); }
+ static __forceinline vfloat8 loadu(const vboolf8& mask, const void* ptr) { return _mm256_mask_loadu_ps(_mm256_setzero_ps(),mask,(float*)ptr); }
+
+ static __forceinline void store (const vboolf8& mask, void* ptr, const vfloat8& v) { _mm256_mask_store_ps ((float*)ptr,mask,v); }
+ static __forceinline void storeu(const vboolf8& mask, void* ptr, const vfloat8& v) { _mm256_mask_storeu_ps((float*)ptr,mask,v); }
+#elif defined(__aarch64__)
+ static __forceinline vfloat8 load (const vboolf8& mask, const void* ptr) { return _mm256_maskload_ps((float*)ptr,(__m256i)mask.v); }
+ static __forceinline vfloat8 loadu(const vboolf8& mask, const void* ptr) { return _mm256_maskload_ps((float*)ptr,(__m256i)mask.v); }
+
+ static __forceinline void store (const vboolf8& mask, void* ptr, const vfloat8& v) { _mm256_maskstore_ps((float*)ptr,(__m256i)mask.v,v); }
+ static __forceinline void storeu(const vboolf8& mask, void* ptr, const vfloat8& v) { _mm256_maskstore_ps((float*)ptr,(__m256i)mask.v,v); }
+#else
+ static __forceinline vfloat8 load (const vboolf8& mask, const void* ptr) { return _mm256_maskload_ps((float*)ptr,(__m256i)mask); }
+ static __forceinline vfloat8 loadu(const vboolf8& mask, const void* ptr) { return _mm256_maskload_ps((float*)ptr,(__m256i)mask); }
+
+ static __forceinline void store (const vboolf8& mask, void* ptr, const vfloat8& v) { _mm256_maskstore_ps((float*)ptr,(__m256i)mask,v); }
+ static __forceinline void storeu(const vboolf8& mask, void* ptr, const vfloat8& v) { _mm256_maskstore_ps((float*)ptr,(__m256i)mask,v); }
+#endif
+
+#if defined(__AVX2__)
+ static __forceinline vfloat8 load_nt(void* ptr) {
+ return _mm256_castsi256_ps(_mm256_stream_load_si256((__m256i*)ptr));
+ }
+#endif
+
+ static __forceinline void store_nt(void* ptr, const vfloat8& v) {
+ _mm256_stream_ps((float*)ptr,v);
+ }
+
+ template<int scale = 4>
+ static __forceinline vfloat8 gather(const float* ptr, const vint8& index) {
+#if defined(__AVX2__) && !defined(__aarch64__)
+ return _mm256_i32gather_ps(ptr, index ,scale);
+#else
+ return vfloat8(
+ *(float*)(((int8_t*)ptr)+scale*index[0]),
+ *(float*)(((int8_t*)ptr)+scale*index[1]),
+ *(float*)(((int8_t*)ptr)+scale*index[2]),
+ *(float*)(((int8_t*)ptr)+scale*index[3]),
+ *(float*)(((int8_t*)ptr)+scale*index[4]),
+ *(float*)(((int8_t*)ptr)+scale*index[5]),
+ *(float*)(((int8_t*)ptr)+scale*index[6]),
+ *(float*)(((int8_t*)ptr)+scale*index[7]));
+#endif
+ }
+
+ template<int scale = 4>
+ static __forceinline vfloat8 gather(const vboolf8& mask, const float* ptr, const vint8& index) {
+ vfloat8 r = zero;
+#if defined(__AVX512VL__)
+ return _mm256_mmask_i32gather_ps(r, mask, index, ptr, scale);
+#elif defined(__AVX2__) && !defined(__aarch64__)
+ return _mm256_mask_i32gather_ps(r, ptr, index, mask, scale);
+#else
+ if (likely(mask[0])) r[0] = *(float*)(((int8_t*)ptr)+scale*index[0]);
+ if (likely(mask[1])) r[1] = *(float*)(((int8_t*)ptr)+scale*index[1]);
+ if (likely(mask[2])) r[2] = *(float*)(((int8_t*)ptr)+scale*index[2]);
+ if (likely(mask[3])) r[3] = *(float*)(((int8_t*)ptr)+scale*index[3]);
+ if (likely(mask[4])) r[4] = *(float*)(((int8_t*)ptr)+scale*index[4]);
+ if (likely(mask[5])) r[5] = *(float*)(((int8_t*)ptr)+scale*index[5]);
+ if (likely(mask[6])) r[6] = *(float*)(((int8_t*)ptr)+scale*index[6]);
+ if (likely(mask[7])) r[7] = *(float*)(((int8_t*)ptr)+scale*index[7]);
+ return r;
+ #endif
+ }
+
+ template<int scale = 4>
+ static __forceinline void scatter(void* ptr, const vint8& ofs, const vfloat8& v)
+ {
+#if defined(__AVX512VL__)
+ _mm256_i32scatter_ps((float*)ptr, ofs, v, scale);
+#else
+ *(float*)(((int8_t*)ptr)+scale*ofs[0]) = v[0];
+ *(float*)(((int8_t*)ptr)+scale*ofs[1]) = v[1];
+ *(float*)(((int8_t*)ptr)+scale*ofs[2]) = v[2];
+ *(float*)(((int8_t*)ptr)+scale*ofs[3]) = v[3];
+ *(float*)(((int8_t*)ptr)+scale*ofs[4]) = v[4];
+ *(float*)(((int8_t*)ptr)+scale*ofs[5]) = v[5];
+ *(float*)(((int8_t*)ptr)+scale*ofs[6]) = v[6];
+ *(float*)(((int8_t*)ptr)+scale*ofs[7]) = v[7];
+#endif
+ }
+
+ template<int scale = 4>
+ static __forceinline void scatter(const vboolf8& mask, void* ptr, const vint8& ofs, const vfloat8& v)
+ {
+#if defined(__AVX512VL__)
+ _mm256_mask_i32scatter_ps((float*)ptr, mask, ofs, v, scale);
+#else
+ if (likely(mask[0])) *(float*)(((int8_t*)ptr)+scale*ofs[0]) = v[0];
+ if (likely(mask[1])) *(float*)(((int8_t*)ptr)+scale*ofs[1]) = v[1];
+ if (likely(mask[2])) *(float*)(((int8_t*)ptr)+scale*ofs[2]) = v[2];
+ if (likely(mask[3])) *(float*)(((int8_t*)ptr)+scale*ofs[3]) = v[3];
+ if (likely(mask[4])) *(float*)(((int8_t*)ptr)+scale*ofs[4]) = v[4];
+ if (likely(mask[5])) *(float*)(((int8_t*)ptr)+scale*ofs[5]) = v[5];
+ if (likely(mask[6])) *(float*)(((int8_t*)ptr)+scale*ofs[6]) = v[6];
+ if (likely(mask[7])) *(float*)(((int8_t*)ptr)+scale*ofs[7]) = v[7];
+#endif
+ }
+
+ static __forceinline void store(const vboolf8& mask, int8_t* ptr, const vint8& ofs, const vfloat8& v) {
+ scatter<1>(mask,ptr,ofs,v);
+ }
+ static __forceinline void store(const vboolf8& mask, float* ptr, const vint8& ofs, const vfloat8& v) {
+ scatter<4>(mask,ptr,ofs,v);
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Array Access
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline const float& operator [](size_t index) const { assert(index < 8); return f[index]; }
+ __forceinline float& operator [](size_t index) { assert(index < 8); return f[index]; }
+ };
+
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Unary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vfloat8 asFloat(const vint8& a) { return _mm256_castsi256_ps(a); }
+ __forceinline vint8 asInt (const vfloat8& a) { return _mm256_castps_si256(a); }
+
+ __forceinline vint8 toInt (const vfloat8& a) { return vint8(a); }
+ __forceinline vfloat8 toFloat(const vint8& a) { return vfloat8(a); }
+
+ __forceinline vfloat8 operator +(const vfloat8& a) { return a; }
+#if !defined(__aarch64__)
+ __forceinline vfloat8 operator -(const vfloat8& a) {
+ const __m256 mask = _mm256_castsi256_ps(_mm256_set1_epi32(0x80000000));
+ return _mm256_xor_ps(a, mask);
+ }
+#else
+ __forceinline vfloat8 operator -(const vfloat8& a) {
+ __m256 res;
+ res.lo = vnegq_f32(a.v.lo);
+ res.hi = vnegq_f32(a.v.hi);
+ return res;
+}
+#endif
+
+#if !defined(__aarch64__)
+__forceinline vfloat8 abs(const vfloat8& a) {
+ const __m256 mask = _mm256_castsi256_ps(_mm256_set1_epi32(0x7fffffff));
+ return _mm256_and_ps(a, mask);
+}
+#else
+__forceinline vfloat8 abs(const vfloat8& a) {
+ __m256 res;
+ res.lo = vabsq_f32(a.v.lo);
+ res.hi = vabsq_f32(a.v.hi);
+ return res;
+}
+#endif
+
+#if !defined(__aarch64__)
+ __forceinline vfloat8 sign (const vfloat8& a) { return _mm256_blendv_ps(vfloat8(one), -vfloat8(one), _mm256_cmp_ps(a, vfloat8(zero), _CMP_NGE_UQ)); }
+#else
+ __forceinline vfloat8 sign (const vfloat8& a) { return _mm256_blendv_ps(vfloat8(one), -vfloat8(one), _mm256_cmplt_ps(a, vfloat8(zero))); }
+#endif
+ __forceinline vfloat8 signmsk(const vfloat8& a) { return _mm256_and_ps(a,_mm256_castsi256_ps(_mm256_set1_epi32(0x80000000))); }
+
+
+ static __forceinline vfloat8 rcp(const vfloat8& a)
+ {
+#if defined(BUILD_IOS) && defined(__aarch64__)
+ // ios devices are faster doing full divide, no need for NR fixup
+ vfloat8 ret;
+ const float32x4_t one = vdupq_n_f32(1.0f);
+ ret.v.lo = vdivq_f32(one, a.v.lo);
+ ret.v.hi = vdivq_f32(one, a.v.hi);
+ return ret;
+#endif
+
+#if defined(__AVX512VL__)
+ const vfloat8 r = _mm256_rcp14_ps(a);
+#else
+ const vfloat8 r = _mm256_rcp_ps(a);
+#endif
+
+#if defined(__AVX2__) //&& !defined(aarch64)
+ return _mm256_mul_ps(r, _mm256_fnmadd_ps(r, a, vfloat8(2.0f)));
+#else
+ return _mm256_mul_ps(r, _mm256_sub_ps(vfloat8(2.0f), _mm256_mul_ps(r, a)));
+#endif
+ }
+ __forceinline vfloat8 sqr (const vfloat8& a) { return _mm256_mul_ps(a,a); }
+ __forceinline vfloat8 sqrt(const vfloat8& a) { return _mm256_sqrt_ps(a); }
+
+ static __forceinline vfloat8 rsqrt(const vfloat8& a)
+ {
+#if defined(__AVX512VL__)
+ const vfloat8 r = _mm256_rsqrt14_ps(a);
+#else
+ const vfloat8 r = _mm256_rsqrt_ps(a);
+#endif
+
+#if defined(__AVX2__)
+ return _mm256_fmadd_ps(_mm256_set1_ps(1.5f), r,
+ _mm256_mul_ps(_mm256_mul_ps(_mm256_mul_ps(a, _mm256_set1_ps(-0.5f)), r), _mm256_mul_ps(r, r)));
+#else
+ return _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(1.5f), r),
+ _mm256_mul_ps(_mm256_mul_ps(_mm256_mul_ps(a, _mm256_set1_ps(-0.5f)), r), _mm256_mul_ps(r, r)));
+#endif
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Binary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vfloat8 operator +(const vfloat8& a, const vfloat8& b) { return _mm256_add_ps(a, b); }
+ __forceinline vfloat8 operator +(const vfloat8& a, float b) { return a + vfloat8(b); }
+ __forceinline vfloat8 operator +(float a, const vfloat8& b) { return vfloat8(a) + b; }
+
+ __forceinline vfloat8 operator -(const vfloat8& a, const vfloat8& b) { return _mm256_sub_ps(a, b); }
+ __forceinline vfloat8 operator -(const vfloat8& a, float b) { return a - vfloat8(b); }
+ __forceinline vfloat8 operator -(float a, const vfloat8& b) { return vfloat8(a) - b; }
+
+ __forceinline vfloat8 operator *(const vfloat8& a, const vfloat8& b) { return _mm256_mul_ps(a, b); }
+ __forceinline vfloat8 operator *(const vfloat8& a, float b) { return a * vfloat8(b); }
+ __forceinline vfloat8 operator *(float a, const vfloat8& b) { return vfloat8(a) * b; }
+
+ __forceinline vfloat8 operator /(const vfloat8& a, const vfloat8& b) { return _mm256_div_ps(a, b); }
+ __forceinline vfloat8 operator /(const vfloat8& a, float b) { return a / vfloat8(b); }
+ __forceinline vfloat8 operator /(float a, const vfloat8& b) { return vfloat8(a) / b; }
+
+ __forceinline vfloat8 operator &(const vfloat8& a, const vfloat8& b) { return _mm256_and_ps(a,b); }
+ __forceinline vfloat8 operator |(const vfloat8& a, const vfloat8& b) { return _mm256_or_ps(a,b); }
+ __forceinline vfloat8 operator ^(const vfloat8& a, const vfloat8& b) { return _mm256_xor_ps(a,b); }
+ __forceinline vfloat8 operator ^(const vfloat8& a, const vint8& b) { return _mm256_xor_ps(a,_mm256_castsi256_ps(b)); }
+
+ __forceinline vfloat8 min(const vfloat8& a, const vfloat8& b) { return _mm256_min_ps(a, b); }
+ __forceinline vfloat8 min(const vfloat8& a, float b) { return _mm256_min_ps(a, vfloat8(b)); }
+ __forceinline vfloat8 min(float a, const vfloat8& b) { return _mm256_min_ps(vfloat8(a), b); }
+
+ __forceinline vfloat8 max(const vfloat8& a, const vfloat8& b) { return _mm256_max_ps(a, b); }
+ __forceinline vfloat8 max(const vfloat8& a, float b) { return _mm256_max_ps(a, vfloat8(b)); }
+ __forceinline vfloat8 max(float a, const vfloat8& b) { return _mm256_max_ps(vfloat8(a), b); }
+
+ /* need "static __forceinline for MSVC, otherwise we'll link the wrong version in debug mode */
+#if defined(__AVX2__)
+
+ static __forceinline vfloat8 mini(const vfloat8& a, const vfloat8& b) {
+ const vint8 ai = _mm256_castps_si256(a);
+ const vint8 bi = _mm256_castps_si256(b);
+ const vint8 ci = _mm256_min_epi32(ai,bi);
+ return _mm256_castsi256_ps(ci);
+ }
+
+ static __forceinline vfloat8 maxi(const vfloat8& a, const vfloat8& b) {
+ const vint8 ai = _mm256_castps_si256(a);
+ const vint8 bi = _mm256_castps_si256(b);
+ const vint8 ci = _mm256_max_epi32(ai,bi);
+ return _mm256_castsi256_ps(ci);
+ }
+
+ static __forceinline vfloat8 minui(const vfloat8& a, const vfloat8& b) {
+ const vint8 ai = _mm256_castps_si256(a);
+ const vint8 bi = _mm256_castps_si256(b);
+ const vint8 ci = _mm256_min_epu32(ai,bi);
+ return _mm256_castsi256_ps(ci);
+ }
+
+ static __forceinline vfloat8 maxui(const vfloat8& a, const vfloat8& b) {
+ const vint8 ai = _mm256_castps_si256(a);
+ const vint8 bi = _mm256_castps_si256(b);
+ const vint8 ci = _mm256_max_epu32(ai,bi);
+ return _mm256_castsi256_ps(ci);
+ }
+
+#else
+
+ static __forceinline vfloat8 mini(const vfloat8& a, const vfloat8& b) {
+ return asFloat(min(asInt(a),asInt(b)));
+ }
+
+ static __forceinline vfloat8 maxi(const vfloat8& a, const vfloat8& b) {
+ return asFloat(max(asInt(a),asInt(b)));
+ }
+
+#endif
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Ternary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+#if defined(__AVX2__)
+ static __forceinline vfloat8 madd (const vfloat8& a, const vfloat8& b, const vfloat8& c) { return _mm256_fmadd_ps(a,b,c); }
+ static __forceinline vfloat8 msub (const vfloat8& a, const vfloat8& b, const vfloat8& c) { return _mm256_fmsub_ps(a,b,c); }
+ static __forceinline vfloat8 nmadd (const vfloat8& a, const vfloat8& b, const vfloat8& c) { return _mm256_fnmadd_ps(a,b,c); }
+ static __forceinline vfloat8 nmsub (const vfloat8& a, const vfloat8& b, const vfloat8& c) { return _mm256_fnmsub_ps(a,b,c); }
+#else
+ static __forceinline vfloat8 madd (const vfloat8& a, const vfloat8& b, const vfloat8& c) { return a*b+c; }
+ static __forceinline vfloat8 msub (const vfloat8& a, const vfloat8& b, const vfloat8& c) { return a*b-c; }
+ static __forceinline vfloat8 nmadd (const vfloat8& a, const vfloat8& b, const vfloat8& c) { return -a*b+c;}
+ static __forceinline vfloat8 nmsub (const vfloat8& a, const vfloat8& b, const vfloat8& c) { return -a*b-c; }
+#endif
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Assignment Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vfloat8& operator +=(vfloat8& a, const vfloat8& b) { return a = a + b; }
+ __forceinline vfloat8& operator +=(vfloat8& a, float b) { return a = a + b; }
+
+ __forceinline vfloat8& operator -=(vfloat8& a, const vfloat8& b) { return a = a - b; }
+ __forceinline vfloat8& operator -=(vfloat8& a, float b) { return a = a - b; }
+
+ __forceinline vfloat8& operator *=(vfloat8& a, const vfloat8& b) { return a = a * b; }
+ __forceinline vfloat8& operator *=(vfloat8& a, float b) { return a = a * b; }
+
+ __forceinline vfloat8& operator /=(vfloat8& a, const vfloat8& b) { return a = a / b; }
+ __forceinline vfloat8& operator /=(vfloat8& a, float b) { return a = a / b; }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Comparison Operators + Select
+ ////////////////////////////////////////////////////////////////////////////////
+
+#if defined(__AVX512VL__)
+ static __forceinline vboolf8 operator ==(const vfloat8& a, const vfloat8& b) { return _mm256_cmp_ps_mask(a, b, _MM_CMPINT_EQ); }
+ static __forceinline vboolf8 operator !=(const vfloat8& a, const vfloat8& b) { return _mm256_cmp_ps_mask(a, b, _MM_CMPINT_NE); }
+ static __forceinline vboolf8 operator < (const vfloat8& a, const vfloat8& b) { return _mm256_cmp_ps_mask(a, b, _MM_CMPINT_LT); }
+ static __forceinline vboolf8 operator >=(const vfloat8& a, const vfloat8& b) { return _mm256_cmp_ps_mask(a, b, _MM_CMPINT_GE); }
+ static __forceinline vboolf8 operator > (const vfloat8& a, const vfloat8& b) { return _mm256_cmp_ps_mask(a, b, _MM_CMPINT_GT); }
+ static __forceinline vboolf8 operator <=(const vfloat8& a, const vfloat8& b) { return _mm256_cmp_ps_mask(a, b, _MM_CMPINT_LE); }
+
+ static __forceinline vfloat8 select(const vboolf8& m, const vfloat8& t, const vfloat8& f) {
+ return _mm256_mask_blend_ps(m, f, t);
+ }
+#elif !defined(__aarch64__)
+ __forceinline vboolf8 operator ==(const vfloat8& a, const vfloat8& b) { return _mm256_cmp_ps(a, b, _CMP_EQ_OQ); }
+ __forceinline vboolf8 operator !=(const vfloat8& a, const vfloat8& b) { return _mm256_cmp_ps(a, b, _CMP_NEQ_UQ); }
+ __forceinline vboolf8 operator < (const vfloat8& a, const vfloat8& b) { return _mm256_cmp_ps(a, b, _CMP_LT_OS); }
+ __forceinline vboolf8 operator >=(const vfloat8& a, const vfloat8& b) { return _mm256_cmp_ps(a, b, _CMP_NLT_US); }
+ __forceinline vboolf8 operator > (const vfloat8& a, const vfloat8& b) { return _mm256_cmp_ps(a, b, _CMP_NLE_US); }
+ __forceinline vboolf8 operator <=(const vfloat8& a, const vfloat8& b) { return _mm256_cmp_ps(a, b, _CMP_LE_OS); }
+
+ __forceinline vfloat8 select(const vboolf8& m, const vfloat8& t, const vfloat8& f) {
+ return _mm256_blendv_ps(f, t, m);
+ }
+#else
+ __forceinline vboolf8 operator ==(const vfloat8& a, const vfloat8& b) { return _mm256_cmpeq_ps(a, b); }
+ __forceinline vboolf8 operator !=(const vfloat8& a, const vfloat8& b) { return _mm256_cmpneq_ps(a, b); }
+ __forceinline vboolf8 operator < (const vfloat8& a, const vfloat8& b) { return _mm256_cmplt_ps(a, b); }
+ __forceinline vboolf8 operator >=(const vfloat8& a, const vfloat8& b) { return _mm256_cmpge_ps(a, b); }
+ __forceinline vboolf8 operator > (const vfloat8& a, const vfloat8& b) { return _mm256_cmpgt_ps(a, b); }
+ __forceinline vboolf8 operator <=(const vfloat8& a, const vfloat8& b) { return _mm256_cmple_ps(a, b); }
+
+ __forceinline vfloat8 select(const vboolf8& m, const vfloat8& t, const vfloat8& f) {
+ return _mm256_blendv_ps(f, t, m);
+ }
+
+#endif
+
+ template<int mask>
+ __forceinline vfloat8 select(const vfloat8& t, const vfloat8& f) {
+ return _mm256_blend_ps(f, t, mask);
+ }
+
+ __forceinline vboolf8 operator ==(const vfloat8& a, const float& b) { return a == vfloat8(b); }
+ __forceinline vboolf8 operator ==(const float& a, const vfloat8& b) { return vfloat8(a) == b; }
+
+ __forceinline vboolf8 operator !=(const vfloat8& a, const float& b) { return a != vfloat8(b); }
+ __forceinline vboolf8 operator !=(const float& a, const vfloat8& b) { return vfloat8(a) != b; }
+
+ __forceinline vboolf8 operator < (const vfloat8& a, const float& b) { return a < vfloat8(b); }
+ __forceinline vboolf8 operator < (const float& a, const vfloat8& b) { return vfloat8(a) < b; }
+
+ __forceinline vboolf8 operator >=(const vfloat8& a, const float& b) { return a >= vfloat8(b); }
+ __forceinline vboolf8 operator >=(const float& a, const vfloat8& b) { return vfloat8(a) >= b; }
+
+ __forceinline vboolf8 operator > (const vfloat8& a, const float& b) { return a > vfloat8(b); }
+ __forceinline vboolf8 operator > (const float& a, const vfloat8& b) { return vfloat8(a) > b; }
+
+ __forceinline vboolf8 operator <=(const vfloat8& a, const float& b) { return a <= vfloat8(b); }
+ __forceinline vboolf8 operator <=(const float& a, const vfloat8& b) { return vfloat8(a) <= b; }
+
+ __forceinline vboolf8 eq(const vfloat8& a, const vfloat8& b) { return a == b; }
+ __forceinline vboolf8 ne(const vfloat8& a, const vfloat8& b) { return a != b; }
+ __forceinline vboolf8 lt(const vfloat8& a, const vfloat8& b) { return a < b; }
+ __forceinline vboolf8 ge(const vfloat8& a, const vfloat8& b) { return a >= b; }
+ __forceinline vboolf8 gt(const vfloat8& a, const vfloat8& b) { return a > b; }
+ __forceinline vboolf8 le(const vfloat8& a, const vfloat8& b) { return a <= b; }
+
+#if defined(__AVX512VL__)
+ static __forceinline vboolf8 eq(const vboolf8& mask, const vfloat8& a, const vfloat8& b) { return _mm256_mask_cmp_ps_mask(mask, a, b, _MM_CMPINT_EQ); }
+ static __forceinline vboolf8 ne(const vboolf8& mask, const vfloat8& a, const vfloat8& b) { return _mm256_mask_cmp_ps_mask(mask, a, b, _MM_CMPINT_NE); }
+ static __forceinline vboolf8 lt(const vboolf8& mask, const vfloat8& a, const vfloat8& b) { return _mm256_mask_cmp_ps_mask(mask, a, b, _MM_CMPINT_LT); }
+ static __forceinline vboolf8 ge(const vboolf8& mask, const vfloat8& a, const vfloat8& b) { return _mm256_mask_cmp_ps_mask(mask, a, b, _MM_CMPINT_GE); }
+ static __forceinline vboolf8 gt(const vboolf8& mask, const vfloat8& a, const vfloat8& b) { return _mm256_mask_cmp_ps_mask(mask, a, b, _MM_CMPINT_GT); }
+ static __forceinline vboolf8 le(const vboolf8& mask, const vfloat8& a, const vfloat8& b) { return _mm256_mask_cmp_ps_mask(mask, a, b, _MM_CMPINT_LE); }
+#else
+ static __forceinline vboolf8 eq(const vboolf8& mask, const vfloat8& a, const vfloat8& b) { return mask & (a == b); }
+ static __forceinline vboolf8 ne(const vboolf8& mask, const vfloat8& a, const vfloat8& b) { return mask & (a != b); }
+ static __forceinline vboolf8 lt(const vboolf8& mask, const vfloat8& a, const vfloat8& b) { return mask & (a < b); }
+ static __forceinline vboolf8 ge(const vboolf8& mask, const vfloat8& a, const vfloat8& b) { return mask & (a >= b); }
+ static __forceinline vboolf8 gt(const vboolf8& mask, const vfloat8& a, const vfloat8& b) { return mask & (a > b); }
+ static __forceinline vboolf8 le(const vboolf8& mask, const vfloat8& a, const vfloat8& b) { return mask & (a <= b); }
+#endif
+
+ __forceinline vfloat8 lerp(const vfloat8& a, const vfloat8& b, const vfloat8& t) {
+ return madd(t,b-a,a);
+ }
+
+ __forceinline bool isvalid (const vfloat8& v) {
+ return all((v > vfloat8(-FLT_LARGE)) & (v < vfloat8(+FLT_LARGE)));
+ }
+
+ __forceinline bool is_finite (const vfloat8& a) {
+ return all((a >= vfloat8(-FLT_MAX)) & (a <= vfloat8(+FLT_MAX)));
+ }
+
+ __forceinline bool is_finite (const vboolf8& valid, const vfloat8& a) {
+ return all(valid, (a >= vfloat8(-FLT_MAX)) & (a <= vfloat8(+FLT_MAX)));
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Rounding Functions
+ ////////////////////////////////////////////////////////////////////////////////
+
+#if !defined(__aarch64__)
+ __forceinline vfloat8 floor(const vfloat8& a) { return _mm256_round_ps(a, _MM_FROUND_TO_NEG_INF ); }
+ __forceinline vfloat8 ceil (const vfloat8& a) { return _mm256_round_ps(a, _MM_FROUND_TO_POS_INF ); }
+ __forceinline vfloat8 trunc(const vfloat8& a) { return _mm256_round_ps(a, _MM_FROUND_TO_ZERO ); }
+ __forceinline vfloat8 round(const vfloat8& a) { return _mm256_round_ps(a, _MM_FROUND_TO_NEAREST_INT); }
+#else
+ __forceinline vfloat8 floor(const vfloat8& a) { return _mm256_floor_ps(a); }
+ __forceinline vfloat8 ceil (const vfloat8& a) { return _mm256_ceil_ps(a); }
+#endif
+
+
+ __forceinline vfloat8 frac (const vfloat8& a) { return a-floor(a); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Movement/Shifting/Shuffling Functions
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vfloat8 unpacklo(const vfloat8& a, const vfloat8& b) { return _mm256_unpacklo_ps(a, b); }
+ __forceinline vfloat8 unpackhi(const vfloat8& a, const vfloat8& b) { return _mm256_unpackhi_ps(a, b); }
+
+ template<int i>
+ __forceinline vfloat8 shuffle(const vfloat8& v) {
+ return _mm256_permute_ps(v, _MM_SHUFFLE(i, i, i, i));
+ }
+
+ template<int i0, int i1>
+ __forceinline vfloat8 shuffle4(const vfloat8& v) {
+ return _mm256_permute2f128_ps(v, v, (i1 << 4) | (i0 << 0));
+ }
+
+ template<int i0, int i1>
+ __forceinline vfloat8 shuffle4(const vfloat8& a, const vfloat8& b) {
+ return _mm256_permute2f128_ps(a, b, (i1 << 4) | (i0 << 0));
+ }
+
+ template<int i0, int i1, int i2, int i3>
+ __forceinline vfloat8 shuffle(const vfloat8& v) {
+ return _mm256_permute_ps(v, _MM_SHUFFLE(i3, i2, i1, i0));
+ }
+
+ template<int i0, int i1, int i2, int i3>
+ __forceinline vfloat8 shuffle(const vfloat8& a, const vfloat8& b) {
+ return _mm256_shuffle_ps(a, b, _MM_SHUFFLE(i3, i2, i1, i0));
+ }
+
+#if !defined(__aarch64__)
+ template<> __forceinline vfloat8 shuffle<0, 0, 2, 2>(const vfloat8& v) { return _mm256_moveldup_ps(v); }
+ template<> __forceinline vfloat8 shuffle<1, 1, 3, 3>(const vfloat8& v) { return _mm256_movehdup_ps(v); }
+ template<> __forceinline vfloat8 shuffle<0, 1, 0, 1>(const vfloat8& v) { return _mm256_castpd_ps(_mm256_movedup_pd(_mm256_castps_pd(v))); }
+#endif
+
+ __forceinline vfloat8 broadcast(const float* ptr) { return _mm256_broadcast_ss(ptr); }
+ template<size_t i> __forceinline vfloat8 insert4(const vfloat8& a, const vfloat4& b) { return _mm256_insertf128_ps(a, b, i); }
+ template<size_t i> __forceinline vfloat4 extract4 (const vfloat8& a) { return _mm256_extractf128_ps(a, i); }
+ template<> __forceinline vfloat4 extract4<0>(const vfloat8& a) { return _mm256_castps256_ps128(a); }
+
+ __forceinline float toScalar(const vfloat8& v) { return _mm_cvtss_f32(_mm256_castps256_ps128(v)); }
+
+ __forceinline vfloat8 assign(const vfloat4& a) { return _mm256_castps128_ps256(a); }
+
+#if defined (__AVX2__) && !defined(__aarch64__)
+ __forceinline vfloat8 permute(const vfloat8& a, const __m256i& index) {
+ return _mm256_permutevar8x32_ps(a, index);
+ }
+#endif
+
+#if defined(__AVX512VL__)
+ template<int i>
+ static __forceinline vfloat8 align_shift_right(const vfloat8& a, const vfloat8& b) {
+ return _mm256_castsi256_ps(_mm256_alignr_epi32(_mm256_castps_si256(a), _mm256_castps_si256(b), i));
+ }
+#endif
+
+#if defined (__AVX_I__)
+ template<const int mode>
+ static __forceinline vint4 convert_to_hf16(const vfloat8& a) {
+ return _mm256_cvtps_ph(a, mode);
+ }
+
+ static __forceinline vfloat8 convert_from_hf16(const vint4& a) {
+ return _mm256_cvtph_ps(a);
+ }
+#endif
+
+ __forceinline vfloat4 broadcast4f(const vfloat8& a, const size_t k) {
+ return vfloat4::broadcast(&a[k]);
+ }
+
+ __forceinline vfloat8 broadcast8f(const vfloat8& a, const size_t k) {
+ return vfloat8::broadcast(&a[k]);
+ }
+
+#if defined(__AVX512VL__)
+ static __forceinline vfloat8 shift_right_1(const vfloat8& x) {
+ return align_shift_right<1>(zero,x);
+ }
+#else
+ static __forceinline vfloat8 shift_right_1(const vfloat8& x) {
+ const vfloat8 t0 = shuffle<1,2,3,0>(x);
+ const vfloat8 t1 = shuffle4<1,0>(t0);
+ return _mm256_blend_ps(t0,t1,0x88);
+ }
+#endif
+
+ __forceinline vint8 floori(const vfloat8& a) {
+ return vint8(floor(a));
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Transpose
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline void transpose(const vfloat8& r0, const vfloat8& r1, const vfloat8& r2, const vfloat8& r3, vfloat8& c0, vfloat8& c1, vfloat8& c2, vfloat8& c3)
+ {
+ vfloat8 l02 = unpacklo(r0,r2);
+ vfloat8 h02 = unpackhi(r0,r2);
+ vfloat8 l13 = unpacklo(r1,r3);
+ vfloat8 h13 = unpackhi(r1,r3);
+ c0 = unpacklo(l02,l13);
+ c1 = unpackhi(l02,l13);
+ c2 = unpacklo(h02,h13);
+ c3 = unpackhi(h02,h13);
+ }
+
+ __forceinline void transpose(const vfloat8& r0, const vfloat8& r1, const vfloat8& r2, const vfloat8& r3, vfloat8& c0, vfloat8& c1, vfloat8& c2)
+ {
+ vfloat8 l02 = unpacklo(r0,r2);
+ vfloat8 h02 = unpackhi(r0,r2);
+ vfloat8 l13 = unpacklo(r1,r3);
+ vfloat8 h13 = unpackhi(r1,r3);
+ c0 = unpacklo(l02,l13);
+ c1 = unpackhi(l02,l13);
+ c2 = unpacklo(h02,h13);
+ }
+
+ __forceinline void transpose(const vfloat8& r0, const vfloat8& r1, const vfloat8& r2, const vfloat8& r3, const vfloat8& r4, const vfloat8& r5, const vfloat8& r6, const vfloat8& r7,
+ vfloat8& c0, vfloat8& c1, vfloat8& c2, vfloat8& c3, vfloat8& c4, vfloat8& c5, vfloat8& c6, vfloat8& c7)
+ {
+ vfloat8 h0,h1,h2,h3; transpose(r0,r1,r2,r3,h0,h1,h2,h3);
+ vfloat8 h4,h5,h6,h7; transpose(r4,r5,r6,r7,h4,h5,h6,h7);
+ c0 = shuffle4<0,2>(h0,h4);
+ c1 = shuffle4<0,2>(h1,h5);
+ c2 = shuffle4<0,2>(h2,h6);
+ c3 = shuffle4<0,2>(h3,h7);
+ c4 = shuffle4<1,3>(h0,h4);
+ c5 = shuffle4<1,3>(h1,h5);
+ c6 = shuffle4<1,3>(h2,h6);
+ c7 = shuffle4<1,3>(h3,h7);
+ }
+
+ __forceinline void transpose(const vfloat4& r0, const vfloat4& r1, const vfloat4& r2, const vfloat4& r3, const vfloat4& r4, const vfloat4& r5, const vfloat4& r6, const vfloat4& r7,
+ vfloat8& c0, vfloat8& c1, vfloat8& c2, vfloat8& c3)
+ {
+ transpose(vfloat8(r0,r4), vfloat8(r1,r5), vfloat8(r2,r6), vfloat8(r3,r7), c0, c1, c2, c3);
+ }
+
+ __forceinline void transpose(const vfloat4& r0, const vfloat4& r1, const vfloat4& r2, const vfloat4& r3, const vfloat4& r4, const vfloat4& r5, const vfloat4& r6, const vfloat4& r7,
+ vfloat8& c0, vfloat8& c1, vfloat8& c2)
+ {
+ transpose(vfloat8(r0,r4), vfloat8(r1,r5), vfloat8(r2,r6), vfloat8(r3,r7), c0, c1, c2);
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Reductions
+ ////////////////////////////////////////////////////////////////////////////////
+#if !defined(__aarch64__)
+ __forceinline vfloat8 vreduce_min2(const vfloat8& v) { return min(v,shuffle<1,0,3,2>(v)); }
+ __forceinline vfloat8 vreduce_min4(const vfloat8& v) { vfloat8 v1 = vreduce_min2(v); return min(v1,shuffle<2,3,0,1>(v1)); }
+ __forceinline vfloat8 vreduce_min (const vfloat8& v) { vfloat8 v1 = vreduce_min4(v); return min(v1,shuffle4<1,0>(v1)); }
+
+ __forceinline vfloat8 vreduce_max2(const vfloat8& v) { return max(v,shuffle<1,0,3,2>(v)); }
+ __forceinline vfloat8 vreduce_max4(const vfloat8& v) { vfloat8 v1 = vreduce_max2(v); return max(v1,shuffle<2,3,0,1>(v1)); }
+ __forceinline vfloat8 vreduce_max (const vfloat8& v) { vfloat8 v1 = vreduce_max4(v); return max(v1,shuffle4<1,0>(v1)); }
+
+ __forceinline vfloat8 vreduce_add2(const vfloat8& v) { return v + shuffle<1,0,3,2>(v); }
+ __forceinline vfloat8 vreduce_add4(const vfloat8& v) { vfloat8 v1 = vreduce_add2(v); return v1 + shuffle<2,3,0,1>(v1); }
+ __forceinline vfloat8 vreduce_add (const vfloat8& v) { vfloat8 v1 = vreduce_add4(v); return v1 + shuffle4<1,0>(v1); }
+
+ __forceinline float reduce_min(const vfloat8& v) { return toScalar(vreduce_min(v)); }
+ __forceinline float reduce_max(const vfloat8& v) { return toScalar(vreduce_max(v)); }
+ __forceinline float reduce_add(const vfloat8& v) { return toScalar(vreduce_add(v)); }
+#else
+ __forceinline float reduce_min(const vfloat8& v) { return vminvq_f32(_mm_min_ps(v.v.lo,v.v.hi)); }
+ __forceinline float reduce_max(const vfloat8& v) { return vmaxvq_f32(_mm_max_ps(v.v.lo,v.v.hi)); }
+ __forceinline vfloat8 vreduce_min(const vfloat8& v) { return vfloat8(reduce_min(v)); }
+ __forceinline vfloat8 vreduce_max(const vfloat8& v) { return vfloat8(reduce_max(v)); }
+ __forceinline float reduce_add(const vfloat8& v) { return vaddvq_f32(_mm_add_ps(v.v.lo,v.v.hi)); }
+
+#endif
+ __forceinline size_t select_min(const vboolf8& valid, const vfloat8& v)
+ {
+ const vfloat8 a = select(valid,v,vfloat8(pos_inf));
+ const vbool8 valid_min = valid & (a == vreduce_min(a));
+ return bsf(movemask(any(valid_min) ? valid_min : valid));
+ }
+
+ __forceinline size_t select_max(const vboolf8& valid, const vfloat8& v)
+ {
+ const vfloat8 a = select(valid,v,vfloat8(neg_inf));
+ const vbool8 valid_max = valid & (a == vreduce_max(a));
+ return bsf(movemask(any(valid_max) ? valid_max : valid));
+ }
+
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Euclidian Space Operators (pairs of Vec3fa's)
+ ////////////////////////////////////////////////////////////////////////////////
+
+ //__forceinline vfloat8 dot(const vfloat8& a, const vfloat8& b) {
+ // return vreduce_add4(a*b);
+ //}
+
+ __forceinline vfloat8 dot(const vfloat8& a, const vfloat8& b) {
+ return _mm256_dp_ps(a,b,0x7F);
+ }
+
+ __forceinline vfloat8 cross(const vfloat8& a, const vfloat8& b)
+ {
+ const vfloat8 a0 = a;
+ const vfloat8 b0 = shuffle<1,2,0,3>(b);
+ const vfloat8 a1 = shuffle<1,2,0,3>(a);
+ const vfloat8 b1 = b;
+ return shuffle<1,2,0,3>(msub(a0,b0,a1*b1));
+ }
+
+ //__forceinline float sqr_length (const vfloat<8>& a) { return dot(a,a); }
+ //__forceinline float rcp_length (const vfloat<8>& a) { return rsqrt(dot(a,a)); }
+ //__forceinline float rcp_length2(const vfloat<8>& a) { return rcp(dot(a,a)); }
+ //__forceinline float length (const vfloat<8>& a) { return sqrt(dot(a,a)); }
+ __forceinline vfloat<8> normalize(const vfloat<8>& a) { return a*rsqrt(dot(a,a)); }
+ //__forceinline float distance(const vfloat<8>& a, const vfloat<8>& b) { return length(a-b); }
+ //__forceinline float halfArea(const vfloat<8>& d) { return madd(d.x,(d.y+d.z),d.y*d.z); }
+ //__forceinline float area (const vfloat<8>& d) { return 2.0f*halfArea(d); }
+ //__forceinline vfloat<8> reflect(const vfloat<8>& V, const vfloat<8>& N) { return 2.0f*dot(V,N)*N-V; }
+
+ //__forceinline vfloat<8> normalize_safe(const vfloat<8>& a) {
+ // const float d = dot(a,a); if (unlikely(d == 0.0f)) return a; else return a*rsqrt(d);
+ //}
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// In Register Sorting
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vfloat8 sort_ascending(const vfloat8& v)
+ {
+ const vfloat8 a0 = v;
+ const vfloat8 b0 = shuffle<1,0,3,2>(a0);
+ const vfloat8 c0 = min(a0,b0);
+ const vfloat8 d0 = max(a0,b0);
+ const vfloat8 a1 = select<0x99 /* 0b10011001 */>(c0,d0);
+ const vfloat8 b1 = shuffle<2,3,0,1>(a1);
+ const vfloat8 c1 = min(a1,b1);
+ const vfloat8 d1 = max(a1,b1);
+ const vfloat8 a2 = select<0xc3 /* 0b11000011 */>(c1,d1);
+ const vfloat8 b2 = shuffle<1,0,3,2>(a2);
+ const vfloat8 c2 = min(a2,b2);
+ const vfloat8 d2 = max(a2,b2);
+ const vfloat8 a3 = select<0xa5 /* 0b10100101 */>(c2,d2);
+ const vfloat8 b3 = shuffle4<1,0>(a3);
+ const vfloat8 c3 = min(a3,b3);
+ const vfloat8 d3 = max(a3,b3);
+ const vfloat8 a4 = select<0xf /* 0b00001111 */>(c3,d3);
+ const vfloat8 b4 = shuffle<2,3,0,1>(a4);
+ const vfloat8 c4 = min(a4,b4);
+ const vfloat8 d4 = max(a4,b4);
+ const vfloat8 a5 = select<0x33 /* 0b00110011 */>(c4,d4);
+ const vfloat8 b5 = shuffle<1,0,3,2>(a5);
+ const vfloat8 c5 = min(a5,b5);
+ const vfloat8 d5 = max(a5,b5);
+ const vfloat8 a6 = select<0x55 /* 0b01010101 */>(c5,d5);
+ return a6;
+ }
+
+ __forceinline vfloat8 sort_descending(const vfloat8& v)
+ {
+ const vfloat8 a0 = v;
+ const vfloat8 b0 = shuffle<1,0,3,2>(a0);
+ const vfloat8 c0 = max(a0,b0);
+ const vfloat8 d0 = min(a0,b0);
+ const vfloat8 a1 = select<0x99 /* 0b10011001 */>(c0,d0);
+ const vfloat8 b1 = shuffle<2,3,0,1>(a1);
+ const vfloat8 c1 = max(a1,b1);
+ const vfloat8 d1 = min(a1,b1);
+ const vfloat8 a2 = select<0xc3 /* 0b11000011 */>(c1,d1);
+ const vfloat8 b2 = shuffle<1,0,3,2>(a2);
+ const vfloat8 c2 = max(a2,b2);
+ const vfloat8 d2 = min(a2,b2);
+ const vfloat8 a3 = select<0xa5 /* 0b10100101 */>(c2,d2);
+ const vfloat8 b3 = shuffle4<1,0>(a3);
+ const vfloat8 c3 = max(a3,b3);
+ const vfloat8 d3 = min(a3,b3);
+ const vfloat8 a4 = select<0xf /* 0b00001111 */>(c3,d3);
+ const vfloat8 b4 = shuffle<2,3,0,1>(a4);
+ const vfloat8 c4 = max(a4,b4);
+ const vfloat8 d4 = min(a4,b4);
+ const vfloat8 a5 = select<0x33 /* 0b00110011 */>(c4,d4);
+ const vfloat8 b5 = shuffle<1,0,3,2>(a5);
+ const vfloat8 c5 = max(a5,b5);
+ const vfloat8 d5 = min(a5,b5);
+ const vfloat8 a6 = select<0x55 /* 0b01010101 */>(c5,d5);
+ return a6;
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Output Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline embree_ostream operator <<(embree_ostream cout, const vfloat8& a) {
+ return cout << "<" << a[0] << ", " << a[1] << ", " << a[2] << ", " << a[3] << ", " << a[4] << ", " << a[5] << ", " << a[6] << ", " << a[7] << ">";
+ }
+}
diff --git a/thirdparty/embree-aarch64/common/simd/vint16_avx512.h b/thirdparty/embree-aarch64/common/simd/vint16_avx512.h
new file mode 100644
index 0000000000..3249bc2b45
--- /dev/null
+++ b/thirdparty/embree-aarch64/common/simd/vint16_avx512.h
@@ -0,0 +1,490 @@
+// Copyright 2009-2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+namespace embree
+{
+ /* 16-wide AVX-512 integer type */
+ template<>
+ struct vint<16>
+ {
+ ALIGNED_STRUCT_(64);
+
+ typedef vboolf16 Bool;
+ typedef vint16 Int;
+ typedef vfloat16 Float;
+
+ enum { size = 16 }; // number of SIMD elements
+ union { // data
+ __m512i v;
+ int i[16];
+ };
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constructors, Assignment & Cast Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vint() {}
+ __forceinline vint(const vint16& t) { v = t.v; }
+ __forceinline vint16& operator =(const vint16& f) { v = f.v; return *this; }
+
+ __forceinline vint(const __m512i& t) { v = t; }
+ __forceinline operator __m512i() const { return v; }
+ __forceinline operator __m256i() const { return _mm512_castsi512_si256(v); }
+
+ __forceinline vint(int i) {
+ v = _mm512_set1_epi32(i);
+ }
+
+ __forceinline vint(int a, int b, int c, int d) {
+ v = _mm512_set4_epi32(d,c,b,a);
+ }
+
+ __forceinline vint(int a0 , int a1 , int a2 , int a3,
+ int a4 , int a5 , int a6 , int a7,
+ int a8 , int a9 , int a10, int a11,
+ int a12, int a13, int a14, int a15)
+ {
+ v = _mm512_set_epi32(a15,a14,a13,a12,a11,a10,a9,a8,a7,a6,a5,a4,a3,a2,a1,a0);
+ }
+
+ __forceinline vint(const vint4& i) {
+ v = _mm512_broadcast_i32x4(i);
+ }
+
+ __forceinline vint(const vint4& a, const vint4& b, const vint4& c, const vint4& d) {
+ v = _mm512_castsi128_si512(a);
+ v = _mm512_inserti32x4(v, b, 1);
+ v = _mm512_inserti32x4(v, c, 2);
+ v = _mm512_inserti32x4(v, d, 3);
+ }
+
+ __forceinline vint(const vint8& i) {
+ v = _mm512_castps_si512(_mm512_castpd_ps(_mm512_broadcast_f64x4(_mm256_castsi256_pd(i))));
+ }
+
+ __forceinline vint(const vint8& a, const vint8& b) {
+ v = _mm512_castsi256_si512(a);
+ v = _mm512_inserti64x4(v, b, 1);
+ }
+
+ __forceinline explicit vint(const __m512& f) {
+ v = _mm512_cvtps_epi32(f);
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constants
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vint(ZeroTy) : v(_mm512_setzero_epi32()) {}
+ __forceinline vint(OneTy) : v(_mm512_set1_epi32(1)) {}
+ __forceinline vint(PosInfTy) : v(_mm512_set1_epi32(pos_inf)) {}
+ __forceinline vint(NegInfTy) : v(_mm512_set1_epi32(neg_inf)) {}
+ __forceinline vint(StepTy) : v(_mm512_set_epi32(15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0)) {}
+ __forceinline vint(ReverseStepTy) : v(_mm512_setr_epi32(15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0)) {}
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Loads and Stores
+ ////////////////////////////////////////////////////////////////////////////////
+
+ static __forceinline vint16 load (const void* addr) { return _mm512_load_si512((int*)addr); }
+
+ static __forceinline vint16 load(const uint8_t* ptr) { return _mm512_cvtepu8_epi32(_mm_load_si128((__m128i*)ptr)); }
+ static __forceinline vint16 load(const unsigned short* ptr) { return _mm512_cvtepu16_epi32(_mm256_load_si256((__m256i*)ptr)); }
+
+ static __forceinline vint16 loadu(const uint8_t* ptr) { return _mm512_cvtepu8_epi32(_mm_loadu_si128((__m128i*)ptr)); }
+ static __forceinline vint16 loadu(const unsigned short* ptr) { return _mm512_cvtepu16_epi32(_mm256_loadu_si256((__m256i*)ptr)); }
+
+ static __forceinline vint16 loadu(const void* addr) { return _mm512_loadu_si512(addr); }
+
+ static __forceinline vint16 load (const vboolf16& mask, const void* addr) { return _mm512_mask_load_epi32 (_mm512_setzero_epi32(),mask,addr); }
+ static __forceinline vint16 loadu(const vboolf16& mask, const void* addr) { return _mm512_mask_loadu_epi32(_mm512_setzero_epi32(),mask,addr); }
+
+ static __forceinline void store (void* ptr, const vint16& v) { _mm512_store_si512 (ptr,v); }
+ static __forceinline void storeu(void* ptr, const vint16& v) { _mm512_storeu_si512(ptr,v); }
+
+ static __forceinline void store (const vboolf16& mask, void* addr, const vint16& v2) { _mm512_mask_store_epi32(addr,mask,v2); }
+ static __forceinline void storeu(const vboolf16& mask, void* ptr, const vint16& f) { _mm512_mask_storeu_epi32((int*)ptr,mask,f); }
+
+ static __forceinline void store_nt(void* __restrict__ ptr, const vint16& a) { _mm512_stream_si512((__m512i*)ptr,a); }
+
+ /* pass by value to avoid compiler generating inefficient code */
+ static __forceinline void storeu_compact(const vboolf16 mask, void* addr, vint16 reg) {
+ _mm512_mask_compressstoreu_epi32(addr,mask,reg);
+ }
+
+ static __forceinline void storeu_compact_single(const vboolf16 mask, void* addr, vint16 reg) {
+ //_mm512_mask_compressstoreu_epi32(addr,mask,reg);
+ *(float*)addr = mm512_cvtss_f32(_mm512_mask_compress_ps(_mm512_castsi512_ps(reg),mask,_mm512_castsi512_ps(reg)));
+ }
+
+ static __forceinline vint16 compact64bit(const vboolf16& mask, vint16 &v) {
+ return _mm512_mask_compress_epi64(v,mask,v);
+ }
+
+ static __forceinline vint16 compact(const vboolf16& mask, vint16 &v) {
+ return _mm512_mask_compress_epi32(v,mask,v);
+ }
+
+ static __forceinline vint16 compact(const vboolf16& mask, const vint16 &a, vint16 &b) {
+ return _mm512_mask_compress_epi32(a,mask,b);
+ }
+
+ static __forceinline vint16 expand(const vboolf16& mask, const vint16& a, vint16& b) {
+ return _mm512_mask_expand_epi32(b,mask,a);
+ }
+
+ template<int scale = 4>
+ static __forceinline vint16 gather(const int* ptr, const vint16& index) {
+ return _mm512_i32gather_epi32(index,ptr,scale);
+ }
+
+ template<int scale = 4>
+ static __forceinline vint16 gather(const vboolf16& mask, const int* ptr, const vint16& index) {
+ return _mm512_mask_i32gather_epi32(_mm512_undefined_epi32(),mask,index,ptr,scale);
+ }
+
+ template<int scale = 4>
+ static __forceinline vint16 gather(const vboolf16& mask, vint16& dest, const int* ptr, const vint16& index) {
+ return _mm512_mask_i32gather_epi32(dest,mask,index,ptr,scale);
+ }
+
+ template<int scale = 4>
+ static __forceinline void scatter(int* ptr, const vint16& index, const vint16& v) {
+ _mm512_i32scatter_epi32((int*)ptr,index,v,scale);
+ }
+
+ template<int scale = 4>
+ static __forceinline void scatter(const vboolf16& mask, int* ptr, const vint16& index, const vint16& v) {
+ _mm512_mask_i32scatter_epi32((int*)ptr,mask,index,v,scale);
+ }
+
+ static __forceinline vint16 broadcast64bit(size_t v) {
+ return _mm512_set1_epi64(v);
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Array Access
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline int& operator [](size_t index) { assert(index < 16); return i[index]; }
+ __forceinline const int& operator [](size_t index) const { assert(index < 16); return i[index]; }
+
+ __forceinline unsigned int uint (size_t index) const { assert(index < 16); return ((unsigned int*)i)[index]; }
+ __forceinline size_t& uint64_t(size_t index) const { assert(index < 8); return ((size_t*)i)[index]; }
+ };
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Unary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboolf16 asBool(const vint16& a) { return _mm512_movepi32_mask(a); }
+
+ __forceinline vint16 operator +(const vint16& a) { return a; }
+ __forceinline vint16 operator -(const vint16& a) { return _mm512_sub_epi32(_mm512_setzero_epi32(), a); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Binary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vint16 operator +(const vint16& a, const vint16& b) { return _mm512_add_epi32(a, b); }
+ __forceinline vint16 operator +(const vint16& a, int b) { return a + vint16(b); }
+ __forceinline vint16 operator +(int a, const vint16& b) { return vint16(a) + b; }
+
+ __forceinline vint16 operator -(const vint16& a, const vint16& b) { return _mm512_sub_epi32(a, b); }
+ __forceinline vint16 operator -(const vint16& a, int b) { return a - vint16(b); }
+ __forceinline vint16 operator -(int a, const vint16& b) { return vint16(a) - b; }
+
+ __forceinline vint16 operator *(const vint16& a, const vint16& b) { return _mm512_mullo_epi32(a, b); }
+ __forceinline vint16 operator *(const vint16& a, int b) { return a * vint16(b); }
+ __forceinline vint16 operator *(int a, const vint16& b) { return vint16(a) * b; }
+
+ __forceinline vint16 operator &(const vint16& a, const vint16& b) { return _mm512_and_epi32(a, b); }
+ __forceinline vint16 operator &(const vint16& a, int b) { return a & vint16(b); }
+ __forceinline vint16 operator &(int a, const vint16& b) { return vint16(a) & b; }
+
+ __forceinline vint16 operator |(const vint16& a, const vint16& b) { return _mm512_or_epi32(a, b); }
+ __forceinline vint16 operator |(const vint16& a, int b) { return a | vint16(b); }
+ __forceinline vint16 operator |(int a, const vint16& b) { return vint16(a) | b; }
+
+ __forceinline vint16 operator ^(const vint16& a, const vint16& b) { return _mm512_xor_epi32(a, b); }
+ __forceinline vint16 operator ^(const vint16& a, int b) { return a ^ vint16(b); }
+ __forceinline vint16 operator ^(int a, const vint16& b) { return vint16(a) ^ b; }
+
+ __forceinline vint16 operator <<(const vint16& a, int n) { return _mm512_slli_epi32(a, n); }
+ __forceinline vint16 operator >>(const vint16& a, int n) { return _mm512_srai_epi32(a, n); }
+
+ __forceinline vint16 operator <<(const vint16& a, const vint16& n) { return _mm512_sllv_epi32(a, n); }
+ __forceinline vint16 operator >>(const vint16& a, const vint16& n) { return _mm512_srav_epi32(a, n); }
+
+ __forceinline vint16 sll (const vint16& a, int b) { return _mm512_slli_epi32(a, b); }
+ __forceinline vint16 sra (const vint16& a, int b) { return _mm512_srai_epi32(a, b); }
+ __forceinline vint16 srl (const vint16& a, int b) { return _mm512_srli_epi32(a, b); }
+
+ __forceinline vint16 min(const vint16& a, const vint16& b) { return _mm512_min_epi32(a, b); }
+ __forceinline vint16 min(const vint16& a, int b) { return min(a,vint16(b)); }
+ __forceinline vint16 min(int a, const vint16& b) { return min(vint16(a),b); }
+
+ __forceinline vint16 max(const vint16& a, const vint16& b) { return _mm512_max_epi32(a, b); }
+ __forceinline vint16 max(const vint16& a, int b) { return max(a,vint16(b)); }
+ __forceinline vint16 max(int a, const vint16& b) { return max(vint16(a),b); }
+
+ __forceinline vint16 umin(const vint16& a, const vint16& b) { return _mm512_min_epu32(a, b); }
+ __forceinline vint16 umax(const vint16& a, const vint16& b) { return _mm512_max_epu32(a, b); }
+
+ __forceinline vint16 mask_add(const vboolf16& mask, vint16& c, const vint16& a, const vint16& b) { return _mm512_mask_add_epi32(c,mask,a,b); }
+ __forceinline vint16 mask_sub(const vboolf16& mask, vint16& c, const vint16& a, const vint16& b) { return _mm512_mask_sub_epi32(c,mask,a,b); }
+
+ __forceinline vint16 mask_and(const vboolf16& m, vint16& c, const vint16& a, const vint16& b) { return _mm512_mask_and_epi32(c,m,a,b); }
+ __forceinline vint16 mask_or (const vboolf16& m, vint16& c, const vint16& a, const vint16& b) { return _mm512_mask_or_epi32(c,m,a,b); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Assignment Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vint16& operator +=(vint16& a, const vint16& b) { return a = a + b; }
+ __forceinline vint16& operator +=(vint16& a, int b) { return a = a + b; }
+
+ __forceinline vint16& operator -=(vint16& a, const vint16& b) { return a = a - b; }
+ __forceinline vint16& operator -=(vint16& a, int b) { return a = a - b; }
+
+ __forceinline vint16& operator *=(vint16& a, const vint16& b) { return a = a * b; }
+ __forceinline vint16& operator *=(vint16& a, int b) { return a = a * b; }
+
+ __forceinline vint16& operator &=(vint16& a, const vint16& b) { return a = a & b; }
+ __forceinline vint16& operator &=(vint16& a, int b) { return a = a & b; }
+
+ __forceinline vint16& operator |=(vint16& a, const vint16& b) { return a = a | b; }
+ __forceinline vint16& operator |=(vint16& a, int b) { return a = a | b; }
+
+ __forceinline vint16& operator <<=(vint16& a, int b) { return a = a << b; }
+ __forceinline vint16& operator >>=(vint16& a, int b) { return a = a >> b; }
+
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Comparison Operators + Select
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboolf16 operator ==(const vint16& a, const vint16& b) { return _mm512_cmp_epi32_mask(a,b,_MM_CMPINT_EQ); }
+ __forceinline vboolf16 operator ==(const vint16& a, int b) { return a == vint16(b); }
+ __forceinline vboolf16 operator ==(int a, const vint16& b) { return vint16(a) == b; }
+
+ __forceinline vboolf16 operator !=(const vint16& a, const vint16& b) { return _mm512_cmp_epi32_mask(a,b,_MM_CMPINT_NE); }
+ __forceinline vboolf16 operator !=(const vint16& a, int b) { return a != vint16(b); }
+ __forceinline vboolf16 operator !=(int a, const vint16& b) { return vint16(a) != b; }
+
+ __forceinline vboolf16 operator < (const vint16& a, const vint16& b) { return _mm512_cmp_epi32_mask(a,b,_MM_CMPINT_LT); }
+ __forceinline vboolf16 operator < (const vint16& a, int b) { return a < vint16(b); }
+ __forceinline vboolf16 operator < (int a, const vint16& b) { return vint16(a) < b; }
+
+ __forceinline vboolf16 operator >=(const vint16& a, const vint16& b) { return _mm512_cmp_epi32_mask(a,b,_MM_CMPINT_GE); }
+ __forceinline vboolf16 operator >=(const vint16& a, int b) { return a >= vint16(b); }
+ __forceinline vboolf16 operator >=(int a, const vint16& b) { return vint16(a) >= b; }
+
+ __forceinline vboolf16 operator > (const vint16& a, const vint16& b) { return _mm512_cmp_epi32_mask(a,b,_MM_CMPINT_GT); }
+ __forceinline vboolf16 operator > (const vint16& a, int b) { return a > vint16(b); }
+ __forceinline vboolf16 operator > (int a, const vint16& b) { return vint16(a) > b; }
+
+ __forceinline vboolf16 operator <=(const vint16& a, const vint16& b) { return _mm512_cmp_epi32_mask(a,b,_MM_CMPINT_LE); }
+ __forceinline vboolf16 operator <=(const vint16& a, int b) { return a <= vint16(b); }
+ __forceinline vboolf16 operator <=(int a, const vint16& b) { return vint16(a) <= b; }
+
+ __forceinline vboolf16 eq(const vint16& a, const vint16& b) { return _mm512_cmp_epi32_mask(a,b,_MM_CMPINT_EQ); }
+ __forceinline vboolf16 ne(const vint16& a, const vint16& b) { return _mm512_cmp_epi32_mask(a,b,_MM_CMPINT_NE); }
+ __forceinline vboolf16 lt(const vint16& a, const vint16& b) { return _mm512_cmp_epi32_mask(a,b,_MM_CMPINT_LT); }
+ __forceinline vboolf16 ge(const vint16& a, const vint16& b) { return _mm512_cmp_epi32_mask(a,b,_MM_CMPINT_GE); }
+ __forceinline vboolf16 gt(const vint16& a, const vint16& b) { return _mm512_cmp_epi32_mask(a,b,_MM_CMPINT_GT); }
+ __forceinline vboolf16 le(const vint16& a, const vint16& b) { return _mm512_cmp_epi32_mask(a,b,_MM_CMPINT_LE); }
+ __forceinline vboolf16 uint_le(const vint16& a, const vint16& b) { return _mm512_cmp_epu32_mask(a,b,_MM_CMPINT_LE); }
+ __forceinline vboolf16 uint_gt(const vint16& a, const vint16& b) { return _mm512_cmp_epu32_mask(a,b,_MM_CMPINT_GT); }
+
+ __forceinline vboolf16 eq(const vboolf16 mask, const vint16& a, const vint16& b) { return _mm512_mask_cmp_epi32_mask(mask,a,b,_MM_CMPINT_EQ); }
+ __forceinline vboolf16 ne(const vboolf16 mask, const vint16& a, const vint16& b) { return _mm512_mask_cmp_epi32_mask(mask,a,b,_MM_CMPINT_NE); }
+ __forceinline vboolf16 lt(const vboolf16 mask, const vint16& a, const vint16& b) { return _mm512_mask_cmp_epi32_mask(mask,a,b,_MM_CMPINT_LT); }
+ __forceinline vboolf16 ge(const vboolf16 mask, const vint16& a, const vint16& b) { return _mm512_mask_cmp_epi32_mask(mask,a,b,_MM_CMPINT_GE); }
+ __forceinline vboolf16 gt(const vboolf16 mask, const vint16& a, const vint16& b) { return _mm512_mask_cmp_epi32_mask(mask,a,b,_MM_CMPINT_GT); }
+ __forceinline vboolf16 le(const vboolf16 mask, const vint16& a, const vint16& b) { return _mm512_mask_cmp_epi32_mask(mask,a,b,_MM_CMPINT_LE); }
+ __forceinline vboolf16 uint_le(const vboolf16 mask, const vint16& a, const vint16& b) { return _mm512_mask_cmp_epu32_mask(mask,a,b,_MM_CMPINT_LE); }
+ __forceinline vboolf16 uint_gt(const vboolf16 mask, const vint16& a, const vint16& b) { return _mm512_mask_cmp_epu32_mask(mask,a,b,_MM_CMPINT_GT); }
+
+
+ __forceinline vint16 select(const vboolf16& m, const vint16& t, const vint16& f) {
+ return _mm512_mask_or_epi32(f,m,t,t);
+ }
+
+ __forceinline void xchg(const vboolf16& m, vint16& a, vint16& b) {
+ const vint16 c = a; a = select(m,b,a); b = select(m,c,b);
+ }
+
+ __forceinline vboolf16 test(const vboolf16& m, const vint16& a, const vint16& b) {
+ return _mm512_mask_test_epi32_mask(m,a,b);
+ }
+
+ __forceinline vboolf16 test(const vint16& a, const vint16& b) {
+ return _mm512_test_epi32_mask(a,b);
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ // Movement/Shifting/Shuffling Functions
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vint16 unpacklo(const vint16& a, const vint16& b) { return _mm512_unpacklo_epi32(a, b); }
+ __forceinline vint16 unpackhi(const vint16& a, const vint16& b) { return _mm512_unpackhi_epi32(a, b); }
+
+ template<int i>
+ __forceinline vint16 shuffle(const vint16& v) {
+ return _mm512_castps_si512(_mm512_permute_ps(_mm512_castsi512_ps(v), _MM_SHUFFLE(i, i, i, i)));
+ }
+
+ template<int i0, int i1, int i2, int i3>
+ __forceinline vint16 shuffle(const vint16& v) {
+ return _mm512_castps_si512(_mm512_permute_ps(_mm512_castsi512_ps(v), _MM_SHUFFLE(i3, i2, i1, i0)));
+ }
+
+ template<int i>
+ __forceinline vint16 shuffle4(const vint16& v) {
+ return _mm512_castps_si512(_mm512_shuffle_f32x4(_mm512_castsi512_ps(v), _mm512_castsi512_ps(v), _MM_SHUFFLE(i, i, i, i)));
+ }
+
+ template<int i0, int i1, int i2, int i3>
+ __forceinline vint16 shuffle4(const vint16& v) {
+ return _mm512_castps_si512(_mm512_shuffle_f32x4(_mm512_castsi512_ps(v), _mm512_castsi512_ps(v), _MM_SHUFFLE(i3, i2, i1, i0)));
+ }
+
+ template<int i>
+ __forceinline vint16 align_shift_right(const vint16& a, const vint16& b) {
+ return _mm512_alignr_epi32(a, b, i);
+ };
+
+ __forceinline int toScalar(const vint16& v) {
+ return _mm_cvtsi128_si32(_mm512_castsi512_si128(v));
+ }
+
+ template<int i> __forceinline vint16 insert4(const vint16& a, const vint4& b) { return _mm512_inserti32x4(a, b, i); }
+
+ __forceinline size_t extract64bit(const vint16& v) {
+ return _mm_cvtsi128_si64(_mm512_castsi512_si128(v));
+ }
+
+ template<int N, int i>
+ vint<N> extractN(const vint16& v);
+
+ template<> __forceinline vint4 extractN<4,0>(const vint16& v) { return _mm512_castsi512_si128(v); }
+ template<> __forceinline vint4 extractN<4,1>(const vint16& v) { return _mm512_extracti32x4_epi32(v, 1); }
+ template<> __forceinline vint4 extractN<4,2>(const vint16& v) { return _mm512_extracti32x4_epi32(v, 2); }
+ template<> __forceinline vint4 extractN<4,3>(const vint16& v) { return _mm512_extracti32x4_epi32(v, 3); }
+
+ template<> __forceinline vint8 extractN<8,0>(const vint16& v) { return _mm512_castsi512_si256(v); }
+ template<> __forceinline vint8 extractN<8,1>(const vint16& v) { return _mm512_extracti32x8_epi32(v, 1); }
+
+ template<int i> __forceinline vint4 extract4 (const vint16& v) { return _mm512_extracti32x4_epi32(v, i); }
+ template<> __forceinline vint4 extract4<0>(const vint16& v) { return _mm512_castsi512_si128(v); }
+
+ template<int i> __forceinline vint8 extract8 (const vint16& v) { return _mm512_extracti32x8_epi32(v, i); }
+ template<> __forceinline vint8 extract8<0>(const vint16& v) { return _mm512_castsi512_si256(v); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Reductions
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vint16 vreduce_min2(vint16 x) { return min(x, shuffle<1,0,3,2>(x)); }
+ __forceinline vint16 vreduce_min4(vint16 x) { x = vreduce_min2(x); return min(x, shuffle<2,3,0,1>(x)); }
+ __forceinline vint16 vreduce_min8(vint16 x) { x = vreduce_min4(x); return min(x, shuffle4<1,0,3,2>(x)); }
+ __forceinline vint16 vreduce_min (vint16 x) { x = vreduce_min8(x); return min(x, shuffle4<2,3,0,1>(x)); }
+
+ __forceinline vint16 vreduce_max2(vint16 x) { return max(x, shuffle<1,0,3,2>(x)); }
+ __forceinline vint16 vreduce_max4(vint16 x) { x = vreduce_max2(x); return max(x, shuffle<2,3,0,1>(x)); }
+ __forceinline vint16 vreduce_max8(vint16 x) { x = vreduce_max4(x); return max(x, shuffle4<1,0,3,2>(x)); }
+ __forceinline vint16 vreduce_max (vint16 x) { x = vreduce_max8(x); return max(x, shuffle4<2,3,0,1>(x)); }
+
+ __forceinline vint16 vreduce_and2(vint16 x) { return x & shuffle<1,0,3,2>(x); }
+ __forceinline vint16 vreduce_and4(vint16 x) { x = vreduce_and2(x); return x & shuffle<2,3,0,1>(x); }
+ __forceinline vint16 vreduce_and8(vint16 x) { x = vreduce_and4(x); return x & shuffle4<1,0,3,2>(x); }
+ __forceinline vint16 vreduce_and (vint16 x) { x = vreduce_and8(x); return x & shuffle4<2,3,0,1>(x); }
+
+ __forceinline vint16 vreduce_or2(vint16 x) { return x | shuffle<1,0,3,2>(x); }
+ __forceinline vint16 vreduce_or4(vint16 x) { x = vreduce_or2(x); return x | shuffle<2,3,0,1>(x); }
+ __forceinline vint16 vreduce_or8(vint16 x) { x = vreduce_or4(x); return x | shuffle4<1,0,3,2>(x); }
+ __forceinline vint16 vreduce_or (vint16 x) { x = vreduce_or8(x); return x | shuffle4<2,3,0,1>(x); }
+
+ __forceinline vint16 vreduce_add2(vint16 x) { return x + shuffle<1,0,3,2>(x); }
+ __forceinline vint16 vreduce_add4(vint16 x) { x = vreduce_add2(x); return x + shuffle<2,3,0,1>(x); }
+ __forceinline vint16 vreduce_add8(vint16 x) { x = vreduce_add4(x); return x + shuffle4<1,0,3,2>(x); }
+ __forceinline vint16 vreduce_add (vint16 x) { x = vreduce_add8(x); return x + shuffle4<2,3,0,1>(x); }
+
+ __forceinline int reduce_min(const vint16& v) { return toScalar(vreduce_min(v)); }
+ __forceinline int reduce_max(const vint16& v) { return toScalar(vreduce_max(v)); }
+ __forceinline int reduce_and(const vint16& v) { return toScalar(vreduce_and(v)); }
+ __forceinline int reduce_or (const vint16& v) { return toScalar(vreduce_or (v)); }
+ __forceinline int reduce_add(const vint16& v) { return toScalar(vreduce_add(v)); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Memory load and store operations
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vint16 conflict(const vint16& index)
+ {
+ return _mm512_conflict_epi32(index);
+ }
+
+ __forceinline vint16 conflict(const vboolf16& mask, vint16& dest, const vint16& index)
+ {
+ return _mm512_mask_conflict_epi32(dest,mask,index);
+ }
+
+ __forceinline vint16 convert_uint32_t(const __m512& f) {
+ return _mm512_cvtps_epu32(f);
+ }
+
+ __forceinline vint16 permute(vint16 v, vint16 index) {
+ return _mm512_permutexvar_epi32(index,v);
+ }
+
+ __forceinline vint16 reverse(const vint16 &a) {
+ return permute(a,vint16(reverse_step));
+ }
+
+ __forceinline vint16 prefix_sum(const vint16& a)
+ {
+ const vint16 z(zero);
+ vint16 v = a;
+ v = v + align_shift_right<16-1>(v,z);
+ v = v + align_shift_right<16-2>(v,z);
+ v = v + align_shift_right<16-4>(v,z);
+ v = v + align_shift_right<16-8>(v,z);
+ return v;
+ }
+
+ __forceinline vint16 reverse_prefix_sum(const vint16& a)
+ {
+ const vint16 z(zero);
+ vint16 v = a;
+ v = v + align_shift_right<1>(z,v);
+ v = v + align_shift_right<2>(z,v);
+ v = v + align_shift_right<4>(z,v);
+ v = v + align_shift_right<8>(z,v);
+ return v;
+ }
+
+ /* this should use a vbool8 and a vint8_64...*/
+ template<int scale = 1, int hint = _MM_HINT_T0>
+ __forceinline void gather_prefetch64(const void* base_addr, const vbool16& mask, const vint16& offset)
+ {
+#if defined(__AVX512PF__)
+ _mm512_mask_prefetch_i64gather_pd(offset, mask, base_addr, scale, hint);
+#endif
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Output Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline embree_ostream operator <<(embree_ostream cout, const vint16& v)
+ {
+ cout << "<" << v[0];
+ for (int i=1; i<16; i++) cout << ", " << v[i];
+ cout << ">";
+ return cout;
+ }
+}
diff --git a/thirdparty/embree-aarch64/common/simd/vint4_sse2.h b/thirdparty/embree-aarch64/common/simd/vint4_sse2.h
new file mode 100644
index 0000000000..96f105a7c5
--- /dev/null
+++ b/thirdparty/embree-aarch64/common/simd/vint4_sse2.h
@@ -0,0 +1,681 @@
+// Copyright 2009-2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+#include "../math/math.h"
+
+namespace embree
+{
+ /* 4-wide SSE integer type */
+ template<>
+ struct vint<4>
+ {
+ ALIGNED_STRUCT_(16);
+
+ typedef vboolf4 Bool;
+ typedef vint4 Int;
+ typedef vfloat4 Float;
+
+ enum { size = 4 }; // number of SIMD elements
+ union { __m128i v; int i[4]; }; // data
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constructors, Assignment & Cast Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vint() {}
+ __forceinline vint(const vint4& a) { v = a.v; }
+ __forceinline vint4& operator =(const vint4& a) { v = a.v; return *this; }
+
+ __forceinline vint(__m128i a) : v(a) {}
+ __forceinline operator const __m128i&() const { return v; }
+ __forceinline operator __m128i&() { return v; }
+
+ __forceinline vint(int a) : v(_mm_set1_epi32(a)) {}
+ __forceinline vint(int a, int b, int c, int d) : v(_mm_set_epi32(d, c, b, a)) {}
+
+ __forceinline explicit vint(__m128 a) : v(_mm_cvtps_epi32(a)) {}
+#if defined(__AVX512VL__)
+ __forceinline explicit vint(const vboolf4& a) : v(_mm_movm_epi32(a)) {}
+#else
+ __forceinline explicit vint(const vboolf4& a) : v(_mm_castps_si128((__m128)a)) {}
+#endif
+
+ __forceinline vint(long long a, long long b) : v(_mm_set_epi64x(b,a)) {}
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constants
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vint(ZeroTy) : v(_mm_setzero_si128()) {}
+ __forceinline vint(OneTy) : v(_mm_set_epi32(1, 1, 1, 1)) {}
+ __forceinline vint(PosInfTy) : v(_mm_set_epi32(pos_inf, pos_inf, pos_inf, pos_inf)) {}
+ __forceinline vint(NegInfTy) : v(_mm_set_epi32(neg_inf, neg_inf, neg_inf, neg_inf)) {}
+ __forceinline vint(StepTy) : v(_mm_set_epi32(3, 2, 1, 0)) {}
+ __forceinline vint(ReverseStepTy) : v(_mm_set_epi32(0, 1, 2, 3)) {}
+
+ __forceinline vint(TrueTy) { v = _mm_cmpeq_epi32(v,v); }
+ __forceinline vint(UndefinedTy) : v(_mm_castps_si128(_mm_undefined_ps())) {}
+
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Loads and Stores
+ ////////////////////////////////////////////////////////////////////////////////
+
+ static __forceinline vint4 load (const void* a) { return _mm_load_si128((__m128i*)a); }
+ static __forceinline vint4 loadu(const void* a) { return _mm_loadu_si128((__m128i*)a); }
+
+ static __forceinline void store (void* ptr, const vint4& v) { _mm_store_si128((__m128i*)ptr,v); }
+ static __forceinline void storeu(void* ptr, const vint4& v) { _mm_storeu_si128((__m128i*)ptr,v); }
+
+#if defined(__AVX512VL__)
+
+ static __forceinline vint4 compact(const vboolf4& mask, vint4 &v) {
+ return _mm_mask_compress_epi32(v, mask, v);
+ }
+ static __forceinline vint4 compact(const vboolf4& mask, vint4 &a, const vint4& b) {
+ return _mm_mask_compress_epi32(a, mask, b);
+ }
+
+ static __forceinline vint4 load (const vboolf4& mask, const void* ptr) { return _mm_mask_load_epi32 (_mm_setzero_si128(),mask,ptr); }
+ static __forceinline vint4 loadu(const vboolf4& mask, const void* ptr) { return _mm_mask_loadu_epi32(_mm_setzero_si128(),mask,ptr); }
+
+ static __forceinline void store (const vboolf4& mask, void* ptr, const vint4& v) { _mm_mask_store_epi32 (ptr,mask,v); }
+ static __forceinline void storeu(const vboolf4& mask, void* ptr, const vint4& v) { _mm_mask_storeu_epi32(ptr,mask,v); }
+#elif defined(__AVX__)
+ static __forceinline vint4 load (const vbool4& mask, const void* a) { return _mm_castps_si128(_mm_maskload_ps((float*)a,mask)); }
+ static __forceinline vint4 loadu(const vbool4& mask, const void* a) { return _mm_castps_si128(_mm_maskload_ps((float*)a,mask)); }
+
+ static __forceinline void store (const vboolf4& mask, void* ptr, const vint4& i) { _mm_maskstore_ps((float*)ptr,(__m128i)mask,_mm_castsi128_ps(i)); }
+ static __forceinline void storeu(const vboolf4& mask, void* ptr, const vint4& i) { _mm_maskstore_ps((float*)ptr,(__m128i)mask,_mm_castsi128_ps(i)); }
+#else
+ static __forceinline vint4 load (const vbool4& mask, const void* a) { return _mm_and_si128(_mm_load_si128 ((__m128i*)a),mask); }
+ static __forceinline vint4 loadu(const vbool4& mask, const void* a) { return _mm_and_si128(_mm_loadu_si128((__m128i*)a),mask); }
+
+ static __forceinline void store (const vboolf4& mask, void* ptr, const vint4& i) { store (ptr,select(mask,i,load (ptr))); }
+ static __forceinline void storeu(const vboolf4& mask, void* ptr, const vint4& i) { storeu(ptr,select(mask,i,loadu(ptr))); }
+#endif
+
+
+#if defined(__aarch64__)
+ static __forceinline vint4 load(const uint8_t* ptr) {
+ return _mm_load4epu8_epi32(((__m128i*)ptr));
+ }
+ static __forceinline vint4 loadu(const uint8_t* ptr) {
+ return _mm_load4epu8_epi32(((__m128i*)ptr));
+ }
+#elif defined(__SSE4_1__)
+ static __forceinline vint4 load(const uint8_t* ptr) {
+ return _mm_cvtepu8_epi32(_mm_loadl_epi64((__m128i*)ptr));
+ }
+
+ static __forceinline vint4 loadu(const uint8_t* ptr) {
+ return _mm_cvtepu8_epi32(_mm_loadl_epi64((__m128i*)ptr));
+ }
+#else
+
+ static __forceinline vint4 load(const uint8_t* ptr) {
+ return vint4(ptr[0],ptr[1],ptr[2],ptr[3]);
+ }
+
+ static __forceinline vint4 loadu(const uint8_t* ptr) {
+ return vint4(ptr[0],ptr[1],ptr[2],ptr[3]);
+ }
+
+#endif
+
+ static __forceinline vint4 load(const unsigned short* ptr) {
+#if defined(__aarch64__)
+ return __m128i(vmovl_u16(vld1_u16(ptr)));
+#elif defined (__SSE4_1__)
+ return _mm_cvtepu16_epi32(_mm_loadu_si128((__m128i*)ptr));
+#else
+ return vint4(ptr[0],ptr[1],ptr[2],ptr[3]);
+#endif
+ }
+
+ static __forceinline void store(uint8_t* ptr, const vint4& v) {
+#if defined(__aarch64__)
+ int32x4_t x = v;
+ uint16x4_t y = vqmovn_u32(uint32x4_t(x));
+ uint8x8_t z = vqmovn_u16(vcombine_u16(y, y));
+ vst1_lane_u32((uint32_t *)ptr,uint32x2_t(z), 0);
+#elif defined(__SSE4_1__)
+ __m128i x = v;
+ x = _mm_packus_epi32(x, x);
+ x = _mm_packus_epi16(x, x);
+ *(int*)ptr = _mm_cvtsi128_si32(x);
+#else
+ for (size_t i=0;i<4;i++)
+ ptr[i] = (uint8_t)v[i];
+#endif
+ }
+
+ static __forceinline void store(unsigned short* ptr, const vint4& v) {
+#if defined(__aarch64__)
+ uint32x4_t x = uint32x4_t(v.v);
+ uint16x4_t y = vqmovn_u32(x);
+ vst1_u16(ptr, y);
+#else
+ for (size_t i=0;i<4;i++)
+ ptr[i] = (unsigned short)v[i];
+#endif
+ }
+
+ static __forceinline vint4 load_nt(void* ptr) {
+#if defined(__aarch64__) || defined(__SSE4_1__)
+ return _mm_stream_load_si128((__m128i*)ptr);
+#else
+ return _mm_load_si128((__m128i*)ptr);
+#endif
+ }
+
+ static __forceinline void store_nt(void* ptr, const vint4& v) {
+#if !defined(__aarch64__) && defined(__SSE4_1__)
+ _mm_stream_ps((float*)ptr, _mm_castsi128_ps(v));
+#else
+ _mm_store_si128((__m128i*)ptr,v);
+#endif
+ }
+
+ template<int scale = 4>
+ static __forceinline vint4 gather(const int* ptr, const vint4& index) {
+#if defined(__AVX2__) && !defined(__aarch64__)
+ return _mm_i32gather_epi32(ptr, index, scale);
+#else
+ return vint4(
+ *(int*)(((int8_t*)ptr)+scale*index[0]),
+ *(int*)(((int8_t*)ptr)+scale*index[1]),
+ *(int*)(((int8_t*)ptr)+scale*index[2]),
+ *(int*)(((int8_t*)ptr)+scale*index[3]));
+#endif
+ }
+
+ template<int scale = 4>
+ static __forceinline vint4 gather(const vboolf4& mask, const int* ptr, const vint4& index) {
+ vint4 r = zero;
+#if defined(__AVX512VL__)
+ return _mm_mmask_i32gather_epi32(r, mask, index, ptr, scale);
+#elif defined(__AVX2__) && !defined(__aarch64__)
+ return _mm_mask_i32gather_epi32(r, ptr, index, mask, scale);
+#else
+ if (likely(mask[0])) r[0] = *(int*)(((int8_t*)ptr)+scale*index[0]);
+ if (likely(mask[1])) r[1] = *(int*)(((int8_t*)ptr)+scale*index[1]);
+ if (likely(mask[2])) r[2] = *(int*)(((int8_t*)ptr)+scale*index[2]);
+ if (likely(mask[3])) r[3] = *(int*)(((int8_t*)ptr)+scale*index[3]);
+ return r;
+#endif
+ }
+
+ template<int scale = 4>
+ static __forceinline void scatter(void* ptr, const vint4& index, const vint4& v)
+ {
+#if defined(__AVX512VL__)
+ _mm_i32scatter_epi32((int*)ptr, index, v, scale);
+#else
+ *(int*)(((int8_t*)ptr)+scale*index[0]) = v[0];
+ *(int*)(((int8_t*)ptr)+scale*index[1]) = v[1];
+ *(int*)(((int8_t*)ptr)+scale*index[2]) = v[2];
+ *(int*)(((int8_t*)ptr)+scale*index[3]) = v[3];
+#endif
+ }
+
+ template<int scale = 4>
+ static __forceinline void scatter(const vboolf4& mask, void* ptr, const vint4& index, const vint4& v)
+ {
+#if defined(__AVX512VL__)
+ _mm_mask_i32scatter_epi32((int*)ptr, mask, index, v, scale);
+#else
+ if (likely(mask[0])) *(int*)(((int8_t*)ptr)+scale*index[0]) = v[0];
+ if (likely(mask[1])) *(int*)(((int8_t*)ptr)+scale*index[1]) = v[1];
+ if (likely(mask[2])) *(int*)(((int8_t*)ptr)+scale*index[2]) = v[2];
+ if (likely(mask[3])) *(int*)(((int8_t*)ptr)+scale*index[3]) = v[3];
+#endif
+ }
+
+#if defined(__x86_64__) || defined(__aarch64__)
+ static __forceinline vint4 broadcast64(long long a) { return _mm_set1_epi64x(a); }
+#endif
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Array Access
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline const int& operator [](size_t index) const { assert(index < 4); return i[index]; }
+ __forceinline int& operator [](size_t index) { assert(index < 4); return i[index]; }
+
+ friend __forceinline vint4 select(const vboolf4& m, const vint4& t, const vint4& f) {
+#if defined(__AVX512VL__)
+ return _mm_mask_blend_epi32(m, (__m128i)f, (__m128i)t);
+#elif defined(__aarch64__)
+ return _mm_castps_si128(_mm_blendv_ps((__m128)f.v,(__m128) t.v, (__m128)m.v));
+#elif defined(__SSE4_1__)
+ return _mm_castps_si128(_mm_blendv_ps(_mm_castsi128_ps(f), _mm_castsi128_ps(t), m));
+#else
+ return _mm_or_si128(_mm_and_si128(m, t), _mm_andnot_si128(m, f));
+#endif
+ }
+ };
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Unary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+#if defined(__AVX512VL__)
+ __forceinline vboolf4 asBool(const vint4& a) { return _mm_movepi32_mask(a); }
+#else
+ __forceinline vboolf4 asBool(const vint4& a) { return _mm_castsi128_ps(a); }
+#endif
+
+ __forceinline vint4 operator +(const vint4& a) { return a; }
+ __forceinline vint4 operator -(const vint4& a) { return _mm_sub_epi32(_mm_setzero_si128(), a); }
+#if defined(__aarch64__)
+ __forceinline vint4 abs(const vint4& a) { return vabsq_s32(a.v); }
+#elif defined(__SSSE3__)
+ __forceinline vint4 abs(const vint4& a) { return _mm_abs_epi32(a); }
+#endif
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Binary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vint4 operator +(const vint4& a, const vint4& b) { return _mm_add_epi32(a, b); }
+ __forceinline vint4 operator +(const vint4& a, int b) { return a + vint4(b); }
+ __forceinline vint4 operator +(int a, const vint4& b) { return vint4(a) + b; }
+
+ __forceinline vint4 operator -(const vint4& a, const vint4& b) { return _mm_sub_epi32(a, b); }
+ __forceinline vint4 operator -(const vint4& a, int b) { return a - vint4(b); }
+ __forceinline vint4 operator -(int a, const vint4& b) { return vint4(a) - b; }
+
+#if (defined(__aarch64__)) || defined(__SSE4_1__)
+ __forceinline vint4 operator *(const vint4& a, const vint4& b) { return _mm_mullo_epi32(a, b); }
+#else
+ __forceinline vint4 operator *(const vint4& a, const vint4& b) { return vint4(a[0]*b[0],a[1]*b[1],a[2]*b[2],a[3]*b[3]); }
+#endif
+ __forceinline vint4 operator *(const vint4& a, int b) { return a * vint4(b); }
+ __forceinline vint4 operator *(int a, const vint4& b) { return vint4(a) * b; }
+
+ __forceinline vint4 operator &(const vint4& a, const vint4& b) { return _mm_and_si128(a, b); }
+ __forceinline vint4 operator &(const vint4& a, int b) { return a & vint4(b); }
+ __forceinline vint4 operator &(int a, const vint4& b) { return vint4(a) & b; }
+
+ __forceinline vint4 operator |(const vint4& a, const vint4& b) { return _mm_or_si128(a, b); }
+ __forceinline vint4 operator |(const vint4& a, int b) { return a | vint4(b); }
+ __forceinline vint4 operator |(int a, const vint4& b) { return vint4(a) | b; }
+
+ __forceinline vint4 operator ^(const vint4& a, const vint4& b) { return _mm_xor_si128(a, b); }
+ __forceinline vint4 operator ^(const vint4& a, int b) { return a ^ vint4(b); }
+ __forceinline vint4 operator ^(int a, const vint4& b) { return vint4(a) ^ b; }
+
+ __forceinline vint4 operator <<(const vint4& a, const int n) { return _mm_slli_epi32(a, n); }
+ __forceinline vint4 operator >>(const vint4& a, const int n) { return _mm_srai_epi32(a, n); }
+
+ __forceinline vint4 sll (const vint4& a, int b) { return _mm_slli_epi32(a, b); }
+ __forceinline vint4 sra (const vint4& a, int b) { return _mm_srai_epi32(a, b); }
+ __forceinline vint4 srl (const vint4& a, int b) { return _mm_srli_epi32(a, b); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Assignment Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vint4& operator +=(vint4& a, const vint4& b) { return a = a + b; }
+ __forceinline vint4& operator +=(vint4& a, int b) { return a = a + b; }
+
+ __forceinline vint4& operator -=(vint4& a, const vint4& b) { return a = a - b; }
+ __forceinline vint4& operator -=(vint4& a, int b) { return a = a - b; }
+
+#if (defined(__aarch64__)) || defined(__SSE4_1__)
+ __forceinline vint4& operator *=(vint4& a, const vint4& b) { return a = a * b; }
+ __forceinline vint4& operator *=(vint4& a, int b) { return a = a * b; }
+#endif
+
+ __forceinline vint4& operator &=(vint4& a, const vint4& b) { return a = a & b; }
+ __forceinline vint4& operator &=(vint4& a, int b) { return a = a & b; }
+
+ __forceinline vint4& operator |=(vint4& a, const vint4& b) { return a = a | b; }
+ __forceinline vint4& operator |=(vint4& a, int b) { return a = a | b; }
+
+ __forceinline vint4& operator <<=(vint4& a, int b) { return a = a << b; }
+ __forceinline vint4& operator >>=(vint4& a, int b) { return a = a >> b; }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Comparison Operators + Select
+ ////////////////////////////////////////////////////////////////////////////////
+
+#if defined(__AVX512VL__)
+ __forceinline vboolf4 operator ==(const vint4& a, const vint4& b) { return _mm_cmp_epi32_mask(a,b,_MM_CMPINT_EQ); }
+ __forceinline vboolf4 operator !=(const vint4& a, const vint4& b) { return _mm_cmp_epi32_mask(a,b,_MM_CMPINT_NE); }
+ __forceinline vboolf4 operator < (const vint4& a, const vint4& b) { return _mm_cmp_epi32_mask(a,b,_MM_CMPINT_LT); }
+ __forceinline vboolf4 operator >=(const vint4& a, const vint4& b) { return _mm_cmp_epi32_mask(a,b,_MM_CMPINT_GE); }
+ __forceinline vboolf4 operator > (const vint4& a, const vint4& b) { return _mm_cmp_epi32_mask(a,b,_MM_CMPINT_GT); }
+ __forceinline vboolf4 operator <=(const vint4& a, const vint4& b) { return _mm_cmp_epi32_mask(a,b,_MM_CMPINT_LE); }
+#else
+ __forceinline vboolf4 operator ==(const vint4& a, const vint4& b) { return _mm_castsi128_ps(_mm_cmpeq_epi32(a, b)); }
+ __forceinline vboolf4 operator !=(const vint4& a, const vint4& b) { return !(a == b); }
+ __forceinline vboolf4 operator < (const vint4& a, const vint4& b) { return _mm_castsi128_ps(_mm_cmplt_epi32(a, b)); }
+ __forceinline vboolf4 operator >=(const vint4& a, const vint4& b) { return !(a < b); }
+ __forceinline vboolf4 operator > (const vint4& a, const vint4& b) { return _mm_castsi128_ps(_mm_cmpgt_epi32(a, b)); }
+ __forceinline vboolf4 operator <=(const vint4& a, const vint4& b) { return !(a > b); }
+#endif
+
+ __forceinline vboolf4 operator ==(const vint4& a, int b) { return a == vint4(b); }
+ __forceinline vboolf4 operator ==(int a, const vint4& b) { return vint4(a) == b; }
+
+ __forceinline vboolf4 operator !=(const vint4& a, int b) { return a != vint4(b); }
+ __forceinline vboolf4 operator !=(int a, const vint4& b) { return vint4(a) != b; }
+
+ __forceinline vboolf4 operator < (const vint4& a, int b) { return a < vint4(b); }
+ __forceinline vboolf4 operator < (int a, const vint4& b) { return vint4(a) < b; }
+
+ __forceinline vboolf4 operator >=(const vint4& a, int b) { return a >= vint4(b); }
+ __forceinline vboolf4 operator >=(int a, const vint4& b) { return vint4(a) >= b; }
+
+ __forceinline vboolf4 operator > (const vint4& a, int b) { return a > vint4(b); }
+ __forceinline vboolf4 operator > (int a, const vint4& b) { return vint4(a) > b; }
+
+ __forceinline vboolf4 operator <=(const vint4& a, int b) { return a <= vint4(b); }
+ __forceinline vboolf4 operator <=(int a, const vint4& b) { return vint4(a) <= b; }
+
+ __forceinline vboolf4 eq(const vint4& a, const vint4& b) { return a == b; }
+ __forceinline vboolf4 ne(const vint4& a, const vint4& b) { return a != b; }
+ __forceinline vboolf4 lt(const vint4& a, const vint4& b) { return a < b; }
+ __forceinline vboolf4 ge(const vint4& a, const vint4& b) { return a >= b; }
+ __forceinline vboolf4 gt(const vint4& a, const vint4& b) { return a > b; }
+ __forceinline vboolf4 le(const vint4& a, const vint4& b) { return a <= b; }
+
+#if defined(__AVX512VL__)
+ __forceinline vboolf4 eq(const vboolf4& mask, const vint4& a, const vint4& b) { return _mm_mask_cmp_epi32_mask(mask, a, b, _MM_CMPINT_EQ); }
+ __forceinline vboolf4 ne(const vboolf4& mask, const vint4& a, const vint4& b) { return _mm_mask_cmp_epi32_mask(mask, a, b, _MM_CMPINT_NE); }
+ __forceinline vboolf4 lt(const vboolf4& mask, const vint4& a, const vint4& b) { return _mm_mask_cmp_epi32_mask(mask, a, b, _MM_CMPINT_LT); }
+ __forceinline vboolf4 ge(const vboolf4& mask, const vint4& a, const vint4& b) { return _mm_mask_cmp_epi32_mask(mask, a, b, _MM_CMPINT_GE); }
+ __forceinline vboolf4 gt(const vboolf4& mask, const vint4& a, const vint4& b) { return _mm_mask_cmp_epi32_mask(mask, a, b, _MM_CMPINT_GT); }
+ __forceinline vboolf4 le(const vboolf4& mask, const vint4& a, const vint4& b) { return _mm_mask_cmp_epi32_mask(mask, a, b, _MM_CMPINT_LE); }
+#else
+ __forceinline vboolf4 eq(const vboolf4& mask, const vint4& a, const vint4& b) { return mask & (a == b); }
+ __forceinline vboolf4 ne(const vboolf4& mask, const vint4& a, const vint4& b) { return mask & (a != b); }
+ __forceinline vboolf4 lt(const vboolf4& mask, const vint4& a, const vint4& b) { return mask & (a < b); }
+ __forceinline vboolf4 ge(const vboolf4& mask, const vint4& a, const vint4& b) { return mask & (a >= b); }
+ __forceinline vboolf4 gt(const vboolf4& mask, const vint4& a, const vint4& b) { return mask & (a > b); }
+ __forceinline vboolf4 le(const vboolf4& mask, const vint4& a, const vint4& b) { return mask & (a <= b); }
+#endif
+
+ template<int mask>
+ __forceinline vint4 select(const vint4& t, const vint4& f) {
+#if defined(__SSE4_1__)
+ return _mm_castps_si128(_mm_blend_ps(_mm_castsi128_ps(f), _mm_castsi128_ps(t), mask));
+#else
+ return select(vboolf4(mask), t, f);
+#endif
+ }
+
+
+#if defined(__aarch64__) || defined(__SSE4_1__)
+ __forceinline vint4 min(const vint4& a, const vint4& b) { return _mm_min_epi32(a, b); }
+ __forceinline vint4 max(const vint4& a, const vint4& b) { return _mm_max_epi32(a, b); }
+
+ __forceinline vint4 umin(const vint4& a, const vint4& b) { return _mm_min_epu32(a, b); }
+ __forceinline vint4 umax(const vint4& a, const vint4& b) { return _mm_max_epu32(a, b); }
+
+#else
+ __forceinline vint4 min(const vint4& a, const vint4& b) { return select(a < b,a,b); }
+ __forceinline vint4 max(const vint4& a, const vint4& b) { return select(a < b,b,a); }
+#endif
+
+ __forceinline vint4 min(const vint4& a, int b) { return min(a,vint4(b)); }
+ __forceinline vint4 min(int a, const vint4& b) { return min(vint4(a),b); }
+ __forceinline vint4 max(const vint4& a, int b) { return max(a,vint4(b)); }
+ __forceinline vint4 max(int a, const vint4& b) { return max(vint4(a),b); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ // Movement/Shifting/Shuffling Functions
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vint4 unpacklo(const vint4& a, const vint4& b) { return _mm_castps_si128(_mm_unpacklo_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b))); }
+ __forceinline vint4 unpackhi(const vint4& a, const vint4& b) { return _mm_castps_si128(_mm_unpackhi_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b))); }
+
+#if defined(__aarch64__)
+ template<int i0, int i1, int i2, int i3>
+ __forceinline vint4 shuffle(const vint4& v) {
+ return vreinterpretq_s32_u8(vqtbl1q_u8( (uint8x16_t)v.v, _MN_SHUFFLE(i0, i1, i2, i3)));
+ }
+ template<int i0, int i1, int i2, int i3>
+ __forceinline vint4 shuffle(const vint4& a, const vint4& b) {
+ return vreinterpretq_s32_u8(vqtbl2q_u8( (uint8x16x2_t){(uint8x16_t)a.v, (uint8x16_t)b.v}, _MF_SHUFFLE(i0, i1, i2, i3)));
+ }
+#else
+ template<int i0, int i1, int i2, int i3>
+ __forceinline vint4 shuffle(const vint4& v) {
+ return _mm_shuffle_epi32(v, _MM_SHUFFLE(i3, i2, i1, i0));
+ }
+ template<int i0, int i1, int i2, int i3>
+ __forceinline vint4 shuffle(const vint4& a, const vint4& b) {
+ return _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b), _MM_SHUFFLE(i3, i2, i1, i0)));
+ }
+#endif
+#if defined(__SSE3__)
+ template<> __forceinline vint4 shuffle<0, 0, 2, 2>(const vint4& v) { return _mm_castps_si128(_mm_moveldup_ps(_mm_castsi128_ps(v))); }
+ template<> __forceinline vint4 shuffle<1, 1, 3, 3>(const vint4& v) { return _mm_castps_si128(_mm_movehdup_ps(_mm_castsi128_ps(v))); }
+ template<> __forceinline vint4 shuffle<0, 1, 0, 1>(const vint4& v) { return _mm_castpd_si128(_mm_movedup_pd (_mm_castsi128_pd(v))); }
+#endif
+
+ template<int i>
+ __forceinline vint4 shuffle(const vint4& v) {
+ return shuffle<i,i,i,i>(v);
+ }
+
+#if defined(__aarch64__)
+ template<int src> __forceinline int extract(const vint4& b);
+ template<int dst> __forceinline vint4 insert(const vint4& a, const int b);
+#elif defined(__SSE4_1__)
+ template<int src> __forceinline int extract(const vint4& b) { return _mm_extract_epi32(b, src); }
+ template<int dst> __forceinline vint4 insert(const vint4& a, const int b) { return _mm_insert_epi32(a, b, dst); }
+#else
+ template<int src> __forceinline int extract(const vint4& b) { return b[src&3]; }
+ template<int dst> __forceinline vint4 insert(const vint4& a, int b) { vint4 c = a; c[dst&3] = b; return c; }
+#endif
+
+#if defined(__aarch64__)
+ template<> __forceinline int extract<0>(const vint4& b) {
+ return b.v[0];
+ }
+ template<> __forceinline int extract<1>(const vint4& b) {
+ return b.v[1];
+ }
+ template<> __forceinline int extract<2>(const vint4& b) {
+ return b.v[2];
+ }
+ template<> __forceinline int extract<3>(const vint4& b) {
+ return b.v[3];
+ }
+ template<> __forceinline vint4 insert<0>(const vint4& a, int b)
+ {
+ vint4 c = a;
+ c[0] = b;
+ return c;
+ }
+ template<> __forceinline vint4 insert<1>(const vint4& a, int b)
+ {
+ vint4 c = a;
+ c[1] = b;
+ return c;
+ }
+ template<> __forceinline vint4 insert<2>(const vint4& a, int b)
+ {
+ vint4 c = a;
+ c[2] = b;
+ return c;
+ }
+ template<> __forceinline vint4 insert<3>(const vint4& a, int b)
+ {
+ vint4 c = a;
+ c[3] = b;
+ return c;
+ }
+
+ __forceinline int toScalar(const vint4& v) {
+ return v[0];
+ }
+
+ __forceinline size_t toSizeT(const vint4& v) {
+ uint64x2_t x = uint64x2_t(v.v);
+ return x[0];
+ }
+#else
+ template<> __forceinline int extract<0>(const vint4& b) { return _mm_cvtsi128_si32(b); }
+
+ __forceinline int toScalar(const vint4& v) { return _mm_cvtsi128_si32(v); }
+
+ __forceinline size_t toSizeT(const vint4& v) {
+#if defined(__WIN32__) && !defined(__X86_64__) // win32 workaround
+ return toScalar(v);
+#elif defined(__ARM_NEON)
+ // FIXME(LTE): Do we need a swap(i.e. use lane 1)?
+ return vgetq_lane_u64(*(reinterpret_cast<const uint64x2_t *>(&v)), 0);
+#else
+ return _mm_cvtsi128_si64(v);
+#endif
+ }
+#endif
+
+#if defined(__AVX512VL__)
+
+ __forceinline vint4 permute(const vint4 &a, const vint4 &index) {
+ return _mm_castps_si128(_mm_permutevar_ps(_mm_castsi128_ps(a),index));
+ }
+
+ template<int i>
+ __forceinline vint4 align_shift_right(const vint4& a, const vint4& b) {
+ return _mm_alignr_epi32(a, b, i);
+ }
+#endif
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Reductions
+ ////////////////////////////////////////////////////////////////////////////////
+
+#if defined(__aarch64__) || defined(__SSE4_1__)
+
+#if defined(__aarch64__)
+ __forceinline vint4 vreduce_min(const vint4& v) { int h = vminvq_s32(v); return vdupq_n_s32(h); }
+ __forceinline vint4 vreduce_max(const vint4& v) { int h = vmaxvq_s32(v); return vdupq_n_s32(h); }
+ __forceinline vint4 vreduce_add(const vint4& v) { int h = vaddvq_s32(v); return vdupq_n_s32(h); }
+
+ __forceinline int reduce_min(const vint4& v) { return vminvq_s32(v); }
+ __forceinline int reduce_max(const vint4& v) { return vmaxvq_s32(v); }
+ __forceinline int reduce_add(const vint4& v) { return vaddvq_s32(v); }
+#else
+ __forceinline vint4 vreduce_min(const vint4& v) { vint4 h = min(shuffle<1,0,3,2>(v),v); return min(shuffle<2,3,0,1>(h),h); }
+ __forceinline vint4 vreduce_max(const vint4& v) { vint4 h = max(shuffle<1,0,3,2>(v),v); return max(shuffle<2,3,0,1>(h),h); }
+ __forceinline vint4 vreduce_add(const vint4& v) { vint4 h = shuffle<1,0,3,2>(v) + v ; return shuffle<2,3,0,1>(h) + h ; }
+
+ __forceinline int reduce_min(const vint4& v) { return toScalar(vreduce_min(v)); }
+ __forceinline int reduce_max(const vint4& v) { return toScalar(vreduce_max(v)); }
+ __forceinline int reduce_add(const vint4& v) { return toScalar(vreduce_add(v)); }
+#endif
+
+ __forceinline size_t select_min(const vint4& v) { return bsf(movemask(v == vreduce_min(v))); }
+ __forceinline size_t select_max(const vint4& v) { return bsf(movemask(v == vreduce_max(v))); }
+
+ __forceinline size_t select_min(const vboolf4& valid, const vint4& v) { const vint4 a = select(valid,v,vint4(pos_inf)); return bsf(movemask(valid & (a == vreduce_min(a)))); }
+ __forceinline size_t select_max(const vboolf4& valid, const vint4& v) { const vint4 a = select(valid,v,vint4(neg_inf)); return bsf(movemask(valid & (a == vreduce_max(a)))); }
+
+#else
+
+ __forceinline int reduce_min(const vint4& v) { return min(v[0],v[1],v[2],v[3]); }
+ __forceinline int reduce_max(const vint4& v) { return max(v[0],v[1],v[2],v[3]); }
+ __forceinline int reduce_add(const vint4& v) { return v[0]+v[1]+v[2]+v[3]; }
+
+#endif
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Sorting networks
+ ////////////////////////////////////////////////////////////////////////////////
+
+#if (defined(__aarch64__)) || defined(__SSE4_1__)
+
+ __forceinline vint4 usort_ascending(const vint4& v)
+ {
+ const vint4 a0 = v;
+ const vint4 b0 = shuffle<1,0,3,2>(a0);
+ const vint4 c0 = umin(a0,b0);
+ const vint4 d0 = umax(a0,b0);
+ const vint4 a1 = select<0x5 /* 0b0101 */>(c0,d0);
+ const vint4 b1 = shuffle<2,3,0,1>(a1);
+ const vint4 c1 = umin(a1,b1);
+ const vint4 d1 = umax(a1,b1);
+ const vint4 a2 = select<0x3 /* 0b0011 */>(c1,d1);
+ const vint4 b2 = shuffle<0,2,1,3>(a2);
+ const vint4 c2 = umin(a2,b2);
+ const vint4 d2 = umax(a2,b2);
+ const vint4 a3 = select<0x2 /* 0b0010 */>(c2,d2);
+ return a3;
+ }
+
+ __forceinline vint4 usort_descending(const vint4& v)
+ {
+ const vint4 a0 = v;
+ const vint4 b0 = shuffle<1,0,3,2>(a0);
+ const vint4 c0 = umax(a0,b0);
+ const vint4 d0 = umin(a0,b0);
+ const vint4 a1 = select<0x5 /* 0b0101 */>(c0,d0);
+ const vint4 b1 = shuffle<2,3,0,1>(a1);
+ const vint4 c1 = umax(a1,b1);
+ const vint4 d1 = umin(a1,b1);
+ const vint4 a2 = select<0x3 /* 0b0011 */>(c1,d1);
+ const vint4 b2 = shuffle<0,2,1,3>(a2);
+ const vint4 c2 = umax(a2,b2);
+ const vint4 d2 = umin(a2,b2);
+ const vint4 a3 = select<0x2 /* 0b0010 */>(c2,d2);
+ return a3;
+ }
+
+#else
+
+ __forceinline vint4 usort_ascending(const vint4& v)
+ {
+ const vint4 a0 = v-vint4(0x80000000);
+ const vint4 b0 = shuffle<1,0,3,2>(a0);
+ const vint4 c0 = min(a0,b0);
+ const vint4 d0 = max(a0,b0);
+ const vint4 a1 = select<0x5 /* 0b0101 */>(c0,d0);
+ const vint4 b1 = shuffle<2,3,0,1>(a1);
+ const vint4 c1 = min(a1,b1);
+ const vint4 d1 = max(a1,b1);
+ const vint4 a2 = select<0x3 /* 0b0011 */>(c1,d1);
+ const vint4 b2 = shuffle<0,2,1,3>(a2);
+ const vint4 c2 = min(a2,b2);
+ const vint4 d2 = max(a2,b2);
+ const vint4 a3 = select<0x2 /* 0b0010 */>(c2,d2);
+ return a3+vint4(0x80000000);
+ }
+
+ __forceinline vint4 usort_descending(const vint4& v)
+ {
+ const vint4 a0 = v-vint4(0x80000000);
+ const vint4 b0 = shuffle<1,0,3,2>(a0);
+ const vint4 c0 = max(a0,b0);
+ const vint4 d0 = min(a0,b0);
+ const vint4 a1 = select<0x5 /* 0b0101 */>(c0,d0);
+ const vint4 b1 = shuffle<2,3,0,1>(a1);
+ const vint4 c1 = max(a1,b1);
+ const vint4 d1 = min(a1,b1);
+ const vint4 a2 = select<0x3 /* 0b0011 */>(c1,d1);
+ const vint4 b2 = shuffle<0,2,1,3>(a2);
+ const vint4 c2 = max(a2,b2);
+ const vint4 d2 = min(a2,b2);
+ const vint4 a3 = select<0x2 /* 0b0010 */>(c2,d2);
+ return a3+vint4(0x80000000);
+ }
+
+#endif
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Output Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline embree_ostream operator <<(embree_ostream cout, const vint4& a) {
+ return cout << "<" << a[0] << ", " << a[1] << ", " << a[2] << ", " << a[3] << ">";
+ }
+}
+
diff --git a/thirdparty/embree-aarch64/common/simd/vint8_avx.h b/thirdparty/embree-aarch64/common/simd/vint8_avx.h
new file mode 100644
index 0000000000..25a771284d
--- /dev/null
+++ b/thirdparty/embree-aarch64/common/simd/vint8_avx.h
@@ -0,0 +1,464 @@
+// Copyright 2009-2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+namespace embree
+{
+ /* 8-wide AVX integer type */
+ template<>
+ struct vint<8>
+ {
+ ALIGNED_STRUCT_(32);
+
+ typedef vboolf8 Bool;
+ typedef vint8 Int;
+ typedef vfloat8 Float;
+
+ enum { size = 8 }; // number of SIMD elements
+ union { // data
+ __m256i v;
+ struct { __m128i vl,vh; };
+ int i[8];
+ };
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constructors, Assignment & Cast Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vint() {}
+ __forceinline vint(const vint8& a) { v = a.v; }
+ __forceinline vint8& operator =(const vint8& a) { v = a.v; return *this; }
+
+ __forceinline vint(__m256i a) : v(a) {}
+ __forceinline operator const __m256i&() const { return v; }
+ __forceinline operator __m256i&() { return v; }
+
+ __forceinline explicit vint(const vint4& a) : v(_mm256_insertf128_si256(_mm256_castsi128_si256(a),a,1)) {}
+ __forceinline vint(const vint4& a, const vint4& b) : v(_mm256_insertf128_si256(_mm256_castsi128_si256(a),b,1)) {}
+ __forceinline vint(const __m128i& a, const __m128i& b) : vl(a), vh(b) {}
+
+ __forceinline explicit vint(const int* a) : v(_mm256_castps_si256(_mm256_loadu_ps((const float*)a))) {}
+ __forceinline vint(int a) : v(_mm256_set1_epi32(a)) {}
+ __forceinline vint(int a, int b) : v(_mm256_set_epi32(b, a, b, a, b, a, b, a)) {}
+ __forceinline vint(int a, int b, int c, int d) : v(_mm256_set_epi32(d, c, b, a, d, c, b, a)) {}
+ __forceinline vint(int a, int b, int c, int d, int e, int f, int g, int vh) : v(_mm256_set_epi32(vh, g, f, e, d, c, b, a)) {}
+
+ __forceinline explicit vint(__m256 a) : v(_mm256_cvtps_epi32(a)) {}
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constants
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vint(ZeroTy) : v(_mm256_setzero_si256()) {}
+ __forceinline vint(OneTy) : v(_mm256_set_epi32(1,1,1,1,1,1,1,1)) {}
+ __forceinline vint(PosInfTy) : v(_mm256_set_epi32(pos_inf,pos_inf,pos_inf,pos_inf,pos_inf,pos_inf,pos_inf,pos_inf)) {}
+ __forceinline vint(NegInfTy) : v(_mm256_set_epi32(neg_inf,neg_inf,neg_inf,neg_inf,neg_inf,neg_inf,neg_inf,neg_inf)) {}
+ __forceinline vint(StepTy) : v(_mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0)) {}
+ __forceinline vint(ReverseStepTy) : v(_mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7)) {}
+ __forceinline vint(UndefinedTy) : v(_mm256_undefined_si256()) {}
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Loads and Stores
+ ////////////////////////////////////////////////////////////////////////////////
+
+ static __forceinline vint8 load (const void* a) { return _mm256_castps_si256(_mm256_load_ps((float*)a)); }
+ static __forceinline vint8 loadu(const void* a) { return _mm256_castps_si256(_mm256_loadu_ps((float*)a)); }
+
+ static __forceinline vint8 load (const vboolf8& mask, const void* a) { return _mm256_castps_si256(_mm256_maskload_ps((float*)a,mask)); }
+ static __forceinline vint8 loadu(const vboolf8& mask, const void* a) { return _mm256_castps_si256(_mm256_maskload_ps((float*)a,mask)); }
+
+ static __forceinline void store (void* ptr, const vint8& f) { _mm256_store_ps((float*)ptr,_mm256_castsi256_ps(f)); }
+ static __forceinline void storeu(void* ptr, const vint8& f) { _mm256_storeu_ps((float*)ptr,_mm256_castsi256_ps(f)); }
+
+#if !defined(__aarch64__)
+ static __forceinline void store (const vboolf8& mask, void* ptr, const vint8& f) { _mm256_maskstore_ps((float*)ptr,(__m256i)mask,_mm256_castsi256_ps(f)); }
+ static __forceinline void storeu(const vboolf8& mask, void* ptr, const vint8& f) { _mm256_maskstore_ps((float*)ptr,(__m256i)mask,_mm256_castsi256_ps(f)); }
+#else
+ static __forceinline void store (const vboolf8& mask, void* ptr, const vint8& f) { _mm256_maskstore_ps((float*)ptr,(__m256i)mask.v,_mm256_castsi256_ps(f)); }
+ static __forceinline void storeu(const vboolf8& mask, void* ptr, const vint8& f) { _mm256_maskstore_ps((float*)ptr,(__m256i)mask.v,_mm256_castsi256_ps(f)); }
+#endif
+
+ static __forceinline void store_nt(void* ptr, const vint8& v) {
+ _mm256_stream_ps((float*)ptr,_mm256_castsi256_ps(v));
+ }
+
+ static __forceinline vint8 load(const uint8_t* ptr) {
+ vint4 il = vint4::load(ptr+0);
+ vint4 ih = vint4::load(ptr+4);
+ return vint8(il,ih);
+ }
+
+ static __forceinline vint8 loadu(const uint8_t* ptr) {
+ vint4 il = vint4::loadu(ptr+0);
+ vint4 ih = vint4::loadu(ptr+4);
+ return vint8(il,ih);
+ }
+
+ static __forceinline vint8 load(const unsigned short* ptr) {
+ vint4 il = vint4::load(ptr+0);
+ vint4 ih = vint4::load(ptr+4);
+ return vint8(il,ih);
+ }
+
+ static __forceinline vint8 loadu(const unsigned short* ptr) {
+ vint4 il = vint4::loadu(ptr+0);
+ vint4 ih = vint4::loadu(ptr+4);
+ return vint8(il,ih);
+ }
+
+ static __forceinline void store(uint8_t* ptr, const vint8& i) {
+ vint4 il(i.vl);
+ vint4 ih(i.vh);
+ vint4::store(ptr + 0,il);
+ vint4::store(ptr + 4,ih);
+ }
+
+ static __forceinline void store(unsigned short* ptr, const vint8& v) {
+ for (size_t i=0;i<8;i++)
+ ptr[i] = (unsigned short)v[i];
+ }
+
+ template<int scale = 4>
+ static __forceinline vint8 gather(const int* ptr, const vint8& index) {
+ return vint8(
+ *(int*)(((int8_t*)ptr)+scale*index[0]),
+ *(int*)(((int8_t*)ptr)+scale*index[1]),
+ *(int*)(((int8_t*)ptr)+scale*index[2]),
+ *(int*)(((int8_t*)ptr)+scale*index[3]),
+ *(int*)(((int8_t*)ptr)+scale*index[4]),
+ *(int*)(((int8_t*)ptr)+scale*index[5]),
+ *(int*)(((int8_t*)ptr)+scale*index[6]),
+ *(int*)(((int8_t*)ptr)+scale*index[7]));
+ }
+
+ template<int scale = 4>
+ static __forceinline vint8 gather(const vboolf8& mask, const int* ptr, const vint8& index) {
+ vint8 r = zero;
+ if (likely(mask[0])) r[0] = *(int*)(((int8_t*)ptr)+scale*index[0]);
+ if (likely(mask[1])) r[1] = *(int*)(((int8_t*)ptr)+scale*index[1]);
+ if (likely(mask[2])) r[2] = *(int*)(((int8_t*)ptr)+scale*index[2]);
+ if (likely(mask[3])) r[3] = *(int*)(((int8_t*)ptr)+scale*index[3]);
+ if (likely(mask[4])) r[4] = *(int*)(((int8_t*)ptr)+scale*index[4]);
+ if (likely(mask[5])) r[5] = *(int*)(((int8_t*)ptr)+scale*index[5]);
+ if (likely(mask[6])) r[6] = *(int*)(((int8_t*)ptr)+scale*index[6]);
+ if (likely(mask[7])) r[7] = *(int*)(((int8_t*)ptr)+scale*index[7]);
+ return r;
+ }
+
+ template<int scale = 4>
+ static __forceinline void scatter(void* ptr, const vint8& ofs, const vint8& v)
+ {
+ *(int*)(((int8_t*)ptr)+scale*ofs[0]) = v[0];
+ *(int*)(((int8_t*)ptr)+scale*ofs[1]) = v[1];
+ *(int*)(((int8_t*)ptr)+scale*ofs[2]) = v[2];
+ *(int*)(((int8_t*)ptr)+scale*ofs[3]) = v[3];
+ *(int*)(((int8_t*)ptr)+scale*ofs[4]) = v[4];
+ *(int*)(((int8_t*)ptr)+scale*ofs[5]) = v[5];
+ *(int*)(((int8_t*)ptr)+scale*ofs[6]) = v[6];
+ *(int*)(((int8_t*)ptr)+scale*ofs[7]) = v[7];
+ }
+
+ template<int scale = 4>
+ static __forceinline void scatter(const vboolf8& mask, void* ptr, const vint8& ofs, const vint8& v)
+ {
+ if (likely(mask[0])) *(int*)(((int8_t*)ptr)+scale*ofs[0]) = v[0];
+ if (likely(mask[1])) *(int*)(((int8_t*)ptr)+scale*ofs[1]) = v[1];
+ if (likely(mask[2])) *(int*)(((int8_t*)ptr)+scale*ofs[2]) = v[2];
+ if (likely(mask[3])) *(int*)(((int8_t*)ptr)+scale*ofs[3]) = v[3];
+ if (likely(mask[4])) *(int*)(((int8_t*)ptr)+scale*ofs[4]) = v[4];
+ if (likely(mask[5])) *(int*)(((int8_t*)ptr)+scale*ofs[5]) = v[5];
+ if (likely(mask[6])) *(int*)(((int8_t*)ptr)+scale*ofs[6]) = v[6];
+ if (likely(mask[7])) *(int*)(((int8_t*)ptr)+scale*ofs[7]) = v[7];
+ }
+
+
+ static __forceinline vint8 broadcast64(const long long& a) { return _mm256_set1_epi64x(a); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Array Access
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline const int& operator [](size_t index) const { assert(index < 8); return i[index]; }
+ __forceinline int& operator [](size_t index) { assert(index < 8); return i[index]; }
+ };
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Unary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboolf8 asBool(const vint8& a) { return _mm256_castsi256_ps(a); }
+
+ __forceinline vint8 operator +(const vint8& a) { return a; }
+ __forceinline vint8 operator -(const vint8& a) { return vint8(_mm_sub_epi32(_mm_setzero_si128(), a.vl), _mm_sub_epi32(_mm_setzero_si128(), a.vh)); }
+ __forceinline vint8 abs (const vint8& a) { return vint8(_mm_abs_epi32(a.vl), _mm_abs_epi32(a.vh)); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Binary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vint8 operator +(const vint8& a, const vint8& b) { return vint8(_mm_add_epi32(a.vl, b.vl), _mm_add_epi32(a.vh, b.vh)); }
+ __forceinline vint8 operator +(const vint8& a, int b) { return a + vint8(b); }
+ __forceinline vint8 operator +(int a, const vint8& b) { return vint8(a) + b; }
+
+ __forceinline vint8 operator -(const vint8& a, const vint8& b) { return vint8(_mm_sub_epi32(a.vl, b.vl), _mm_sub_epi32(a.vh, b.vh)); }
+ __forceinline vint8 operator -(const vint8& a, int b) { return a - vint8(b); }
+ __forceinline vint8 operator -(int a, const vint8& b) { return vint8(a) - b; }
+
+ __forceinline vint8 operator *(const vint8& a, const vint8& b) { return vint8(_mm_mullo_epi32(a.vl, b.vl), _mm_mullo_epi32(a.vh, b.vh)); }
+ __forceinline vint8 operator *(const vint8& a, int b) { return a * vint8(b); }
+ __forceinline vint8 operator *(int a, const vint8& b) { return vint8(a) * b; }
+
+ __forceinline vint8 operator &(const vint8& a, const vint8& b) { return _mm256_castps_si256(_mm256_and_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b))); }
+ __forceinline vint8 operator &(const vint8& a, int b) { return a & vint8(b); }
+ __forceinline vint8 operator &(int a, const vint8& b) { return vint8(a) & b; }
+
+ __forceinline vint8 operator |(const vint8& a, const vint8& b) { return _mm256_castps_si256(_mm256_or_ps (_mm256_castsi256_ps(a), _mm256_castsi256_ps(b))); }
+ __forceinline vint8 operator |(const vint8& a, int b) { return a | vint8(b); }
+ __forceinline vint8 operator |(int a, const vint8& b) { return vint8(a) | b; }
+
+ __forceinline vint8 operator ^(const vint8& a, const vint8& b) { return _mm256_castps_si256(_mm256_xor_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b))); }
+ __forceinline vint8 operator ^(const vint8& a, int b) { return a ^ vint8(b); }
+ __forceinline vint8 operator ^(int a, const vint8& b) { return vint8(a) ^ b; }
+
+ __forceinline vint8 operator <<(const vint8& a, int n) { return vint8(_mm_slli_epi32(a.vl, n), _mm_slli_epi32(a.vh, n)); }
+ __forceinline vint8 operator >>(const vint8& a, int n) { return vint8(_mm_srai_epi32(a.vl, n), _mm_srai_epi32(a.vh, n)); }
+
+ __forceinline vint8 sll (const vint8& a, int b) { return vint8(_mm_slli_epi32(a.vl, b), _mm_slli_epi32(a.vh, b)); }
+ __forceinline vint8 sra (const vint8& a, int b) { return vint8(_mm_srai_epi32(a.vl, b), _mm_srai_epi32(a.vh, b)); }
+ __forceinline vint8 srl (const vint8& a, int b) { return vint8(_mm_srli_epi32(a.vl, b), _mm_srli_epi32(a.vh, b)); }
+
+ __forceinline vint8 min(const vint8& a, const vint8& b) { return vint8(_mm_min_epi32(a.vl, b.vl), _mm_min_epi32(a.vh, b.vh)); }
+ __forceinline vint8 min(const vint8& a, int b) { return min(a,vint8(b)); }
+ __forceinline vint8 min(int a, const vint8& b) { return min(vint8(a),b); }
+
+ __forceinline vint8 max(const vint8& a, const vint8& b) { return vint8(_mm_max_epi32(a.vl, b.vl), _mm_max_epi32(a.vh, b.vh)); }
+ __forceinline vint8 max(const vint8& a, int b) { return max(a,vint8(b)); }
+ __forceinline vint8 max(int a, const vint8& b) { return max(vint8(a),b); }
+
+ __forceinline vint8 umin(const vint8& a, const vint8& b) { return vint8(_mm_min_epu32(a.vl, b.vl), _mm_min_epu32(a.vh, b.vh)); }
+ __forceinline vint8 umin(const vint8& a, int b) { return umin(a,vint8(b)); }
+ __forceinline vint8 umin(int a, const vint8& b) { return umin(vint8(a),b); }
+
+ __forceinline vint8 umax(const vint8& a, const vint8& b) { return vint8(_mm_max_epu32(a.vl, b.vl), _mm_max_epu32(a.vh, b.vh)); }
+ __forceinline vint8 umax(const vint8& a, int b) { return umax(a,vint8(b)); }
+ __forceinline vint8 umax(int a, const vint8& b) { return umax(vint8(a),b); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Assignment Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vint8& operator +=(vint8& a, const vint8& b) { return a = a + b; }
+ __forceinline vint8& operator +=(vint8& a, int b) { return a = a + b; }
+
+ __forceinline vint8& operator -=(vint8& a, const vint8& b) { return a = a - b; }
+ __forceinline vint8& operator -=(vint8& a, int b) { return a = a - b; }
+
+ __forceinline vint8& operator *=(vint8& a, const vint8& b) { return a = a * b; }
+ __forceinline vint8& operator *=(vint8& a, int b) { return a = a * b; }
+
+ __forceinline vint8& operator &=(vint8& a, const vint8& b) { return a = a & b; }
+ __forceinline vint8& operator &=(vint8& a, int b) { return a = a & b; }
+
+ __forceinline vint8& operator |=(vint8& a, const vint8& b) { return a = a | b; }
+ __forceinline vint8& operator |=(vint8& a, int b) { return a = a | b; }
+
+ __forceinline vint8& operator <<=(vint8& a, int b) { return a = a << b; }
+ __forceinline vint8& operator >>=(vint8& a, int b) { return a = a >> b; }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Comparison Operators + Select
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboolf8 operator ==(const vint8& a, const vint8& b) { return vboolf8(_mm_castsi128_ps(_mm_cmpeq_epi32 (a.vl, b.vl)),
+ _mm_castsi128_ps(_mm_cmpeq_epi32 (a.vh, b.vh))); }
+ __forceinline vboolf8 operator ==(const vint8& a, int b) { return a == vint8(b); }
+ __forceinline vboolf8 operator ==(int a, const vint8& b) { return vint8(a) == b; }
+
+ __forceinline vboolf8 operator !=(const vint8& a, const vint8& b) { return !(a == b); }
+ __forceinline vboolf8 operator !=(const vint8& a, int b) { return a != vint8(b); }
+ __forceinline vboolf8 operator !=(int a, const vint8& b) { return vint8(a) != b; }
+
+ __forceinline vboolf8 operator < (const vint8& a, const vint8& b) { return vboolf8(_mm_castsi128_ps(_mm_cmplt_epi32 (a.vl, b.vl)),
+ _mm_castsi128_ps(_mm_cmplt_epi32 (a.vh, b.vh))); }
+ __forceinline vboolf8 operator < (const vint8& a, int b) { return a < vint8(b); }
+ __forceinline vboolf8 operator < (int a, const vint8& b) { return vint8(a) < b; }
+
+ __forceinline vboolf8 operator >=(const vint8& a, const vint8& b) { return !(a < b); }
+ __forceinline vboolf8 operator >=(const vint8& a, int b) { return a >= vint8(b); }
+ __forceinline vboolf8 operator >=(int a, const vint8& b) { return vint8(a) >= b; }
+
+ __forceinline vboolf8 operator > (const vint8& a, const vint8& b) { return vboolf8(_mm_castsi128_ps(_mm_cmpgt_epi32 (a.vl, b.vl)),
+ _mm_castsi128_ps(_mm_cmpgt_epi32 (a.vh, b.vh))); }
+ __forceinline vboolf8 operator > (const vint8& a, int b) { return a > vint8(b); }
+ __forceinline vboolf8 operator > (int a, const vint8& b) { return vint8(a) > b; }
+
+ __forceinline vboolf8 operator <=(const vint8& a, const vint8& b) { return !(a > b); }
+ __forceinline vboolf8 operator <=(const vint8& a, int b) { return a <= vint8(b); }
+ __forceinline vboolf8 operator <=(int a, const vint8& b) { return vint8(a) <= b; }
+
+ __forceinline vboolf8 eq(const vint8& a, const vint8& b) { return a == b; }
+ __forceinline vboolf8 ne(const vint8& a, const vint8& b) { return a != b; }
+ __forceinline vboolf8 lt(const vint8& a, const vint8& b) { return a < b; }
+ __forceinline vboolf8 ge(const vint8& a, const vint8& b) { return a >= b; }
+ __forceinline vboolf8 gt(const vint8& a, const vint8& b) { return a > b; }
+ __forceinline vboolf8 le(const vint8& a, const vint8& b) { return a <= b; }
+
+ __forceinline vboolf8 eq(const vboolf8& mask, const vint8& a, const vint8& b) { return mask & (a == b); }
+ __forceinline vboolf8 ne(const vboolf8& mask, const vint8& a, const vint8& b) { return mask & (a != b); }
+ __forceinline vboolf8 lt(const vboolf8& mask, const vint8& a, const vint8& b) { return mask & (a < b); }
+ __forceinline vboolf8 ge(const vboolf8& mask, const vint8& a, const vint8& b) { return mask & (a >= b); }
+ __forceinline vboolf8 gt(const vboolf8& mask, const vint8& a, const vint8& b) { return mask & (a > b); }
+ __forceinline vboolf8 le(const vboolf8& mask, const vint8& a, const vint8& b) { return mask & (a <= b); }
+
+ __forceinline vint8 select(const vboolf8& m, const vint8& t, const vint8& f) {
+ return _mm256_castps_si256(_mm256_blendv_ps(_mm256_castsi256_ps(f), _mm256_castsi256_ps(t), m));
+ }
+
+ __forceinline vint8 notand(const vboolf8& m, const vint8& f) {
+ return _mm256_castps_si256(_mm256_andnot_ps(m, _mm256_castsi256_ps(f)));
+ }
+
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Movement/Shifting/Shuffling Functions
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vint8 unpacklo(const vint8& a, const vint8& b) { return _mm256_castps_si256(_mm256_unpacklo_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b))); }
+ __forceinline vint8 unpackhi(const vint8& a, const vint8& b) { return _mm256_castps_si256(_mm256_unpackhi_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b))); }
+
+ template<int i>
+ __forceinline vint8 shuffle(const vint8& v) {
+ return _mm256_castps_si256(_mm256_permute_ps(_mm256_castsi256_ps(v), _MM_SHUFFLE(i, i, i, i)));
+ }
+
+ template<int i0, int i1>
+ __forceinline vint8 shuffle4(const vint8& v) {
+ return _mm256_permute2f128_si256(v, v, (i1 << 4) | (i0 << 0));
+ }
+
+ template<int i0, int i1>
+ __forceinline vint8 shuffle4(const vint8& a, const vint8& b) {
+ return _mm256_permute2f128_si256(a, b, (i1 << 4) | (i0 << 0));
+ }
+
+ template<int i0, int i1, int i2, int i3>
+ __forceinline vint8 shuffle(const vint8& v) {
+ return _mm256_castps_si256(_mm256_permute_ps(_mm256_castsi256_ps(v), _MM_SHUFFLE(i3, i2, i1, i0)));
+ }
+
+ template<int i0, int i1, int i2, int i3>
+ __forceinline vint8 shuffle(const vint8& a, const vint8& b) {
+ return _mm256_castps_si256(_mm256_shuffle_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b), _MM_SHUFFLE(i3, i2, i1, i0)));
+ }
+
+ template<> __forceinline vint8 shuffle<0, 0, 2, 2>(const vint8& v) { return _mm256_castps_si256(_mm256_moveldup_ps(_mm256_castsi256_ps(v))); }
+ template<> __forceinline vint8 shuffle<1, 1, 3, 3>(const vint8& v) { return _mm256_castps_si256(_mm256_movehdup_ps(_mm256_castsi256_ps(v))); }
+ template<> __forceinline vint8 shuffle<0, 1, 0, 1>(const vint8& v) { return _mm256_castps_si256(_mm256_castpd_ps(_mm256_movedup_pd(_mm256_castps_pd(_mm256_castsi256_ps(v))))); }
+
+ __forceinline vint8 broadcast(const int* ptr) { return _mm256_castps_si256(_mm256_broadcast_ss((const float*)ptr)); }
+ template<int i> __forceinline vint8 insert4(const vint8& a, const vint4& b) { return _mm256_insertf128_si256(a, b, i); }
+ template<int i> __forceinline vint4 extract4(const vint8& a) { return _mm256_extractf128_si256(a, i); }
+ template<> __forceinline vint4 extract4<0>(const vint8& a) { return _mm256_castsi256_si128(a); }
+
+ __forceinline int toScalar(const vint8& v) { return _mm_cvtsi128_si32(_mm256_castsi256_si128(v)); }
+
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Reductions
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vint8 vreduce_min2(const vint8& v) { return min(v,shuffle<1,0,3,2>(v)); }
+ __forceinline vint8 vreduce_min4(const vint8& v) { vint8 v1 = vreduce_min2(v); return min(v1,shuffle<2,3,0,1>(v1)); }
+ __forceinline vint8 vreduce_min (const vint8& v) { vint8 v1 = vreduce_min4(v); return min(v1,shuffle4<1,0>(v1)); }
+
+ __forceinline vint8 vreduce_max2(const vint8& v) { return max(v,shuffle<1,0,3,2>(v)); }
+ __forceinline vint8 vreduce_max4(const vint8& v) { vint8 v1 = vreduce_max2(v); return max(v1,shuffle<2,3,0,1>(v1)); }
+ __forceinline vint8 vreduce_max (const vint8& v) { vint8 v1 = vreduce_max4(v); return max(v1,shuffle4<1,0>(v1)); }
+
+ __forceinline vint8 vreduce_add2(const vint8& v) { return v + shuffle<1,0,3,2>(v); }
+ __forceinline vint8 vreduce_add4(const vint8& v) { vint8 v1 = vreduce_add2(v); return v1 + shuffle<2,3,0,1>(v1); }
+ __forceinline vint8 vreduce_add (const vint8& v) { vint8 v1 = vreduce_add4(v); return v1 + shuffle4<1,0>(v1); }
+
+ __forceinline int reduce_min(const vint8& v) { return toScalar(vreduce_min(v)); }
+ __forceinline int reduce_max(const vint8& v) { return toScalar(vreduce_max(v)); }
+ __forceinline int reduce_add(const vint8& v) { return toScalar(vreduce_add(v)); }
+
+ __forceinline size_t select_min(const vint8& v) { return bsf(movemask(v == vreduce_min(v))); }
+ __forceinline size_t select_max(const vint8& v) { return bsf(movemask(v == vreduce_max(v))); }
+
+ __forceinline size_t select_min(const vboolf8& valid, const vint8& v) { const vint8 a = select(valid,v,vint8(pos_inf)); return bsf(movemask(valid & (a == vreduce_min(a)))); }
+ __forceinline size_t select_max(const vboolf8& valid, const vint8& v) { const vint8 a = select(valid,v,vint8(neg_inf)); return bsf(movemask(valid & (a == vreduce_max(a)))); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Sorting networks
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vint8 usort_ascending(const vint8& v)
+ {
+ const vint8 a0 = v;
+ const vint8 b0 = shuffle<1,0,3,2>(a0);
+ const vint8 c0 = umin(a0,b0);
+ const vint8 d0 = umax(a0,b0);
+ const vint8 a1 = select(0x99 /* 0b10011001 */,c0,d0);
+ const vint8 b1 = shuffle<2,3,0,1>(a1);
+ const vint8 c1 = umin(a1,b1);
+ const vint8 d1 = umax(a1,b1);
+ const vint8 a2 = select(0xc3 /* 0b11000011 */,c1,d1);
+ const vint8 b2 = shuffle<1,0,3,2>(a2);
+ const vint8 c2 = umin(a2,b2);
+ const vint8 d2 = umax(a2,b2);
+ const vint8 a3 = select(0xa5 /* 0b10100101 */,c2,d2);
+ const vint8 b3 = shuffle4<1,0>(a3);
+ const vint8 c3 = umin(a3,b3);
+ const vint8 d3 = umax(a3,b3);
+ const vint8 a4 = select(0xf /* 0b00001111 */,c3,d3);
+ const vint8 b4 = shuffle<2,3,0,1>(a4);
+ const vint8 c4 = umin(a4,b4);
+ const vint8 d4 = umax(a4,b4);
+ const vint8 a5 = select(0x33 /* 0b00110011 */,c4,d4);
+ const vint8 b5 = shuffle<1,0,3,2>(a5);
+ const vint8 c5 = umin(a5,b5);
+ const vint8 d5 = umax(a5,b5);
+ const vint8 a6 = select(0x55 /* 0b01010101 */,c5,d5);
+ return a6;
+ }
+
+ __forceinline vint8 usort_descending(const vint8& v)
+ {
+ const vint8 a0 = v;
+ const vint8 b0 = shuffle<1,0,3,2>(a0);
+ const vint8 c0 = umax(a0,b0);
+ const vint8 d0 = umin(a0,b0);
+ const vint8 a1 = select(0x99 /* 0b10011001 */,c0,d0);
+ const vint8 b1 = shuffle<2,3,0,1>(a1);
+ const vint8 c1 = umax(a1,b1);
+ const vint8 d1 = umin(a1,b1);
+ const vint8 a2 = select(0xc3 /* 0b11000011 */,c1,d1);
+ const vint8 b2 = shuffle<1,0,3,2>(a2);
+ const vint8 c2 = umax(a2,b2);
+ const vint8 d2 = umin(a2,b2);
+ const vint8 a3 = select(0xa5 /* 0b10100101 */,c2,d2);
+ const vint8 b3 = shuffle4<1,0>(a3);
+ const vint8 c3 = umax(a3,b3);
+ const vint8 d3 = umin(a3,b3);
+ const vint8 a4 = select(0xf /* 0b00001111 */,c3,d3);
+ const vint8 b4 = shuffle<2,3,0,1>(a4);
+ const vint8 c4 = umax(a4,b4);
+ const vint8 d4 = umin(a4,b4);
+ const vint8 a5 = select(0x33 /* 0b00110011 */,c4,d4);
+ const vint8 b5 = shuffle<1,0,3,2>(a5);
+ const vint8 c5 = umax(a5,b5);
+ const vint8 d5 = umin(a5,b5);
+ const vint8 a6 = select(0x55 /* 0b01010101 */,c5,d5);
+ return a6;
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Output Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline embree_ostream operator <<(embree_ostream cout, const vint8& a) {
+ return cout << "<" << a[0] << ", " << a[1] << ", " << a[2] << ", " << a[3] << ", " << a[4] << ", " << a[5] << ", " << a[6] << ", " << a[7] << ">";
+ }
+}
diff --git a/thirdparty/embree-aarch64/common/simd/vint8_avx2.h b/thirdparty/embree-aarch64/common/simd/vint8_avx2.h
new file mode 100644
index 0000000000..4937d972cf
--- /dev/null
+++ b/thirdparty/embree-aarch64/common/simd/vint8_avx2.h
@@ -0,0 +1,512 @@
+// Copyright 2009-2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+namespace embree
+{
+ /* 8-wide AVX integer type */
+ template<>
+ struct vint<8>
+ {
+ ALIGNED_STRUCT_(32);
+
+ typedef vboolf8 Bool;
+ typedef vint8 Int;
+ typedef vfloat8 Float;
+
+ enum { size = 8 }; // number of SIMD elements
+ union { // data
+ __m256i v;
+ int i[8];
+ };
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constructors, Assignment & Cast Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vint() {}
+ __forceinline vint(const vint8& a) { v = a.v; }
+ __forceinline vint8& operator =(const vint8& a) { v = a.v; return *this; }
+
+ __forceinline vint(__m256i a) : v(a) {}
+ __forceinline operator const __m256i&() const { return v; }
+ __forceinline operator __m256i&() { return v; }
+
+ __forceinline explicit vint(const vint4& a) : v(_mm256_insertf128_si256(_mm256_castsi128_si256(a),a,1)) {}
+ __forceinline vint(const vint4& a, const vint4& b) : v(_mm256_insertf128_si256(_mm256_castsi128_si256(a),b,1)) {}
+ __forceinline vint(const __m128i& a, const __m128i& b) : v(_mm256_insertf128_si256(_mm256_castsi128_si256(a),b,1)) {}
+
+ __forceinline explicit vint(const int* a) : v(_mm256_castps_si256(_mm256_loadu_ps((const float*)a))) {}
+ __forceinline vint(int a) : v(_mm256_set1_epi32(a)) {}
+ __forceinline vint(int a, int b) : v(_mm256_set_epi32(b, a, b, a, b, a, b, a)) {}
+ __forceinline vint(int a, int b, int c, int d) : v(_mm256_set_epi32(d, c, b, a, d, c, b, a)) {}
+ __forceinline vint(int a, int b, int c, int d, int e, int f, int g, int h) : v(_mm256_set_epi32(h, g, f, e, d, c, b, a)) {}
+
+ __forceinline explicit vint(__m256 a) : v(_mm256_cvtps_epi32(a)) {}
+
+#if defined(__AVX512VL__)
+ __forceinline explicit vint(const vboolf8& a) : v(_mm256_movm_epi32(a)) {}
+#else
+ __forceinline explicit vint(const vboolf8& a) : v(_mm256_castps_si256((__m256)a)) {}
+#endif
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constants
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vint(ZeroTy) : v(_mm256_setzero_si256()) {}
+ __forceinline vint(OneTy) : v(_mm256_set1_epi32(1)) {}
+ __forceinline vint(PosInfTy) : v(_mm256_set1_epi32(pos_inf)) {}
+ __forceinline vint(NegInfTy) : v(_mm256_set1_epi32(neg_inf)) {}
+ __forceinline vint(StepTy) : v(_mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0)) {}
+ __forceinline vint(ReverseStepTy) : v(_mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7)) {}
+ __forceinline vint(UndefinedTy) : v(_mm256_undefined_si256()) {}
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Loads and Stores
+ ////////////////////////////////////////////////////////////////////////////////
+
+ static __forceinline vint8 load(const uint8_t* ptr) { return _mm256_cvtepu8_epi32(_mm_loadl_epi64((__m128i*)ptr)); }
+ static __forceinline vint8 loadu(const uint8_t* ptr) { return _mm256_cvtepu8_epi32(_mm_loadl_epi64((__m128i*)ptr)); }
+ static __forceinline vint8 load(const unsigned short* ptr) { return _mm256_cvtepu16_epi32(_mm_load_si128((__m128i*)ptr)); }
+ static __forceinline vint8 loadu(const unsigned short* ptr) { return _mm256_cvtepu16_epi32(_mm_loadu_si128((__m128i*)ptr)); }
+
+ static __forceinline vint8 load(const void* ptr) { return _mm256_load_si256((__m256i*)ptr); }
+ static __forceinline vint8 loadu(const void* ptr) { return _mm256_loadu_si256((__m256i*)ptr); }
+
+ static __forceinline void store (void* ptr, const vint8& v) { _mm256_store_si256((__m256i*)ptr,v); }
+ static __forceinline void storeu(void* ptr, const vint8& v) { _mm256_storeu_ps((float*)ptr,_mm256_castsi256_ps(v)); }
+
+#if defined(__AVX512VL__)
+
+ static __forceinline vint8 compact(const vboolf8& mask, vint8 &v) {
+ return _mm256_mask_compress_epi32(v, mask, v);
+ }
+ static __forceinline vint8 compact(const vboolf8& mask, vint8 &a, const vint8& b) {
+ return _mm256_mask_compress_epi32(a, mask, b);
+ }
+
+ static __forceinline vint8 load (const vboolf8& mask, const void* ptr) { return _mm256_mask_load_epi32 (_mm256_setzero_si256(),mask,ptr); }
+ static __forceinline vint8 loadu(const vboolf8& mask, const void* ptr) { return _mm256_mask_loadu_epi32(_mm256_setzero_si256(),mask,ptr); }
+
+ static __forceinline void store (const vboolf8& mask, void* ptr, const vint8& v) { _mm256_mask_store_epi32 (ptr,mask,v); }
+ static __forceinline void storeu(const vboolf8& mask, void* ptr, const vint8& v) { _mm256_mask_storeu_epi32(ptr,mask,v); }
+#else
+ static __forceinline vint8 load (const vboolf8& mask, const void* ptr) { return _mm256_castps_si256(_mm256_maskload_ps((float*)ptr,mask)); }
+ static __forceinline vint8 loadu(const vboolf8& mask, const void* ptr) { return _mm256_castps_si256(_mm256_maskload_ps((float*)ptr,mask)); }
+
+ static __forceinline void store (const vboolf8& mask, void* ptr, const vint8& v) { _mm256_maskstore_epi32((int*)ptr,mask,v); }
+ static __forceinline void storeu(const vboolf8& mask, void* ptr, const vint8& v) { _mm256_maskstore_epi32((int*)ptr,mask,v); }
+#endif
+
+ static __forceinline vint8 load_nt(void* ptr) {
+ return _mm256_stream_load_si256((__m256i*)ptr);
+ }
+
+ static __forceinline void store_nt(void* ptr, const vint8& v) {
+ _mm256_stream_ps((float*)ptr,_mm256_castsi256_ps(v));
+ }
+
+ static __forceinline void store(uint8_t* ptr, const vint8& i)
+ {
+ for (size_t j=0; j<8; j++)
+ ptr[j] = i[j];
+ }
+
+ static __forceinline void store(unsigned short* ptr, const vint8& v) {
+ for (size_t i=0;i<8;i++)
+ ptr[i] = (unsigned short)v[i];
+ }
+
+ template<int scale = 4>
+ static __forceinline vint8 gather(const int *const ptr, const vint8& index) {
+ return _mm256_i32gather_epi32(ptr, index, scale);
+ }
+
+ template<int scale = 4>
+ static __forceinline vint8 gather(const vboolf8& mask, const int *const ptr, const vint8& index) {
+ vint8 r = zero;
+#if defined(__AVX512VL__)
+ return _mm256_mmask_i32gather_epi32(r, mask, index, ptr, scale);
+#else
+ return _mm256_mask_i32gather_epi32(r, ptr, index, mask, scale);
+#endif
+ }
+
+ template<int scale = 4>
+ static __forceinline void scatter(void* ptr, const vint8& ofs, const vint8& v)
+ {
+#if defined(__AVX512VL__)
+ _mm256_i32scatter_epi32((int*)ptr, ofs, v, scale);
+#else
+ *(int*)(((int8_t*)ptr)+scale*ofs[0]) = v[0];
+ *(int*)(((int8_t*)ptr)+scale*ofs[1]) = v[1];
+ *(int*)(((int8_t*)ptr)+scale*ofs[2]) = v[2];
+ *(int*)(((int8_t*)ptr)+scale*ofs[3]) = v[3];
+ *(int*)(((int8_t*)ptr)+scale*ofs[4]) = v[4];
+ *(int*)(((int8_t*)ptr)+scale*ofs[5]) = v[5];
+ *(int*)(((int8_t*)ptr)+scale*ofs[6]) = v[6];
+ *(int*)(((int8_t*)ptr)+scale*ofs[7]) = v[7];
+#endif
+ }
+
+ template<int scale = 4>
+ static __forceinline void scatter(const vboolf8& mask, void* ptr, const vint8& ofs, const vint8& v)
+ {
+#if defined(__AVX512VL__)
+ _mm256_mask_i32scatter_epi32((int*)ptr, mask, ofs, v, scale);
+#else
+ if (likely(mask[0])) *(int*)(((int8_t*)ptr)+scale*ofs[0]) = v[0];
+ if (likely(mask[1])) *(int*)(((int8_t*)ptr)+scale*ofs[1]) = v[1];
+ if (likely(mask[2])) *(int*)(((int8_t*)ptr)+scale*ofs[2]) = v[2];
+ if (likely(mask[3])) *(int*)(((int8_t*)ptr)+scale*ofs[3]) = v[3];
+ if (likely(mask[4])) *(int*)(((int8_t*)ptr)+scale*ofs[4]) = v[4];
+ if (likely(mask[5])) *(int*)(((int8_t*)ptr)+scale*ofs[5]) = v[5];
+ if (likely(mask[6])) *(int*)(((int8_t*)ptr)+scale*ofs[6]) = v[6];
+ if (likely(mask[7])) *(int*)(((int8_t*)ptr)+scale*ofs[7]) = v[7];
+#endif
+ }
+
+ static __forceinline vint8 broadcast64(const long long &a) { return _mm256_set1_epi64x(a); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Array Access
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline const int& operator [](size_t index) const { assert(index < 8); return i[index]; }
+ __forceinline int& operator [](size_t index) { assert(index < 8); return i[index]; }
+ };
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Unary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+#if defined(__AVX512VL__)
+ static __forceinline vboolf8 asBool(const vint8& a) { return _mm256_movepi32_mask(a); }
+#else
+ static __forceinline vboolf8 asBool(const vint8& a) { return _mm256_castsi256_ps(a); }
+#endif
+
+ __forceinline vint8 operator +(const vint8& a) { return a; }
+ __forceinline vint8 operator -(const vint8& a) { return _mm256_sub_epi32(_mm256_setzero_si256(), a); }
+ __forceinline vint8 abs (const vint8& a) { return _mm256_abs_epi32(a); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Binary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vint8 operator +(const vint8& a, const vint8& b) { return _mm256_add_epi32(a, b); }
+ __forceinline vint8 operator +(const vint8& a, int b) { return a + vint8(b); }
+ __forceinline vint8 operator +(int a, const vint8& b) { return vint8(a) + b; }
+
+ __forceinline vint8 operator -(const vint8& a, const vint8& b) { return _mm256_sub_epi32(a, b); }
+ __forceinline vint8 operator -(const vint8& a, int b) { return a - vint8(b); }
+ __forceinline vint8 operator -(int a, const vint8& b) { return vint8(a) - b; }
+
+ __forceinline vint8 operator *(const vint8& a, const vint8& b) { return _mm256_mullo_epi32(a, b); }
+ __forceinline vint8 operator *(const vint8& a, int b) { return a * vint8(b); }
+ __forceinline vint8 operator *(int a, const vint8& b) { return vint8(a) * b; }
+
+ __forceinline vint8 operator &(const vint8& a, const vint8& b) { return _mm256_and_si256(a, b); }
+ __forceinline vint8 operator &(const vint8& a, int b) { return a & vint8(b); }
+ __forceinline vint8 operator &(int a, const vint8& b) { return vint8(a) & b; }
+
+ __forceinline vint8 operator |(const vint8& a, const vint8& b) { return _mm256_or_si256(a, b); }
+ __forceinline vint8 operator |(const vint8& a, int b) { return a | vint8(b); }
+ __forceinline vint8 operator |(int a, const vint8& b) { return vint8(a) | b; }
+
+ __forceinline vint8 operator ^(const vint8& a, const vint8& b) { return _mm256_xor_si256(a, b); }
+ __forceinline vint8 operator ^(const vint8& a, int b) { return a ^ vint8(b); }
+ __forceinline vint8 operator ^(int a, const vint8& b) { return vint8(a) ^ b; }
+
+ __forceinline vint8 operator <<(const vint8& a, int n) { return _mm256_slli_epi32(a, n); }
+ __forceinline vint8 operator >>(const vint8& a, int n) { return _mm256_srai_epi32(a, n); }
+
+ __forceinline vint8 operator <<(const vint8& a, const vint8& n) { return _mm256_sllv_epi32(a, n); }
+ __forceinline vint8 operator >>(const vint8& a, const vint8& n) { return _mm256_srav_epi32(a, n); }
+
+ __forceinline vint8 sll(const vint8& a, int b) { return _mm256_slli_epi32(a, b); }
+ __forceinline vint8 sra(const vint8& a, int b) { return _mm256_srai_epi32(a, b); }
+ __forceinline vint8 srl(const vint8& a, int b) { return _mm256_srli_epi32(a, b); }
+
+ __forceinline vint8 sll(const vint8& a, const vint8& b) { return _mm256_sllv_epi32(a, b); }
+ __forceinline vint8 sra(const vint8& a, const vint8& b) { return _mm256_srav_epi32(a, b); }
+ __forceinline vint8 srl(const vint8& a, const vint8& b) { return _mm256_srlv_epi32(a, b); }
+
+ __forceinline vint8 min(const vint8& a, const vint8& b) { return _mm256_min_epi32(a, b); }
+ __forceinline vint8 min(const vint8& a, int b) { return min(a,vint8(b)); }
+ __forceinline vint8 min(int a, const vint8& b) { return min(vint8(a),b); }
+
+ __forceinline vint8 max(const vint8& a, const vint8& b) { return _mm256_max_epi32(a, b); }
+ __forceinline vint8 max(const vint8& a, int b) { return max(a,vint8(b)); }
+ __forceinline vint8 max(int a, const vint8& b) { return max(vint8(a),b); }
+
+ __forceinline vint8 umin(const vint8& a, const vint8& b) { return _mm256_min_epu32(a, b); }
+ __forceinline vint8 umax(const vint8& a, const vint8& b) { return _mm256_max_epu32(a, b); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Assignment Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vint8& operator +=(vint8& a, const vint8& b) { return a = a + b; }
+ __forceinline vint8& operator +=(vint8& a, int b) { return a = a + b; }
+
+ __forceinline vint8& operator -=(vint8& a, const vint8& b) { return a = a - b; }
+ __forceinline vint8& operator -=(vint8& a, int b) { return a = a - b; }
+
+ __forceinline vint8& operator *=(vint8& a, const vint8& b) { return a = a * b; }
+ __forceinline vint8& operator *=(vint8& a, int b) { return a = a * b; }
+
+ __forceinline vint8& operator &=(vint8& a, const vint8& b) { return a = a & b; }
+ __forceinline vint8& operator &=(vint8& a, int b) { return a = a & b; }
+
+ __forceinline vint8& operator |=(vint8& a, const vint8& b) { return a = a | b; }
+ __forceinline vint8& operator |=(vint8& a, int b) { return a = a | b; }
+
+ __forceinline vint8& operator <<=(vint8& a, const int b) { return a = a << b; }
+ __forceinline vint8& operator >>=(vint8& a, const int b) { return a = a >> b; }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Comparison Operators + Select
+ ////////////////////////////////////////////////////////////////////////////////
+
+#if defined(__AVX512VL__)
+ static __forceinline vboolf8 operator ==(const vint8& a, const vint8& b) { return _mm256_cmp_epi32_mask(a,b,_MM_CMPINT_EQ); }
+ static __forceinline vboolf8 operator !=(const vint8& a, const vint8& b) { return _mm256_cmp_epi32_mask(a,b,_MM_CMPINT_NE); }
+ static __forceinline vboolf8 operator < (const vint8& a, const vint8& b) { return _mm256_cmp_epi32_mask(a,b,_MM_CMPINT_LT); }
+ static __forceinline vboolf8 operator >=(const vint8& a, const vint8& b) { return _mm256_cmp_epi32_mask(a,b,_MM_CMPINT_GE); }
+ static __forceinline vboolf8 operator > (const vint8& a, const vint8& b) { return _mm256_cmp_epi32_mask(a,b,_MM_CMPINT_GT); }
+ static __forceinline vboolf8 operator <=(const vint8& a, const vint8& b) { return _mm256_cmp_epi32_mask(a,b,_MM_CMPINT_LE); }
+
+ static __forceinline vint8 select(const vboolf8& m, const vint8& t, const vint8& f) {
+ return _mm256_mask_blend_epi32(m, (__m256i)f, (__m256i)t);
+ }
+#else
+ static __forceinline vboolf8 operator ==(const vint8& a, const vint8& b) { return _mm256_castsi256_ps(_mm256_cmpeq_epi32(a, b)); }
+ static __forceinline vboolf8 operator !=(const vint8& a, const vint8& b) { return !(a == b); }
+ static __forceinline vboolf8 operator < (const vint8& a, const vint8& b) { return _mm256_castsi256_ps(_mm256_cmpgt_epi32(b, a)); }
+ static __forceinline vboolf8 operator >=(const vint8& a, const vint8& b) { return !(a < b); }
+ static __forceinline vboolf8 operator > (const vint8& a, const vint8& b) { return _mm256_castsi256_ps(_mm256_cmpgt_epi32(a, b)); }
+ static __forceinline vboolf8 operator <=(const vint8& a, const vint8& b) { return !(a > b); }
+
+ static __forceinline vint8 select(const vboolf8& m, const vint8& t, const vint8& f) {
+ return _mm256_castps_si256(_mm256_blendv_ps(_mm256_castsi256_ps(f), _mm256_castsi256_ps(t), m));
+ }
+#endif
+
+ template<int mask>
+ __forceinline vint8 select(const vint8& t, const vint8& f) {
+ return _mm256_blend_epi32(f, t, mask);
+ }
+
+ __forceinline vboolf8 operator ==(const vint8& a, int b) { return a == vint8(b); }
+ __forceinline vboolf8 operator ==(int a, const vint8& b) { return vint8(a) == b; }
+
+ __forceinline vboolf8 operator !=(const vint8& a, int b) { return a != vint8(b); }
+ __forceinline vboolf8 operator !=(int a, const vint8& b) { return vint8(a) != b; }
+
+ __forceinline vboolf8 operator < (const vint8& a, int b) { return a < vint8(b); }
+ __forceinline vboolf8 operator < (int a, const vint8& b) { return vint8(a) < b; }
+
+ __forceinline vboolf8 operator >=(const vint8& a, int b) { return a >= vint8(b); }
+ __forceinline vboolf8 operator >=(int a, const vint8& b) { return vint8(a) >= b; }
+
+ __forceinline vboolf8 operator > (const vint8& a, int b) { return a > vint8(b); }
+ __forceinline vboolf8 operator > (int a, const vint8& b) { return vint8(a) > b; }
+
+ __forceinline vboolf8 operator <=(const vint8& a, int b) { return a <= vint8(b); }
+ __forceinline vboolf8 operator <=(int a, const vint8& b) { return vint8(a) <= b; }
+
+ __forceinline vboolf8 eq(const vint8& a, const vint8& b) { return a == b; }
+ __forceinline vboolf8 ne(const vint8& a, const vint8& b) { return a != b; }
+ __forceinline vboolf8 lt(const vint8& a, const vint8& b) { return a < b; }
+ __forceinline vboolf8 ge(const vint8& a, const vint8& b) { return a >= b; }
+ __forceinline vboolf8 gt(const vint8& a, const vint8& b) { return a > b; }
+ __forceinline vboolf8 le(const vint8& a, const vint8& b) { return a <= b; }
+
+#if defined(__AVX512VL__)
+ static __forceinline vboolf8 eq(const vboolf8& mask, const vint8& a, const vint8& b) { return _mm256_mask_cmp_epi32_mask(mask, a, b, _MM_CMPINT_EQ); }
+ static __forceinline vboolf8 ne(const vboolf8& mask, const vint8& a, const vint8& b) { return _mm256_mask_cmp_epi32_mask(mask, a, b, _MM_CMPINT_NE); }
+ static __forceinline vboolf8 lt(const vboolf8& mask, const vint8& a, const vint8& b) { return _mm256_mask_cmp_epi32_mask(mask, a, b, _MM_CMPINT_LT); }
+ static __forceinline vboolf8 ge(const vboolf8& mask, const vint8& a, const vint8& b) { return _mm256_mask_cmp_epi32_mask(mask, a, b, _MM_CMPINT_GE); }
+ static __forceinline vboolf8 gt(const vboolf8& mask, const vint8& a, const vint8& b) { return _mm256_mask_cmp_epi32_mask(mask, a, b, _MM_CMPINT_GT); }
+ static __forceinline vboolf8 le(const vboolf8& mask, const vint8& a, const vint8& b) { return _mm256_mask_cmp_epi32_mask(mask, a, b, _MM_CMPINT_LE); }
+#else
+ static __forceinline vboolf8 eq(const vboolf8& mask, const vint8& a, const vint8& b) { return mask & (a == b); }
+ static __forceinline vboolf8 ne(const vboolf8& mask, const vint8& a, const vint8& b) { return mask & (a != b); }
+ static __forceinline vboolf8 lt(const vboolf8& mask, const vint8& a, const vint8& b) { return mask & (a < b); }
+ static __forceinline vboolf8 ge(const vboolf8& mask, const vint8& a, const vint8& b) { return mask & (a >= b); }
+ static __forceinline vboolf8 gt(const vboolf8& mask, const vint8& a, const vint8& b) { return mask & (a > b); }
+ static __forceinline vboolf8 le(const vboolf8& mask, const vint8& a, const vint8& b) { return mask & (a <= b); }
+#endif
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Movement/Shifting/Shuffling Functions
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vint8 unpacklo(const vint8& a, const vint8& b) { return _mm256_unpacklo_epi32(a, b); }
+ __forceinline vint8 unpackhi(const vint8& a, const vint8& b) { return _mm256_unpackhi_epi32(a, b); }
+
+ template<int i>
+ __forceinline vint8 shuffle(const vint8& v) {
+ return _mm256_castps_si256(_mm256_permute_ps(_mm256_castsi256_ps(v), _MM_SHUFFLE(i, i, i, i)));
+ }
+
+ template<int i0, int i1>
+ __forceinline vint8 shuffle4(const vint8& v) {
+ return _mm256_permute2f128_si256(v, v, (i1 << 4) | (i0 << 0));
+ }
+
+ template<int i0, int i1>
+ __forceinline vint8 shuffle4(const vint8& a, const vint8& b) {
+ return _mm256_permute2f128_si256(a, b, (i1 << 4) | (i0 << 0));
+ }
+
+ template<int i0, int i1, int i2, int i3>
+ __forceinline vint8 shuffle(const vint8& v) {
+ return _mm256_castps_si256(_mm256_permute_ps(_mm256_castsi256_ps(v), _MM_SHUFFLE(i3, i2, i1, i0)));
+ }
+
+ template<int i0, int i1, int i2, int i3>
+ __forceinline vint8 shuffle(const vint8& a, const vint8& b) {
+ return _mm256_castps_si256(_mm256_shuffle_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b), _MM_SHUFFLE(i3, i2, i1, i0)));
+ }
+
+ template<> __forceinline vint8 shuffle<0, 0, 2, 2>(const vint8& v) { return _mm256_castps_si256(_mm256_moveldup_ps(_mm256_castsi256_ps(v))); }
+ template<> __forceinline vint8 shuffle<1, 1, 3, 3>(const vint8& v) { return _mm256_castps_si256(_mm256_movehdup_ps(_mm256_castsi256_ps(v))); }
+ template<> __forceinline vint8 shuffle<0, 1, 0, 1>(const vint8& v) { return _mm256_castps_si256(_mm256_castpd_ps(_mm256_movedup_pd(_mm256_castps_pd(_mm256_castsi256_ps(v))))); }
+
+ __forceinline vint8 broadcast(const int* ptr) { return _mm256_castps_si256(_mm256_broadcast_ss((const float*)ptr)); }
+
+ template<int i> __forceinline vint8 insert4(const vint8& a, const vint4& b) { return _mm256_insertf128_si256(a, b, i); }
+ template<int i> __forceinline vint4 extract4(const vint8& a) { return _mm256_extractf128_si256(a, i); }
+ template<> __forceinline vint4 extract4<0>(const vint8& a) { return _mm256_castsi256_si128(a); }
+
+ __forceinline int toScalar(const vint8& v) { return _mm_cvtsi128_si32(_mm256_castsi256_si128(v)); }
+
+#if !defined(__aarch64__)
+
+__forceinline vint8 permute(const vint8& v, const __m256i& index) {
+ return _mm256_permutevar8x32_epi32(v, index);
+ }
+
+ __forceinline vint8 shuffle(const vint8& v, const __m256i& index) {
+ return _mm256_castps_si256(_mm256_permutevar_ps(_mm256_castsi256_ps(v), index));
+ }
+
+
+
+ template<int i>
+ static __forceinline vint8 align_shift_right(const vint8& a, const vint8& b) {
+#if defined(__AVX512VL__)
+ return _mm256_alignr_epi32(a, b, i);
+#else
+ return _mm256_alignr_epi8(a, b, 4*i);
+#endif
+ }
+
+#endif
+
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Reductions
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vint8 vreduce_min2(const vint8& v) { return min(v,shuffle<1,0,3,2>(v)); }
+ __forceinline vint8 vreduce_min4(const vint8& v) { vint8 v1 = vreduce_min2(v); return min(v1,shuffle<2,3,0,1>(v1)); }
+ __forceinline vint8 vreduce_min (const vint8& v) { vint8 v1 = vreduce_min4(v); return min(v1,shuffle4<1,0>(v1)); }
+
+ __forceinline vint8 vreduce_max2(const vint8& v) { return max(v,shuffle<1,0,3,2>(v)); }
+ __forceinline vint8 vreduce_max4(const vint8& v) { vint8 v1 = vreduce_max2(v); return max(v1,shuffle<2,3,0,1>(v1)); }
+ __forceinline vint8 vreduce_max (const vint8& v) { vint8 v1 = vreduce_max4(v); return max(v1,shuffle4<1,0>(v1)); }
+
+ __forceinline vint8 vreduce_add2(const vint8& v) { return v + shuffle<1,0,3,2>(v); }
+ __forceinline vint8 vreduce_add4(const vint8& v) { vint8 v1 = vreduce_add2(v); return v1 + shuffle<2,3,0,1>(v1); }
+ __forceinline vint8 vreduce_add (const vint8& v) { vint8 v1 = vreduce_add4(v); return v1 + shuffle4<1,0>(v1); }
+
+ __forceinline int reduce_min(const vint8& v) { return toScalar(vreduce_min(v)); }
+ __forceinline int reduce_max(const vint8& v) { return toScalar(vreduce_max(v)); }
+ __forceinline int reduce_add(const vint8& v) { return toScalar(vreduce_add(v)); }
+
+ __forceinline size_t select_min(const vint8& v) { return bsf(movemask(v == vreduce_min(v))); }
+ __forceinline size_t select_max(const vint8& v) { return bsf(movemask(v == vreduce_max(v))); }
+
+ __forceinline size_t select_min(const vboolf8& valid, const vint8& v) { const vint8 a = select(valid,v,vint8(pos_inf)); return bsf(movemask(valid & (a == vreduce_min(a)))); }
+ __forceinline size_t select_max(const vboolf8& valid, const vint8& v) { const vint8 a = select(valid,v,vint8(neg_inf)); return bsf(movemask(valid & (a == vreduce_max(a)))); }
+
+
+ __forceinline vint8 assign(const vint4& a) { return _mm256_castsi128_si256(a); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Sorting networks
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vint8 usort_ascending(const vint8& v)
+ {
+ const vint8 a0 = v;
+ const vint8 b0 = shuffle<1,0,3,2>(a0);
+ const vint8 c0 = umin(a0,b0);
+ const vint8 d0 = umax(a0,b0);
+ const vint8 a1 = select<0x99 /* 0b10011001 */>(c0,d0);
+ const vint8 b1 = shuffle<2,3,0,1>(a1);
+ const vint8 c1 = umin(a1,b1);
+ const vint8 d1 = umax(a1,b1);
+ const vint8 a2 = select<0xc3 /* 0b11000011 */>(c1,d1);
+ const vint8 b2 = shuffle<1,0,3,2>(a2);
+ const vint8 c2 = umin(a2,b2);
+ const vint8 d2 = umax(a2,b2);
+ const vint8 a3 = select<0xa5 /* 0b10100101 */>(c2,d2);
+ const vint8 b3 = shuffle4<1,0>(a3);
+ const vint8 c3 = umin(a3,b3);
+ const vint8 d3 = umax(a3,b3);
+ const vint8 a4 = select<0xf /* 0b00001111 */>(c3,d3);
+ const vint8 b4 = shuffle<2,3,0,1>(a4);
+ const vint8 c4 = umin(a4,b4);
+ const vint8 d4 = umax(a4,b4);
+ const vint8 a5 = select<0x33 /* 0b00110011 */>(c4,d4);
+ const vint8 b5 = shuffle<1,0,3,2>(a5);
+ const vint8 c5 = umin(a5,b5);
+ const vint8 d5 = umax(a5,b5);
+ const vint8 a6 = select<0x55 /* 0b01010101 */>(c5,d5);
+ return a6;
+ }
+
+ __forceinline vint8 usort_descending(const vint8& v)
+ {
+ const vint8 a0 = v;
+ const vint8 b0 = shuffle<1,0,3,2>(a0);
+ const vint8 c0 = umax(a0,b0);
+ const vint8 d0 = umin(a0,b0);
+ const vint8 a1 = select<0x99 /* 0b10011001 */>(c0,d0);
+ const vint8 b1 = shuffle<2,3,0,1>(a1);
+ const vint8 c1 = umax(a1,b1);
+ const vint8 d1 = umin(a1,b1);
+ const vint8 a2 = select<0xc3 /* 0b11000011 */>(c1,d1);
+ const vint8 b2 = shuffle<1,0,3,2>(a2);
+ const vint8 c2 = umax(a2,b2);
+ const vint8 d2 = umin(a2,b2);
+ const vint8 a3 = select<0xa5 /* 0b10100101 */>(c2,d2);
+ const vint8 b3 = shuffle4<1,0>(a3);
+ const vint8 c3 = umax(a3,b3);
+ const vint8 d3 = umin(a3,b3);
+ const vint8 a4 = select<0xf /* 0b00001111 */>(c3,d3);
+ const vint8 b4 = shuffle<2,3,0,1>(a4);
+ const vint8 c4 = umax(a4,b4);
+ const vint8 d4 = umin(a4,b4);
+ const vint8 a5 = select<0x33 /* 0b00110011 */>(c4,d4);
+ const vint8 b5 = shuffle<1,0,3,2>(a5);
+ const vint8 c5 = umax(a5,b5);
+ const vint8 d5 = umin(a5,b5);
+ const vint8 a6 = select<0x55 /* 0b01010101 */>(c5,d5);
+ return a6;
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Output Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline embree_ostream operator <<(embree_ostream cout, const vint8& a) {
+ return cout << "<" << a[0] << ", " << a[1] << ", " << a[2] << ", " << a[3] << ", " << a[4] << ", " << a[5] << ", " << a[6] << ", " << a[7] << ">";
+ }
+}
diff --git a/thirdparty/embree-aarch64/common/simd/vllong4_avx2.h b/thirdparty/embree-aarch64/common/simd/vllong4_avx2.h
new file mode 100644
index 0000000000..de3ebc16a7
--- /dev/null
+++ b/thirdparty/embree-aarch64/common/simd/vllong4_avx2.h
@@ -0,0 +1,358 @@
+// Copyright 2009-2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+namespace embree
+{
+ /* 4-wide AVX2 64-bit long long type */
+ template<>
+ struct vllong<4>
+ {
+ ALIGNED_STRUCT_(32);
+
+ typedef vboold4 Bool;
+
+ enum { size = 4 }; // number of SIMD elements
+ union { // data
+ __m256i v;
+ long long i[4];
+ };
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constructors, Assignment & Cast Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vllong() {}
+ __forceinline vllong(const vllong4& t) { v = t.v; }
+ __forceinline vllong4& operator =(const vllong4& f) { v = f.v; return *this; }
+
+ __forceinline vllong(const __m256i& t) { v = t; }
+ __forceinline operator __m256i() const { return v; }
+ __forceinline operator __m256d() const { return _mm256_castsi256_pd(v); }
+
+
+ __forceinline vllong(long long i) {
+ v = _mm256_set1_epi64x(i);
+ }
+
+ __forceinline vllong(long long a, long long b, long long c, long long d) {
+ v = _mm256_set_epi64x(d,c,b,a);
+ }
+
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constants
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vllong(ZeroTy) : v(_mm256_setzero_si256()) {}
+ __forceinline vllong(OneTy) : v(_mm256_set1_epi64x(1)) {}
+ __forceinline vllong(StepTy) : v(_mm256_set_epi64x(3,2,1,0)) {}
+ __forceinline vllong(ReverseStepTy) : v(_mm256_set_epi64x(0,1,2,3)) {}
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Loads and Stores
+ ////////////////////////////////////////////////////////////////////////////////
+
+ static __forceinline void store_nt(void* __restrict__ ptr, const vllong4& a) {
+ _mm256_stream_ps((float*)ptr,_mm256_castsi256_ps(a));
+ }
+
+ static __forceinline vllong4 loadu(const void* addr)
+ {
+ return _mm256_loadu_si256((__m256i*)addr);
+ }
+
+ static __forceinline vllong4 load(const vllong4* addr) {
+ return _mm256_load_si256((__m256i*)addr);
+ }
+
+ static __forceinline vllong4 load(const long long* addr) {
+ return _mm256_load_si256((__m256i*)addr);
+ }
+
+ static __forceinline void store(void* ptr, const vllong4& v) {
+ _mm256_store_si256((__m256i*)ptr,v);
+ }
+
+ static __forceinline void storeu(void* ptr, const vllong4& v) {
+ _mm256_storeu_si256((__m256i*)ptr,v);
+ }
+
+ static __forceinline void storeu(const vboold4& mask, long long* ptr, const vllong4& f) {
+#if defined(__AVX512VL__)
+ _mm256_mask_storeu_epi64(ptr,mask,f);
+#else
+ _mm256_maskstore_pd((double*)ptr,mask,_mm256_castsi256_pd(f));
+#endif
+ }
+
+ static __forceinline void store(const vboold4& mask, void* ptr, const vllong4& f) {
+#if defined(__AVX512VL__)
+ _mm256_mask_store_epi64(ptr,mask,f);
+#else
+ _mm256_maskstore_pd((double*)ptr,mask,_mm256_castsi256_pd(f));
+#endif
+ }
+
+ static __forceinline vllong4 broadcast64bit(size_t v) {
+ return _mm256_set1_epi64x(v);
+ }
+
+ static __forceinline size_t extract64bit(const vllong4& v)
+ {
+ return _mm_cvtsi128_si64(_mm256_castsi256_si128(v));
+ }
+
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Array Access
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline long long& operator [](size_t index) { assert(index < 4); return i[index]; }
+ __forceinline const long long& operator [](size_t index) const { assert(index < 4); return i[index]; }
+
+ };
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Select
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vllong4 select(const vboold4& m, const vllong4& t, const vllong4& f) {
+ #if defined(__AVX512VL__)
+ return _mm256_mask_blend_epi64(m, f, t);
+ #else
+ return _mm256_castpd_si256(_mm256_blendv_pd(_mm256_castsi256_pd(f), _mm256_castsi256_pd(t), m));
+ #endif
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Unary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+#if defined(__AVX512VL__)
+ __forceinline vboold4 asBool(const vllong4& a) { return _mm256_movepi64_mask(a); }
+#else
+ __forceinline vboold4 asBool(const vllong4& a) { return _mm256_castsi256_pd(a); }
+#endif
+
+ __forceinline vllong4 operator +(const vllong4& a) { return a; }
+ __forceinline vllong4 operator -(const vllong4& a) { return _mm256_sub_epi64(_mm256_setzero_si256(), a); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Binary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vllong4 operator +(const vllong4& a, const vllong4& b) { return _mm256_add_epi64(a, b); }
+ __forceinline vllong4 operator +(const vllong4& a, long long b) { return a + vllong4(b); }
+ __forceinline vllong4 operator +(long long a, const vllong4& b) { return vllong4(a) + b; }
+
+ __forceinline vllong4 operator -(const vllong4& a, const vllong4& b) { return _mm256_sub_epi64(a, b); }
+ __forceinline vllong4 operator -(const vllong4& a, long long b) { return a - vllong4(b); }
+ __forceinline vllong4 operator -(long long a, const vllong4& b) { return vllong4(a) - b; }
+
+ /* only low 32bit part */
+ __forceinline vllong4 operator *(const vllong4& a, const vllong4& b) { return _mm256_mul_epi32(a, b); }
+ __forceinline vllong4 operator *(const vllong4& a, long long b) { return a * vllong4(b); }
+ __forceinline vllong4 operator *(long long a, const vllong4& b) { return vllong4(a) * b; }
+
+ __forceinline vllong4 operator &(const vllong4& a, const vllong4& b) { return _mm256_and_si256(a, b); }
+ __forceinline vllong4 operator &(const vllong4& a, long long b) { return a & vllong4(b); }
+ __forceinline vllong4 operator &(long long a, const vllong4& b) { return vllong4(a) & b; }
+
+ __forceinline vllong4 operator |(const vllong4& a, const vllong4& b) { return _mm256_or_si256(a, b); }
+ __forceinline vllong4 operator |(const vllong4& a, long long b) { return a | vllong4(b); }
+ __forceinline vllong4 operator |(long long a, const vllong4& b) { return vllong4(a) | b; }
+
+ __forceinline vllong4 operator ^(const vllong4& a, const vllong4& b) { return _mm256_xor_si256(a, b); }
+ __forceinline vllong4 operator ^(const vllong4& a, long long b) { return a ^ vllong4(b); }
+ __forceinline vllong4 operator ^(long long a, const vllong4& b) { return vllong4(a) ^ b; }
+
+ __forceinline vllong4 operator <<(const vllong4& a, long long n) { return _mm256_slli_epi64(a, (int)n); }
+ //__forceinline vllong4 operator >>(const vllong4& a, long long n) { return _mm256_srai_epi64(a, n); }
+
+ __forceinline vllong4 operator <<(const vllong4& a, const vllong4& n) { return _mm256_sllv_epi64(a, n); }
+ //__forceinline vllong4 operator >>(const vllong4& a, const vllong4& n) { return _mm256_srav_epi64(a, n); }
+ //__forceinline vllong4 sra(const vllong4& a, long long b) { return _mm256_srai_epi64(a, b); }
+
+ __forceinline vllong4 srl(const vllong4& a, long long b) { return _mm256_srli_epi64(a, (int)b); }
+
+ //__forceinline vllong4 min(const vllong4& a, const vllong4& b) { return _mm256_min_epi64(a, b); }
+ //__forceinline vllong4 min(const vllong4& a, long long b) { return min(a,vllong4(b)); }
+ //__forceinline vllong4 min(long long a, const vllong4& b) { return min(vllong4(a),b); }
+
+ //__forceinline vllong4 max(const vllong4& a, const vllong4& b) { return _mm256_max_epi64(a, b); }
+ //__forceinline vllong4 max(const vllong4& a, long long b) { return max(a,vllong4(b)); }
+ //__forceinline vllong4 max(long long a, const vllong4& b) { return max(vllong4(a),b); }
+
+#if defined(__AVX512VL__)
+ __forceinline vllong4 mask_and(const vboold4& m, const vllong4& c, const vllong4& a, const vllong4& b) { return _mm256_mask_and_epi64(c,m,a,b); }
+ __forceinline vllong4 mask_or (const vboold4& m, const vllong4& c, const vllong4& a, const vllong4& b) { return _mm256_mask_or_epi64(c,m,a,b); }
+#else
+ __forceinline vllong4 mask_and(const vboold4& m, const vllong4& c, const vllong4& a, const vllong4& b) { return select(m, a & b, c); }
+ __forceinline vllong4 mask_or (const vboold4& m, const vllong4& c, const vllong4& a, const vllong4& b) { return select(m, a | b, c); }
+#endif
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Assignment Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vllong4& operator +=(vllong4& a, const vllong4& b) { return a = a + b; }
+ __forceinline vllong4& operator +=(vllong4& a, long long b) { return a = a + b; }
+
+ __forceinline vllong4& operator -=(vllong4& a, const vllong4& b) { return a = a - b; }
+ __forceinline vllong4& operator -=(vllong4& a, long long b) { return a = a - b; }
+
+ __forceinline vllong4& operator *=(vllong4& a, const vllong4& b) { return a = a * b; }
+ __forceinline vllong4& operator *=(vllong4& a, long long b) { return a = a * b; }
+
+ __forceinline vllong4& operator &=(vllong4& a, const vllong4& b) { return a = a & b; }
+ __forceinline vllong4& operator &=(vllong4& a, long long b) { return a = a & b; }
+
+ __forceinline vllong4& operator |=(vllong4& a, const vllong4& b) { return a = a | b; }
+ __forceinline vllong4& operator |=(vllong4& a, long long b) { return a = a | b; }
+
+ __forceinline vllong4& operator <<=(vllong4& a, long long b) { return a = a << b; }
+ //__forceinline vllong4& operator >>=(vllong4& a, long long b) { return a = a >> b; }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Comparison Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+#if defined(__AVX512VL__)
+ __forceinline vboold4 operator ==(const vllong4& a, const vllong4& b) { return _mm256_cmp_epi64_mask(a,b,_MM_CMPINT_EQ); }
+ __forceinline vboold4 operator !=(const vllong4& a, const vllong4& b) { return _mm256_cmp_epi64_mask(a,b,_MM_CMPINT_NE); }
+ __forceinline vboold4 operator < (const vllong4& a, const vllong4& b) { return _mm256_cmp_epi64_mask(a,b,_MM_CMPINT_LT); }
+ __forceinline vboold4 operator >=(const vllong4& a, const vllong4& b) { return _mm256_cmp_epi64_mask(a,b,_MM_CMPINT_GE); }
+ __forceinline vboold4 operator > (const vllong4& a, const vllong4& b) { return _mm256_cmp_epi64_mask(a,b,_MM_CMPINT_GT); }
+ __forceinline vboold4 operator <=(const vllong4& a, const vllong4& b) { return _mm256_cmp_epi64_mask(a,b,_MM_CMPINT_LE); }
+#else
+ __forceinline vboold4 operator ==(const vllong4& a, const vllong4& b) { return _mm256_cmpeq_epi64(a,b); }
+ __forceinline vboold4 operator !=(const vllong4& a, const vllong4& b) { return !(a == b); }
+ __forceinline vboold4 operator > (const vllong4& a, const vllong4& b) { return _mm256_cmpgt_epi64(a,b); }
+ __forceinline vboold4 operator < (const vllong4& a, const vllong4& b) { return _mm256_cmpgt_epi64(b,a); }
+ __forceinline vboold4 operator >=(const vllong4& a, const vllong4& b) { return !(a < b); }
+ __forceinline vboold4 operator <=(const vllong4& a, const vllong4& b) { return !(a > b); }
+#endif
+
+ __forceinline vboold4 operator ==(const vllong4& a, long long b) { return a == vllong4(b); }
+ __forceinline vboold4 operator ==(long long a, const vllong4& b) { return vllong4(a) == b; }
+
+ __forceinline vboold4 operator !=(const vllong4& a, long long b) { return a != vllong4(b); }
+ __forceinline vboold4 operator !=(long long a, const vllong4& b) { return vllong4(a) != b; }
+
+ __forceinline vboold4 operator > (const vllong4& a, long long b) { return a > vllong4(b); }
+ __forceinline vboold4 operator > (long long a, const vllong4& b) { return vllong4(a) > b; }
+
+ __forceinline vboold4 operator < (const vllong4& a, long long b) { return a < vllong4(b); }
+ __forceinline vboold4 operator < (long long a, const vllong4& b) { return vllong4(a) < b; }
+
+ __forceinline vboold4 operator >=(const vllong4& a, long long b) { return a >= vllong4(b); }
+ __forceinline vboold4 operator >=(long long a, const vllong4& b) { return vllong4(a) >= b; }
+
+ __forceinline vboold4 operator <=(const vllong4& a, long long b) { return a <= vllong4(b); }
+ __forceinline vboold4 operator <=(long long a, const vllong4& b) { return vllong4(a) <= b; }
+
+ __forceinline vboold4 eq(const vllong4& a, const vllong4& b) { return a == b; }
+ __forceinline vboold4 ne(const vllong4& a, const vllong4& b) { return a != b; }
+ __forceinline vboold4 lt(const vllong4& a, const vllong4& b) { return a < b; }
+ __forceinline vboold4 ge(const vllong4& a, const vllong4& b) { return a >= b; }
+ __forceinline vboold4 gt(const vllong4& a, const vllong4& b) { return a > b; }
+ __forceinline vboold4 le(const vllong4& a, const vllong4& b) { return a <= b; }
+
+#if defined(__AVX512VL__)
+ __forceinline vboold4 eq(const vboold4& mask, const vllong4& a, const vllong4& b) { return _mm256_mask_cmp_epi64_mask(mask, a, b, _MM_CMPINT_EQ); }
+ __forceinline vboold4 ne(const vboold4& mask, const vllong4& a, const vllong4& b) { return _mm256_mask_cmp_epi64_mask(mask, a, b, _MM_CMPINT_NE); }
+ __forceinline vboold4 lt(const vboold4& mask, const vllong4& a, const vllong4& b) { return _mm256_mask_cmp_epi64_mask(mask, a, b, _MM_CMPINT_LT); }
+ __forceinline vboold4 ge(const vboold4& mask, const vllong4& a, const vllong4& b) { return _mm256_mask_cmp_epi64_mask(mask, a, b, _MM_CMPINT_GE); }
+ __forceinline vboold4 gt(const vboold4& mask, const vllong4& a, const vllong4& b) { return _mm256_mask_cmp_epi64_mask(mask, a, b, _MM_CMPINT_GT); }
+ __forceinline vboold4 le(const vboold4& mask, const vllong4& a, const vllong4& b) { return _mm256_mask_cmp_epi64_mask(mask, a, b, _MM_CMPINT_LE); }
+#else
+ __forceinline vboold4 eq(const vboold4& mask, const vllong4& a, const vllong4& b) { return mask & (a == b); }
+ __forceinline vboold4 ne(const vboold4& mask, const vllong4& a, const vllong4& b) { return mask & (a != b); }
+ __forceinline vboold4 lt(const vboold4& mask, const vllong4& a, const vllong4& b) { return mask & (a < b); }
+ __forceinline vboold4 ge(const vboold4& mask, const vllong4& a, const vllong4& b) { return mask & (a >= b); }
+ __forceinline vboold4 gt(const vboold4& mask, const vllong4& a, const vllong4& b) { return mask & (a > b); }
+ __forceinline vboold4 le(const vboold4& mask, const vllong4& a, const vllong4& b) { return mask & (a <= b); }
+#endif
+
+ __forceinline void xchg(const vboold4& m, vllong4& a, vllong4& b) {
+ const vllong4 c = a; a = select(m,b,a); b = select(m,c,b);
+ }
+
+ __forceinline vboold4 test(const vllong4& a, const vllong4& b) {
+#if defined(__AVX512VL__)
+ return _mm256_test_epi64_mask(a,b);
+#else
+ return _mm256_testz_si256(a,b);
+#endif
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ // Movement/Shifting/Shuffling Functions
+ ////////////////////////////////////////////////////////////////////////////////
+
+ template<int i0, int i1>
+ __forceinline vllong4 shuffle(const vllong4& v) {
+ return _mm256_castpd_si256(_mm256_permute_pd(_mm256_castsi256_pd(v), (i1 << 3) | (i0 << 2) | (i1 << 1) | i0));
+ }
+
+ template<int i>
+ __forceinline vllong4 shuffle(const vllong4& v) {
+ return shuffle<i, i>(v);
+ }
+
+ template<int i0, int i1>
+ __forceinline vllong4 shuffle2(const vllong4& v) {
+ return _mm256_castpd_si256(_mm256_permute2f128_pd(_mm256_castsi256_pd(v), _mm256_castsi256_pd(v), (i1 << 4) | i0));
+ }
+
+ __forceinline long long toScalar(const vllong4& v) {
+ return _mm_cvtsi128_si64(_mm256_castsi256_si128(v));
+ }
+
+#if defined(__AVX512VL__)
+ __forceinline vllong4 permute(const vllong4& a, const __m256i& index) {
+ // workaround for GCC 7.x
+#if defined(__GNUC__) && !defined(__INTEL_COMPILER) && !defined(__clang__)
+ return _mm256_permutex2var_epi64(a,index,a);
+#else
+ return _mm256_permutexvar_epi64(index,a);
+#endif
+ }
+
+ __forceinline vllong4 permutex2var(const vllong4& index, const vllong4& a, const vllong4& b) {
+ return _mm256_permutex2var_epi64(a,index,b);
+ }
+
+#endif
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Reductions
+ ////////////////////////////////////////////////////////////////////////////////
+
+
+ __forceinline vllong4 vreduce_and2(const vllong4& x) { return x & shuffle<1,0>(x); }
+ __forceinline vllong4 vreduce_and (const vllong4& y) { const vllong4 x = vreduce_and2(y); return x & shuffle2<1,0>(x); }
+
+ __forceinline vllong4 vreduce_or2(const vllong4& x) { return x | shuffle<1,0>(x); }
+ __forceinline vllong4 vreduce_or (const vllong4& y) { const vllong4 x = vreduce_or2(y); return x | shuffle2<1,0>(x); }
+
+ __forceinline vllong4 vreduce_add2(const vllong4& x) { return x + shuffle<1,0>(x); }
+ __forceinline vllong4 vreduce_add (const vllong4& y) { const vllong4 x = vreduce_add2(y); return x + shuffle2<1,0>(x); }
+
+ __forceinline long long reduce_add(const vllong4& a) { return toScalar(vreduce_add(a)); }
+ __forceinline long long reduce_or (const vllong4& a) { return toScalar(vreduce_or(a)); }
+ __forceinline long long reduce_and(const vllong4& a) { return toScalar(vreduce_and(a)); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Output Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline embree_ostream operator <<(embree_ostream cout, const vllong4& v)
+ {
+ cout << "<" << v[0];
+ for (size_t i=1; i<4; i++) cout << ", " << v[i];
+ cout << ">";
+ return cout;
+ }
+}
diff --git a/thirdparty/embree-aarch64/common/simd/vllong8_avx512.h b/thirdparty/embree-aarch64/common/simd/vllong8_avx512.h
new file mode 100644
index 0000000000..76dddd8991
--- /dev/null
+++ b/thirdparty/embree-aarch64/common/simd/vllong8_avx512.h
@@ -0,0 +1,381 @@
+// Copyright 2009-2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+namespace embree
+{
+ /* 8-wide AVX-512 64-bit long long type */
+ template<>
+ struct vllong<8>
+ {
+ ALIGNED_STRUCT_(64);
+
+ typedef vboold8 Bool;
+
+ enum { size = 8 }; // number of SIMD elements
+ union { // data
+ __m512i v;
+ long long i[8];
+ };
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constructors, Assignment & Cast Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vllong() {}
+ __forceinline vllong(const vllong8& t) { v = t.v; }
+ __forceinline vllong8& operator =(const vllong8& f) { v = f.v; return *this; }
+
+ __forceinline vllong(const __m512i& t) { v = t; }
+ __forceinline operator __m512i() const { return v; }
+ __forceinline operator __m256i() const { return _mm512_castsi512_si256(v); }
+
+ __forceinline vllong(long long i) {
+ v = _mm512_set1_epi64(i);
+ }
+
+ __forceinline vllong(long long a, long long b, long long c, long long d) {
+ v = _mm512_set4_epi64(d,c,b,a);
+ }
+
+ __forceinline vllong(long long a0, long long a1, long long a2, long long a3,
+ long long a4, long long a5, long long a6, long long a7)
+ {
+ v = _mm512_set_epi64(a7,a6,a5,a4,a3,a2,a1,a0);
+ }
+
+ __forceinline vllong(const vllong<4>& i) {
+ v = _mm512_broadcast_i64x4(i);
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constants
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vllong(ZeroTy) : v(_mm512_setzero_epi32()) {}
+ __forceinline vllong(OneTy) : v(_mm512_set1_epi64(1)) {}
+ __forceinline vllong(StepTy) : v(_mm512_set_epi64(7,6,5,4,3,2,1,0)) {}
+ __forceinline vllong(ReverseStepTy) : v(_mm512_setr_epi64(7,6,5,4,3,2,1,0)) {}
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Loads and Stores
+ ////////////////////////////////////////////////////////////////////////////////
+
+ static __forceinline void store_nt(void* __restrict__ ptr, const vllong8& a) {
+ _mm512_stream_si512((__m512i*)ptr,a);
+ }
+
+ static __forceinline vllong8 loadu(const void* addr) {
+ return _mm512_loadu_si512(addr);
+ }
+
+ static __forceinline vllong8 load(const vllong8* addr) {
+ return _mm512_load_si512(addr);
+ }
+
+ static __forceinline vllong8 load(const long long* addr) {
+ return _mm512_load_si512(addr);
+ }
+
+ static __forceinline vllong8 load(const uint8_t* ptr) {
+ return _mm512_cvtepu8_epi64(*(__m128i*)ptr);
+ }
+
+ static __forceinline void store(void* ptr, const vllong8& v) {
+ _mm512_store_si512(ptr,v);
+ }
+
+ static __forceinline void storeu(void* ptr, const vllong8& v) {
+ _mm512_storeu_si512(ptr,v);
+ }
+
+ static __forceinline void storeu(const vboold8& mask, long long* ptr, const vllong8& f) {
+ _mm512_mask_storeu_epi64(ptr,mask,f);
+ }
+
+ static __forceinline void store(const vboold8& mask, void* addr, const vllong8& v2) {
+ _mm512_mask_store_epi64(addr,mask,v2);
+ }
+
+ /* pass by value to avoid compiler generating inefficient code */
+ static __forceinline void storeu_compact(const vboold8 mask, void* addr, const vllong8& reg) {
+ _mm512_mask_compressstoreu_epi64(addr,mask,reg);
+ }
+
+ static __forceinline vllong8 compact64bit(const vboold8& mask, vllong8& v) {
+ return _mm512_mask_compress_epi64(v,mask,v);
+ }
+
+ static __forceinline vllong8 compact64bit(const vboold8& mask, vllong8& dest, const vllong8& source) {
+ return _mm512_mask_compress_epi64(dest,mask,source);
+ }
+
+ static __forceinline vllong8 compact(const vboold8& mask, vllong8& v) {
+ return _mm512_mask_compress_epi64(v,mask,v);
+ }
+
+ static __forceinline vllong8 compact(const vboold8& mask, const vllong8& a, vllong8& b) {
+ return _mm512_mask_compress_epi64(a,mask,b);
+ }
+
+ static __forceinline vllong8 expand(const vboold8& mask, const vllong8& a, vllong8& b) {
+ return _mm512_mask_expand_epi64(b,mask,a);
+ }
+
+ static __forceinline vllong8 broadcast64bit(size_t v) {
+ return _mm512_set1_epi64(v);
+ }
+
+ static __forceinline size_t extract64bit(const vllong8& v)
+ {
+ return _mm_cvtsi128_si64(_mm512_castsi512_si128(v));
+ }
+
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Array Access
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline long long& operator [](size_t index) { assert(index < 8); return i[index]; }
+ __forceinline const long long& operator [](size_t index) const { assert(index < 8); return i[index]; }
+
+ };
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Unary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboold8 asBool(const vllong8& a) { return _mm512_movepi64_mask(a); }
+
+ __forceinline vllong8 operator +(const vllong8& a) { return a; }
+ __forceinline vllong8 operator -(const vllong8& a) { return _mm512_sub_epi64(_mm512_setzero_epi32(), a); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Binary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vllong8 operator +(const vllong8& a, const vllong8& b) { return _mm512_add_epi64(a, b); }
+ __forceinline vllong8 operator +(const vllong8& a, long long b) { return a + vllong8(b); }
+ __forceinline vllong8 operator +(long long a, const vllong8& b) { return vllong8(a) + b; }
+
+ __forceinline vllong8 operator -(const vllong8& a, const vllong8& b) { return _mm512_sub_epi64(a, b); }
+ __forceinline vllong8 operator -(const vllong8& a, long long b) { return a - vllong8(b); }
+ __forceinline vllong8 operator -(long long a, const vllong8& b) { return vllong8(a) - b; }
+
+ __forceinline vllong8 operator *(const vllong8& a, const vllong8& b) { return _mm512_mullo_epi64(a, b); }
+ __forceinline vllong8 operator *(const vllong8& a, long long b) { return a * vllong8(b); }
+ __forceinline vllong8 operator *(long long a, const vllong8& b) { return vllong8(a) * b; }
+
+ __forceinline vllong8 operator &(const vllong8& a, const vllong8& b) { return _mm512_and_epi64(a, b); }
+ __forceinline vllong8 operator &(const vllong8& a, long long b) { return a & vllong8(b); }
+ __forceinline vllong8 operator &(long long a, const vllong8& b) { return vllong8(a) & b; }
+
+ __forceinline vllong8 operator |(const vllong8& a, const vllong8& b) { return _mm512_or_epi64(a, b); }
+ __forceinline vllong8 operator |(const vllong8& a, long long b) { return a | vllong8(b); }
+ __forceinline vllong8 operator |(long long a, const vllong8& b) { return vllong8(a) | b; }
+
+ __forceinline vllong8 operator ^(const vllong8& a, const vllong8& b) { return _mm512_xor_epi64(a, b); }
+ __forceinline vllong8 operator ^(const vllong8& a, long long b) { return a ^ vllong8(b); }
+ __forceinline vllong8 operator ^(long long a, const vllong8& b) { return vllong8(a) ^ b; }
+
+ __forceinline vllong8 operator <<(const vllong8& a, long long n) { return _mm512_slli_epi64(a, n); }
+ __forceinline vllong8 operator >>(const vllong8& a, long long n) { return _mm512_srai_epi64(a, n); }
+
+ __forceinline vllong8 operator <<(const vllong8& a, const vllong8& n) { return _mm512_sllv_epi64(a, n); }
+ __forceinline vllong8 operator >>(const vllong8& a, const vllong8& n) { return _mm512_srav_epi64(a, n); }
+
+ __forceinline vllong8 sll (const vllong8& a, long long b) { return _mm512_slli_epi64(a, b); }
+ __forceinline vllong8 sra (const vllong8& a, long long b) { return _mm512_srai_epi64(a, b); }
+ __forceinline vllong8 srl (const vllong8& a, long long b) { return _mm512_srli_epi64(a, b); }
+
+ __forceinline vllong8 min(const vllong8& a, const vllong8& b) { return _mm512_min_epi64(a, b); }
+ __forceinline vllong8 min(const vllong8& a, long long b) { return min(a,vllong8(b)); }
+ __forceinline vllong8 min(long long a, const vllong8& b) { return min(vllong8(a),b); }
+
+ __forceinline vllong8 max(const vllong8& a, const vllong8& b) { return _mm512_max_epi64(a, b); }
+ __forceinline vllong8 max(const vllong8& a, long long b) { return max(a,vllong8(b)); }
+ __forceinline vllong8 max(long long a, const vllong8& b) { return max(vllong8(a),b); }
+
+ __forceinline vllong8 mask_add(const vboold8& m, const vllong8& c, const vllong8& a, const vllong8& b) { return _mm512_mask_add_epi64(c,m,a,b); }
+ __forceinline vllong8 mask_sub(const vboold8& m, const vllong8& c, const vllong8& a, const vllong8& b) { return _mm512_mask_sub_epi64(c,m,a,b); }
+
+ __forceinline vllong8 mask_and(const vboold8& m, const vllong8& c, const vllong8& a, const vllong8& b) { return _mm512_mask_and_epi64(c,m,a,b); }
+ __forceinline vllong8 mask_or (const vboold8& m, const vllong8& c, const vllong8& a, const vllong8& b) { return _mm512_mask_or_epi64(c,m,a,b); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Assignment Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vllong8& operator +=(vllong8& a, const vllong8& b) { return a = a + b; }
+ __forceinline vllong8& operator +=(vllong8& a, long long b) { return a = a + b; }
+
+ __forceinline vllong8& operator -=(vllong8& a, const vllong8& b) { return a = a - b; }
+ __forceinline vllong8& operator -=(vllong8& a, long long b) { return a = a - b; }
+
+ __forceinline vllong8& operator *=(vllong8& a, const vllong8& b) { return a = a * b; }
+ __forceinline vllong8& operator *=(vllong8& a, long long b) { return a = a * b; }
+
+ __forceinline vllong8& operator &=(vllong8& a, const vllong8& b) { return a = a & b; }
+ __forceinline vllong8& operator &=(vllong8& a, long long b) { return a = a & b; }
+
+ __forceinline vllong8& operator |=(vllong8& a, const vllong8& b) { return a = a | b; }
+ __forceinline vllong8& operator |=(vllong8& a, long long b) { return a = a | b; }
+
+ __forceinline vllong8& operator <<=(vllong8& a, long long b) { return a = a << b; }
+ __forceinline vllong8& operator >>=(vllong8& a, long long b) { return a = a >> b; }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Comparison Operators + Select
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboold8 operator ==(const vllong8& a, const vllong8& b) { return _mm512_cmp_epi64_mask(a,b,_MM_CMPINT_EQ); }
+ __forceinline vboold8 operator ==(const vllong8& a, long long b) { return a == vllong8(b); }
+ __forceinline vboold8 operator ==(long long a, const vllong8& b) { return vllong8(a) == b; }
+
+ __forceinline vboold8 operator !=(const vllong8& a, const vllong8& b) { return _mm512_cmp_epi64_mask(a,b,_MM_CMPINT_NE); }
+ __forceinline vboold8 operator !=(const vllong8& a, long long b) { return a != vllong8(b); }
+ __forceinline vboold8 operator !=(long long a, const vllong8& b) { return vllong8(a) != b; }
+
+ __forceinline vboold8 operator < (const vllong8& a, const vllong8& b) { return _mm512_cmp_epi64_mask(a,b,_MM_CMPINT_LT); }
+ __forceinline vboold8 operator < (const vllong8& a, long long b) { return a < vllong8(b); }
+ __forceinline vboold8 operator < (long long a, const vllong8& b) { return vllong8(a) < b; }
+
+ __forceinline vboold8 operator >=(const vllong8& a, const vllong8& b) { return _mm512_cmp_epi64_mask(a,b,_MM_CMPINT_GE); }
+ __forceinline vboold8 operator >=(const vllong8& a, long long b) { return a >= vllong8(b); }
+ __forceinline vboold8 operator >=(long long a, const vllong8& b) { return vllong8(a) >= b; }
+
+ __forceinline vboold8 operator > (const vllong8& a, const vllong8& b) { return _mm512_cmp_epi64_mask(a,b,_MM_CMPINT_GT); }
+ __forceinline vboold8 operator > (const vllong8& a, long long b) { return a > vllong8(b); }
+ __forceinline vboold8 operator > (long long a, const vllong8& b) { return vllong8(a) > b; }
+
+ __forceinline vboold8 operator <=(const vllong8& a, const vllong8& b) { return _mm512_cmp_epi64_mask(a,b,_MM_CMPINT_LE); }
+ __forceinline vboold8 operator <=(const vllong8& a, long long b) { return a <= vllong8(b); }
+ __forceinline vboold8 operator <=(long long a, const vllong8& b) { return vllong8(a) <= b; }
+
+ __forceinline vboold8 eq(const vllong8& a, const vllong8& b) { return _mm512_cmp_epi64_mask(a,b,_MM_CMPINT_EQ); }
+ __forceinline vboold8 ne(const vllong8& a, const vllong8& b) { return _mm512_cmp_epi64_mask(a,b,_MM_CMPINT_NE); }
+ __forceinline vboold8 lt(const vllong8& a, const vllong8& b) { return _mm512_cmp_epi64_mask(a,b,_MM_CMPINT_LT); }
+ __forceinline vboold8 ge(const vllong8& a, const vllong8& b) { return _mm512_cmp_epi64_mask(a,b,_MM_CMPINT_GE); }
+ __forceinline vboold8 gt(const vllong8& a, const vllong8& b) { return _mm512_cmp_epi64_mask(a,b,_MM_CMPINT_GT); }
+ __forceinline vboold8 le(const vllong8& a, const vllong8& b) { return _mm512_cmp_epi64_mask(a,b,_MM_CMPINT_LE); }
+
+ __forceinline vboold8 eq(const vboold8 mask, const vllong8& a, const vllong8& b) { return _mm512_mask_cmp_epi64_mask(mask,a,b,_MM_CMPINT_EQ); }
+ __forceinline vboold8 ne(const vboold8 mask, const vllong8& a, const vllong8& b) { return _mm512_mask_cmp_epi64_mask(mask,a,b,_MM_CMPINT_NE); }
+ __forceinline vboold8 lt(const vboold8 mask, const vllong8& a, const vllong8& b) { return _mm512_mask_cmp_epi64_mask(mask,a,b,_MM_CMPINT_LT); }
+ __forceinline vboold8 ge(const vboold8 mask, const vllong8& a, const vllong8& b) { return _mm512_mask_cmp_epi64_mask(mask,a,b,_MM_CMPINT_GE); }
+ __forceinline vboold8 gt(const vboold8 mask, const vllong8& a, const vllong8& b) { return _mm512_mask_cmp_epi64_mask(mask,a,b,_MM_CMPINT_GT); }
+ __forceinline vboold8 le(const vboold8 mask, const vllong8& a, const vllong8& b) { return _mm512_mask_cmp_epi64_mask(mask,a,b,_MM_CMPINT_LE); }
+
+ __forceinline vllong8 select(const vboold8& m, const vllong8& t, const vllong8& f) {
+ return _mm512_mask_or_epi64(f,m,t,t);
+ }
+
+ __forceinline void xchg(const vboold8& m, vllong8& a, vllong8& b) {
+ const vllong8 c = a; a = select(m,b,a); b = select(m,c,b);
+ }
+
+ __forceinline vboold8 test(const vboold8& m, const vllong8& a, const vllong8& b) {
+ return _mm512_mask_test_epi64_mask(m,a,b);
+ }
+
+ __forceinline vboold8 test(const vllong8& a, const vllong8& b) {
+ return _mm512_test_epi64_mask(a,b);
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ // Movement/Shifting/Shuffling Functions
+ ////////////////////////////////////////////////////////////////////////////////
+
+ template<int i0, int i1>
+ __forceinline vllong8 shuffle(const vllong8& v) {
+ return _mm512_castpd_si512(_mm512_permute_pd(_mm512_castsi512_pd(v), (i1 << 7) | (i0 << 6) | (i1 << 5) | (i0 << 4) | (i1 << 3) | (i0 << 2) | (i1 << 1) | i0));
+ }
+
+ template<int i>
+ __forceinline vllong8 shuffle(const vllong8& v) {
+ return shuffle<i, i>(v);
+ }
+
+ template<int i0, int i1, int i2, int i3>
+ __forceinline vllong8 shuffle(const vllong8& v) {
+ return _mm512_permutex_epi64(v, _MM_SHUFFLE(i3, i2, i1, i0));
+ }
+
+ template<int i0, int i1>
+ __forceinline vllong8 shuffle4(const vllong8& v) {
+ return _mm512_shuffle_i64x2(v, v, _MM_SHUFFLE(i1*2+1, i1*2, i0*2+1, i0*2));
+ }
+
+ template<int i>
+ __forceinline vllong8 shuffle4(const vllong8& v) {
+ return shuffle4<i, i>(v);
+ }
+
+ template<int i>
+ __forceinline vllong8 align_shift_right(const vllong8& a, const vllong8& b) {
+ return _mm512_alignr_epi64(a, b, i);
+ };
+
+ __forceinline long long toScalar(const vllong8& v) {
+ return _mm_cvtsi128_si64(_mm512_castsi512_si128(v));
+ }
+
+ __forceinline vllong8 zeroExtend32Bit(const __m512i& a) {
+ return _mm512_cvtepu32_epi64(_mm512_castsi512_si256(a));
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Reductions
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vllong8 vreduce_min2(vllong8 x) { return min(x, shuffle<1,0,3,2>(x)); }
+ __forceinline vllong8 vreduce_min4(vllong8 x) { x = vreduce_min2(x); return min(x, shuffle<2,3,0,1>(x)); }
+ __forceinline vllong8 vreduce_min (vllong8 x) { x = vreduce_min4(x); return min(x, shuffle4<1,0>(x)); }
+
+ __forceinline vllong8 vreduce_max2(vllong8 x) { return max(x, shuffle<1,0,3,2>(x)); }
+ __forceinline vllong8 vreduce_max4(vllong8 x) { x = vreduce_max2(x); return max(x, shuffle<2,3,0,1>(x)); }
+ __forceinline vllong8 vreduce_max (vllong8 x) { x = vreduce_max4(x); return max(x, shuffle4<1,0>(x)); }
+
+ __forceinline vllong8 vreduce_and2(vllong8 x) { return x & shuffle<1,0,3,2>(x); }
+ __forceinline vllong8 vreduce_and4(vllong8 x) { x = vreduce_and2(x); return x & shuffle<2,3,0,1>(x); }
+ __forceinline vllong8 vreduce_and (vllong8 x) { x = vreduce_and4(x); return x & shuffle4<1,0>(x); }
+
+ __forceinline vllong8 vreduce_or2(vllong8 x) { return x | shuffle<1,0,3,2>(x); }
+ __forceinline vllong8 vreduce_or4(vllong8 x) { x = vreduce_or2(x); return x | shuffle<2,3,0,1>(x); }
+ __forceinline vllong8 vreduce_or (vllong8 x) { x = vreduce_or4(x); return x | shuffle4<1,0>(x); }
+
+ __forceinline vllong8 vreduce_add2(vllong8 x) { return x + shuffle<1,0,3,2>(x); }
+ __forceinline vllong8 vreduce_add4(vllong8 x) { x = vreduce_add2(x); return x + shuffle<2,3,0,1>(x); }
+ __forceinline vllong8 vreduce_add (vllong8 x) { x = vreduce_add4(x); return x + shuffle4<1,0>(x); }
+
+ __forceinline long long reduce_min(const vllong8& v) { return toScalar(vreduce_min(v)); }
+ __forceinline long long reduce_max(const vllong8& v) { return toScalar(vreduce_max(v)); }
+ __forceinline long long reduce_and(const vllong8& v) { return toScalar(vreduce_and(v)); }
+ __forceinline long long reduce_or (const vllong8& v) { return toScalar(vreduce_or (v)); }
+ __forceinline long long reduce_add(const vllong8& v) { return toScalar(vreduce_add(v)); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Memory load and store operations
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vllong8 permute(const vllong8& v, const vllong8& index) {
+ return _mm512_permutexvar_epi64(index,v);
+ }
+
+ __forceinline vllong8 reverse(const vllong8& a) {
+ return permute(a,vllong8(reverse_step));
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Output Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline embree_ostream operator <<(embree_ostream cout, const vllong8& v)
+ {
+ cout << "<" << v[0];
+ for (size_t i=1; i<8; i++) cout << ", " << v[i];
+ cout << ">";
+ return cout;
+ }
+}
diff --git a/thirdparty/embree-aarch64/common/simd/vuint16_avx512.h b/thirdparty/embree-aarch64/common/simd/vuint16_avx512.h
new file mode 100644
index 0000000000..39752611bb
--- /dev/null
+++ b/thirdparty/embree-aarch64/common/simd/vuint16_avx512.h
@@ -0,0 +1,443 @@
+// Copyright 2009-2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+namespace embree
+{
+ /* 16-wide AVX-512 unsigned integer type */
+ template<>
+ struct vuint<16>
+ {
+ ALIGNED_STRUCT_(64);
+
+ typedef vboolf16 Bool;
+ typedef vuint16 UInt;
+ typedef vfloat16 Float;
+
+ enum { size = 16 }; // number of SIMD elements
+ union { // data
+ __m512i v;
+ unsigned int i[16];
+ };
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constructors, Assignment & Cast Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vuint() {}
+ __forceinline vuint(const vuint16& t) { v = t.v; }
+ __forceinline vuint16& operator =(const vuint16& f) { v = f.v; return *this; }
+
+ __forceinline vuint(const __m512i& t) { v = t; }
+ __forceinline operator __m512i() const { return v; }
+ __forceinline operator __m256i() const { return _mm512_castsi512_si256(v); }
+
+ __forceinline vuint(unsigned int i) {
+ v = _mm512_set1_epi32(i);
+ }
+
+ __forceinline vuint(const vuint4& i) {
+ v = _mm512_broadcast_i32x4(i);
+ }
+
+ __forceinline vuint(const vuint8& i) {
+ v = _mm512_castps_si512(_mm512_castpd_ps(_mm512_broadcast_f64x4(_mm256_castsi256_pd(i))));
+ }
+
+ __forceinline vuint(unsigned int a, unsigned int b, unsigned int c, unsigned int d) {
+ v = _mm512_set4_epi32(d,c,b,a);
+ }
+
+ __forceinline vuint(unsigned int a0 , unsigned int a1 , unsigned int a2 , unsigned int a3,
+ unsigned int a4 , unsigned int a5 , unsigned int a6 , unsigned int a7,
+ unsigned int a8 , unsigned int a9 , unsigned int a10, unsigned int a11,
+ unsigned int a12, unsigned int a13, unsigned int a14, unsigned int a15)
+ {
+ v = _mm512_set_epi32(a15,a14,a13,a12,a11,a10,a9,a8,a7,a6,a5,a4,a3,a2,a1,a0);
+ }
+
+ __forceinline explicit vuint(const __m512& f) {
+ v = _mm512_cvtps_epu32(f);
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constants
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vuint(ZeroTy) : v(_mm512_setzero_epi32()) {}
+ __forceinline vuint(OneTy) : v(_mm512_set1_epi32(1)) {}
+ __forceinline vuint(StepTy) : v(_mm512_set_epi32(15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0)) {}
+ __forceinline vuint(ReverseStepTy) : v(_mm512_setr_epi32(15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0)) {}
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Loads and Stores
+ ////////////////////////////////////////////////////////////////////////////////
+
+ static __forceinline void store_nt(void* __restrict__ ptr, const vuint16& a) {
+ _mm512_stream_si512((__m512i*)ptr,a);
+ }
+
+ static __forceinline vuint16 loadu(const void* addr)
+ {
+ return _mm512_loadu_si512(addr);
+ }
+
+ static __forceinline vuint16 loadu(const uint8_t* ptr) { return _mm512_cvtepu8_epi32(_mm_loadu_si128((__m128i*)ptr)); }
+ static __forceinline vuint16 loadu(const unsigned short* ptr) { return _mm512_cvtepu16_epi32(_mm256_loadu_si256((__m256i*)ptr)); }
+
+ static __forceinline vuint16 load(const vuint16* addr) {
+ return _mm512_load_si512(addr);
+ }
+
+ static __forceinline vuint16 load(const unsigned int* addr) {
+ return _mm512_load_si512(addr);
+ }
+
+ static __forceinline vuint16 load(unsigned short* ptr) { return _mm512_cvtepu16_epi32(*(__m256i*)ptr); }
+
+
+ static __forceinline void store(void* ptr, const vuint16& v) {
+ _mm512_store_si512(ptr,v);
+ }
+
+ static __forceinline void storeu(void* ptr, const vuint16& v) {
+ _mm512_storeu_si512(ptr,v);
+ }
+
+ static __forceinline void storeu(const vboolf16& mask, void* ptr, const vuint16& f) {
+ _mm512_mask_storeu_epi32(ptr,mask,f);
+ }
+
+ static __forceinline void store(const vboolf16& mask, void* addr, const vuint16& v2) {
+ _mm512_mask_store_epi32(addr,mask,v2);
+ }
+
+ /* pass by value to avoid compiler generating inefficient code */
+ static __forceinline void storeu_compact(const vboolf16 mask, void* addr, const vuint16 reg) {
+ _mm512_mask_compressstoreu_epi32(addr,mask,reg);
+ }
+
+ static __forceinline void storeu_compact_single(const vboolf16 mask, void* addr, vuint16 reg) {
+ //_mm512_mask_compressstoreu_epi32(addr,mask,reg);
+ *(float*)addr = mm512_cvtss_f32(_mm512_mask_compress_ps(_mm512_castsi512_ps(reg),mask,_mm512_castsi512_ps(reg)));
+ }
+
+ static __forceinline vuint16 compact64bit(const vboolf16& mask, vuint16& v) {
+ return _mm512_mask_compress_epi64(v,mask,v);
+ }
+
+ static __forceinline vuint16 compact(const vboolf16& mask, vuint16& v) {
+ return _mm512_mask_compress_epi32(v,mask,v);
+ }
+
+ static __forceinline vuint16 compact(const vboolf16& mask, const vuint16& a, vuint16& b) {
+ return _mm512_mask_compress_epi32(a,mask,b);
+ }
+
+ static __forceinline vuint16 expand(const vboolf16& mask, const vuint16& a, vuint16& b) {
+ return _mm512_mask_expand_epi32(b,mask,a);
+ }
+
+ template<int scale = 4>
+ static __forceinline vuint16 gather(const unsigned int* ptr, const vint16& index) {
+ return _mm512_i32gather_epi32(index,ptr,scale);
+ }
+
+ template<int scale = 4>
+ static __forceinline vuint16 gather(const vboolf16& mask, const unsigned int* ptr, const vint16& index) {
+ return _mm512_mask_i32gather_epi32(_mm512_undefined_epi32(),mask,index,ptr,scale);
+ }
+
+ template<int scale = 4>
+ static __forceinline vuint16 gather(const vboolf16& mask, vuint16& dest, const unsigned int* ptr, const vint16& index) {
+ return _mm512_mask_i32gather_epi32(dest,mask,index,ptr,scale);
+ }
+
+ template<int scale = 4>
+ static __forceinline void scatter(unsigned int* ptr, const vint16& index, const vuint16& v) {
+ _mm512_i32scatter_epi32((int*)ptr,index,v,scale);
+ }
+
+ template<int scale = 4>
+ static __forceinline void scatter(const vboolf16& mask, unsigned int* ptr, const vint16& index, const vuint16& v) {
+ _mm512_mask_i32scatter_epi32((int*)ptr,mask,index,v,scale);
+ }
+
+ static __forceinline vuint16 broadcast64bit(size_t v) {
+ return _mm512_set1_epi64(v);
+ }
+
+ static __forceinline size_t extract64bit(const vuint16& v)
+ {
+ return _mm_cvtsi128_si64(_mm512_castsi512_si128(v));
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Array Access
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline unsigned int& operator [](size_t index) { assert(index < 16); return i[index]; }
+ __forceinline const unsigned int& operator [](size_t index) const { assert(index < 16); return i[index]; }
+
+ __forceinline unsigned int uint (size_t index) const { assert(index < 16); return ((unsigned int*)i)[index]; }
+ __forceinline size_t& uint64_t(size_t index) const { assert(index < 8); return ((size_t*)i)[index]; }
+ };
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Unary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboolf16 asBool(const vuint16& a) { return _mm512_movepi32_mask(a); }
+
+ __forceinline vuint16 operator +(const vuint16& a) { return a; }
+ __forceinline vuint16 operator -(const vuint16& a) { return _mm512_sub_epi32(_mm512_setzero_epi32(), a); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Binary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vuint16 operator +(const vuint16& a, const vuint16& b) { return _mm512_add_epi32(a, b); }
+ __forceinline vuint16 operator +(const vuint16& a, unsigned int b) { return a + vuint16(b); }
+ __forceinline vuint16 operator +(unsigned int a, const vuint16& b) { return vuint16(a) + b; }
+
+ __forceinline vuint16 operator -(const vuint16& a, const vuint16& b) { return _mm512_sub_epi32(a, b); }
+ __forceinline vuint16 operator -(const vuint16& a, unsigned int b) { return a - vuint16(b); }
+ __forceinline vuint16 operator -(unsigned int a, const vuint16& b) { return vuint16(a) - b; }
+
+ __forceinline vuint16 operator *(const vuint16& a, const vuint16& b) { return _mm512_mul_epu32(a, b); }
+ __forceinline vuint16 operator *(const vuint16& a, unsigned int b) { return a * vuint16(b); }
+ __forceinline vuint16 operator *(unsigned int a, const vuint16& b) { return vuint16(a) * b; }
+
+ __forceinline vuint16 operator &(const vuint16& a, const vuint16& b) { return _mm512_and_epi32(a, b); }
+ __forceinline vuint16 operator &(const vuint16& a, unsigned int b) { return a & vuint16(b); }
+ __forceinline vuint16 operator &(unsigned int a, const vuint16& b) { return vuint16(a) & b; }
+
+ __forceinline vuint16 operator |(const vuint16& a, const vuint16& b) { return _mm512_or_epi32(a, b); }
+ __forceinline vuint16 operator |(const vuint16& a, unsigned int b) { return a | vuint16(b); }
+ __forceinline vuint16 operator |(unsigned int a, const vuint16& b) { return vuint16(a) | b; }
+
+ __forceinline vuint16 operator ^(const vuint16& a, const vuint16& b) { return _mm512_xor_epi32(a, b); }
+ __forceinline vuint16 operator ^(const vuint16& a, unsigned int b) { return a ^ vuint16(b); }
+ __forceinline vuint16 operator ^(unsigned int a, const vuint16& b) { return vuint16(a) ^ b; }
+
+ __forceinline vuint16 operator <<(const vuint16& a, unsigned int n) { return _mm512_slli_epi32(a, n); }
+ __forceinline vuint16 operator >>(const vuint16& a, unsigned int n) { return _mm512_srli_epi32(a, n); }
+
+ __forceinline vuint16 operator <<(const vuint16& a, const vuint16& n) { return _mm512_sllv_epi32(a, n); }
+ __forceinline vuint16 operator >>(const vuint16& a, const vuint16& n) { return _mm512_srlv_epi32(a, n); }
+
+ __forceinline vuint16 sll (const vuint16& a, unsigned int b) { return _mm512_slli_epi32(a, b); }
+ __forceinline vuint16 sra (const vuint16& a, unsigned int b) { return _mm512_srai_epi32(a, b); }
+ __forceinline vuint16 srl (const vuint16& a, unsigned int b) { return _mm512_srli_epi32(a, b); }
+
+ __forceinline vuint16 min(const vuint16& a, const vuint16& b) { return _mm512_min_epu32(a, b); }
+ __forceinline vuint16 min(const vuint16& a, unsigned int b) { return min(a,vuint16(b)); }
+ __forceinline vuint16 min(unsigned int a, const vuint16& b) { return min(vuint16(a),b); }
+
+ __forceinline vuint16 max(const vuint16& a, const vuint16& b) { return _mm512_max_epu32(a, b); }
+ __forceinline vuint16 max(const vuint16& a, unsigned int b) { return max(a,vuint16(b)); }
+ __forceinline vuint16 max(unsigned int a, const vuint16& b) { return max(vuint16(a),b); }
+
+ __forceinline vuint16 mask_add(const vboolf16& mask, vuint16& c, const vuint16& a, const vuint16& b) { return _mm512_mask_add_epi32(c,mask,a,b); }
+ __forceinline vuint16 mask_sub(const vboolf16& mask, vuint16& c, const vuint16& a, const vuint16& b) { return _mm512_mask_sub_epi32(c,mask,a,b); }
+
+ __forceinline vuint16 mask_and(const vboolf16& m, vuint16& c, const vuint16& a, const vuint16& b) { return _mm512_mask_and_epi32(c,m,a,b); }
+ __forceinline vuint16 mask_or (const vboolf16& m, vuint16& c, const vuint16& a, const vuint16& b) { return _mm512_mask_or_epi32(c,m,a,b); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Assignment Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vuint16& operator +=(vuint16& a, const vuint16& b) { return a = a + b; }
+ __forceinline vuint16& operator +=(vuint16& a, unsigned int b) { return a = a + b; }
+
+ __forceinline vuint16& operator -=(vuint16& a, const vuint16& b) { return a = a - b; }
+ __forceinline vuint16& operator -=(vuint16& a, unsigned int b) { return a = a - b; }
+
+ __forceinline vuint16& operator *=(vuint16& a, const vuint16& b) { return a = a * b; }
+ __forceinline vuint16& operator *=(vuint16& a, unsigned int b) { return a = a * b; }
+
+ __forceinline vuint16& operator &=(vuint16& a, const vuint16& b) { return a = a & b; }
+ __forceinline vuint16& operator &=(vuint16& a, unsigned int b) { return a = a & b; }
+
+ __forceinline vuint16& operator |=(vuint16& a, const vuint16& b) { return a = a | b; }
+ __forceinline vuint16& operator |=(vuint16& a, unsigned int b) { return a = a | b; }
+
+ __forceinline vuint16& operator <<=(vuint16& a, unsigned int b) { return a = a << b; }
+ __forceinline vuint16& operator >>=(vuint16& a, unsigned int b) { return a = a >> b; }
+
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Comparison Operators + Select
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboolf16 operator ==(const vuint16& a, const vuint16& b) { return _mm512_cmp_epu32_mask(a,b,_MM_CMPINT_EQ); }
+ __forceinline vboolf16 operator ==(const vuint16& a, unsigned int b) { return a == vuint16(b); }
+ __forceinline vboolf16 operator ==(unsigned int a, const vuint16& b) { return vuint16(a) == b; }
+
+ __forceinline vboolf16 operator !=(const vuint16& a, const vuint16& b) { return _mm512_cmp_epu32_mask(a,b,_MM_CMPINT_NE); }
+ __forceinline vboolf16 operator !=(const vuint16& a, unsigned int b) { return a != vuint16(b); }
+ __forceinline vboolf16 operator !=(unsigned int a, const vuint16& b) { return vuint16(a) != b; }
+
+ __forceinline vboolf16 operator < (const vuint16& a, const vuint16& b) { return _mm512_cmp_epu32_mask(a,b,_MM_CMPINT_LT); }
+ __forceinline vboolf16 operator < (const vuint16& a, unsigned int b) { return a < vuint16(b); }
+ __forceinline vboolf16 operator < (unsigned int a, const vuint16& b) { return vuint16(a) < b; }
+
+ __forceinline vboolf16 operator >=(const vuint16& a, const vuint16& b) { return _mm512_cmp_epu32_mask(a,b,_MM_CMPINT_GE); }
+ __forceinline vboolf16 operator >=(const vuint16& a, unsigned int b) { return a >= vuint16(b); }
+ __forceinline vboolf16 operator >=(unsigned int a, const vuint16& b) { return vuint16(a) >= b; }
+
+ __forceinline vboolf16 operator > (const vuint16& a, const vuint16& b) { return _mm512_cmp_epu32_mask(a,b,_MM_CMPINT_GT); }
+ __forceinline vboolf16 operator > (const vuint16& a, unsigned int b) { return a > vuint16(b); }
+ __forceinline vboolf16 operator > (unsigned int a, const vuint16& b) { return vuint16(a) > b; }
+
+ __forceinline vboolf16 operator <=(const vuint16& a, const vuint16& b) { return _mm512_cmp_epu32_mask(a,b,_MM_CMPINT_LE); }
+ __forceinline vboolf16 operator <=(const vuint16& a, unsigned int b) { return a <= vuint16(b); }
+ __forceinline vboolf16 operator <=(unsigned int a, const vuint16& b) { return vuint16(a) <= b; }
+
+ __forceinline vboolf16 eq(const vuint16& a, const vuint16& b) { return _mm512_cmp_epu32_mask(a,b,_MM_CMPINT_EQ); }
+ __forceinline vboolf16 ne(const vuint16& a, const vuint16& b) { return _mm512_cmp_epu32_mask(a,b,_MM_CMPINT_NE); }
+ __forceinline vboolf16 lt(const vuint16& a, const vuint16& b) { return _mm512_cmp_epu32_mask(a,b,_MM_CMPINT_LT); }
+ __forceinline vboolf16 ge(const vuint16& a, const vuint16& b) { return _mm512_cmp_epu32_mask(a,b,_MM_CMPINT_GE); }
+ __forceinline vboolf16 gt(const vuint16& a, const vuint16& b) { return _mm512_cmp_epu32_mask(a,b,_MM_CMPINT_GT); }
+ __forceinline vboolf16 le(const vuint16& a, const vuint16& b) { return _mm512_cmp_epu32_mask(a,b,_MM_CMPINT_LE); }
+
+ __forceinline vboolf16 eq(const vboolf16 mask, const vuint16& a, const vuint16& b) { return _mm512_mask_cmp_epu32_mask(mask,a,b,_MM_CMPINT_EQ); }
+ __forceinline vboolf16 ne(const vboolf16 mask, const vuint16& a, const vuint16& b) { return _mm512_mask_cmp_epu32_mask(mask,a,b,_MM_CMPINT_NE); }
+ __forceinline vboolf16 lt(const vboolf16 mask, const vuint16& a, const vuint16& b) { return _mm512_mask_cmp_epu32_mask(mask,a,b,_MM_CMPINT_LT); }
+ __forceinline vboolf16 ge(const vboolf16 mask, const vuint16& a, const vuint16& b) { return _mm512_mask_cmp_epu32_mask(mask,a,b,_MM_CMPINT_GE); }
+ __forceinline vboolf16 gt(const vboolf16 mask, const vuint16& a, const vuint16& b) { return _mm512_mask_cmp_epu32_mask(mask,a,b,_MM_CMPINT_GT); }
+ __forceinline vboolf16 le(const vboolf16 mask, const vuint16& a, const vuint16& b) { return _mm512_mask_cmp_epu32_mask(mask,a,b,_MM_CMPINT_LE); }
+
+
+ __forceinline vuint16 select(const vboolf16& m, const vuint16& t, const vuint16& f) {
+ return _mm512_mask_or_epi32(f,m,t,t);
+ }
+
+ __forceinline void xchg(const vboolf16& m, vuint16& a, vuint16& b) {
+ const vuint16 c = a; a = select(m,b,a); b = select(m,c,b);
+ }
+
+ __forceinline vboolf16 test(const vboolf16& m, const vuint16& a, const vuint16& b) {
+ return _mm512_mask_test_epi32_mask(m,a,b);
+ }
+
+ __forceinline vboolf16 test(const vuint16& a, const vuint16& b) {
+ return _mm512_test_epi32_mask(a,b);
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ // Movement/Shifting/Shuffling Functions
+ ////////////////////////////////////////////////////////////////////////////////
+
+ template<int i>
+ __forceinline vuint16 shuffle(const vuint16& v) {
+ return _mm512_castps_si512(_mm512_permute_ps(_mm512_castsi512_ps(v), _MM_SHUFFLE(i, i, i, i)));
+ }
+
+ template<int i0, int i1, int i2, int i3>
+ __forceinline vuint16 shuffle(const vuint16& v) {
+ return _mm512_castps_si512(_mm512_permute_ps(_mm512_castsi512_ps(v), _MM_SHUFFLE(i3, i2, i1, i0)));
+ }
+
+ template<int i>
+ __forceinline vuint16 shuffle4(const vuint16& v) {
+ return _mm512_castps_si512(_mm512_shuffle_f32x4(_mm512_castsi512_ps(v), _mm512_castsi512_ps(v) ,_MM_SHUFFLE(i, i, i, i)));
+ }
+
+ template<int i0, int i1, int i2, int i3>
+ __forceinline vuint16 shuffle4(const vuint16& v) {
+ return _mm512_castps_si512(_mm512_shuffle_f32x4(_mm512_castsi512_ps(v), _mm512_castsi512_ps(v), _MM_SHUFFLE(i3, i2, i1, i0)));
+ }
+
+ template<int i>
+ __forceinline vuint16 align_shift_right(const vuint16& a, const vuint16& b) {
+ return _mm512_alignr_epi32(a, b, i);
+ };
+
+ __forceinline unsigned int toScalar(const vuint16& v) {
+ return _mm_cvtsi128_si32(_mm512_castsi512_si128(v));
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Reductions
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vuint16 vreduce_min2(vuint16 x) { return min(x, shuffle<1,0,3,2>(x)); }
+ __forceinline vuint16 vreduce_min4(vuint16 x) { x = vreduce_min2(x); return min(x, shuffle<2,3,0,1>(x)); }
+ __forceinline vuint16 vreduce_min8(vuint16 x) { x = vreduce_min4(x); return min(x, shuffle4<1,0,3,2>(x)); }
+ __forceinline vuint16 vreduce_min (vuint16 x) { x = vreduce_min8(x); return min(x, shuffle4<2,3,0,1>(x)); }
+
+ __forceinline vuint16 vreduce_max2(vuint16 x) { return max(x, shuffle<1,0,3,2>(x)); }
+ __forceinline vuint16 vreduce_max4(vuint16 x) { x = vreduce_max2(x); return max(x, shuffle<2,3,0,1>(x)); }
+ __forceinline vuint16 vreduce_max8(vuint16 x) { x = vreduce_max4(x); return max(x, shuffle4<1,0,3,2>(x)); }
+ __forceinline vuint16 vreduce_max (vuint16 x) { x = vreduce_max8(x); return max(x, shuffle4<2,3,0,1>(x)); }
+
+ __forceinline vuint16 vreduce_and2(vuint16 x) { return x & shuffle<1,0,3,2>(x); }
+ __forceinline vuint16 vreduce_and4(vuint16 x) { x = vreduce_and2(x); return x & shuffle<2,3,0,1>(x); }
+ __forceinline vuint16 vreduce_and8(vuint16 x) { x = vreduce_and4(x); return x & shuffle4<1,0,3,2>(x); }
+ __forceinline vuint16 vreduce_and (vuint16 x) { x = vreduce_and8(x); return x & shuffle4<2,3,0,1>(x); }
+
+ __forceinline vuint16 vreduce_or2(vuint16 x) { return x | shuffle<1,0,3,2>(x); }
+ __forceinline vuint16 vreduce_or4(vuint16 x) { x = vreduce_or2(x); return x | shuffle<2,3,0,1>(x); }
+ __forceinline vuint16 vreduce_or8(vuint16 x) { x = vreduce_or4(x); return x | shuffle4<1,0,3,2>(x); }
+ __forceinline vuint16 vreduce_or (vuint16 x) { x = vreduce_or8(x); return x | shuffle4<2,3,0,1>(x); }
+
+ __forceinline vuint16 vreduce_add2(vuint16 x) { return x + shuffle<1,0,3,2>(x); }
+ __forceinline vuint16 vreduce_add4(vuint16 x) { x = vreduce_add2(x); return x + shuffle<2,3,0,1>(x); }
+ __forceinline vuint16 vreduce_add8(vuint16 x) { x = vreduce_add4(x); return x + shuffle4<1,0,3,2>(x); }
+ __forceinline vuint16 vreduce_add (vuint16 x) { x = vreduce_add8(x); return x + shuffle4<2,3,0,1>(x); }
+
+ __forceinline unsigned int reduce_min(const vuint16& v) { return toScalar(vreduce_min(v)); }
+ __forceinline unsigned int reduce_max(const vuint16& v) { return toScalar(vreduce_max(v)); }
+ __forceinline unsigned int reduce_and(const vuint16& v) { return toScalar(vreduce_and(v)); }
+ __forceinline unsigned int reduce_or (const vuint16& v) { return toScalar(vreduce_or (v)); }
+ __forceinline unsigned int reduce_add(const vuint16& v) { return toScalar(vreduce_add(v)); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Memory load and store operations
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vuint16 permute(vuint16 v, vuint16 index) {
+ return _mm512_permutexvar_epi32(index,v);
+ }
+
+ __forceinline vuint16 reverse(const vuint16& a) {
+ return permute(a,vuint16(reverse_step));
+ }
+
+ __forceinline vuint16 prefix_sum(const vuint16& a)
+ {
+ const vuint16 z(zero);
+ vuint16 v = a;
+ v = v + align_shift_right<16-1>(v,z);
+ v = v + align_shift_right<16-2>(v,z);
+ v = v + align_shift_right<16-4>(v,z);
+ v = v + align_shift_right<16-8>(v,z);
+ return v;
+ }
+
+ __forceinline vuint16 reverse_prefix_sum(const vuint16& a)
+ {
+ const vuint16 z(zero);
+ vuint16 v = a;
+ v = v + align_shift_right<1>(z,v);
+ v = v + align_shift_right<2>(z,v);
+ v = v + align_shift_right<4>(z,v);
+ v = v + align_shift_right<8>(z,v);
+ return v;
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Output Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline embree_ostream operator <<(embree_ostream cout, const vuint16& v)
+ {
+ cout << "<" << v[0];
+ for (int i=1; i<16; i++) cout << ", " << v[i];
+ cout << ">";
+ return cout;
+ }
+}
diff --git a/thirdparty/embree-aarch64/common/simd/vuint4_sse2.h b/thirdparty/embree-aarch64/common/simd/vuint4_sse2.h
new file mode 100644
index 0000000000..a3f393ebf2
--- /dev/null
+++ b/thirdparty/embree-aarch64/common/simd/vuint4_sse2.h
@@ -0,0 +1,499 @@
+// Copyright 2009-2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+#include "../math/math.h"
+
+namespace embree
+{
+ /* 4-wide SSE integer type */
+ template<>
+ struct vuint<4>
+ {
+ ALIGNED_STRUCT_(16);
+
+ typedef vboolf4 Bool;
+ typedef vuint4 Int;
+ typedef vfloat4 Float;
+
+ enum { size = 4 }; // number of SIMD elements
+ union { __m128i v; unsigned int i[4]; }; // data
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constructors, Assignment & Cast Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vuint() {}
+ __forceinline vuint(const vuint4& a) { v = a.v; }
+ __forceinline vuint4& operator =(const vuint4& a) { v = a.v; return *this; }
+
+ __forceinline vuint(const __m128i a) : v(a) {}
+ __forceinline operator const __m128i&() const { return v; }
+ __forceinline operator __m128i&() { return v; }
+
+
+ __forceinline vuint(unsigned int a) : v(_mm_set1_epi32(a)) {}
+ __forceinline vuint(unsigned int a, unsigned int b, unsigned int c, unsigned int d) : v(_mm_set_epi32(d, c, b, a)) {}
+
+#if defined(__AVX512VL__)
+ __forceinline explicit vuint(__m128 a) : v(_mm_cvtps_epu32(a)) {}
+#endif
+
+#if defined(__AVX512VL__)
+ __forceinline explicit vuint(const vboolf4& a) : v(_mm_movm_epi32(a)) {}
+#else
+ __forceinline explicit vuint(const vboolf4& a) : v(_mm_castps_si128((__m128)a)) {}
+#endif
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constants
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vuint(ZeroTy) : v(_mm_setzero_si128()) {}
+ __forceinline vuint(OneTy) : v(_mm_set1_epi32(1)) {}
+ __forceinline vuint(PosInfTy) : v(_mm_set1_epi32(unsigned(pos_inf))) {}
+ __forceinline vuint(StepTy) : v(_mm_set_epi32(3, 2, 1, 0)) {}
+ __forceinline vuint(TrueTy) { v = _mm_cmpeq_epi32(v,v); }
+ __forceinline vuint(UndefinedTy) : v(_mm_castps_si128(_mm_undefined_ps())) {}
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Loads and Stores
+ ////////////////////////////////////////////////////////////////////////////////
+
+ static __forceinline vuint4 load (const void* a) { return _mm_load_si128((__m128i*)a); }
+ static __forceinline vuint4 loadu(const void* a) { return _mm_loadu_si128((__m128i*)a); }
+
+ static __forceinline void store (void* ptr, const vuint4& v) { _mm_store_si128((__m128i*)ptr,v); }
+ static __forceinline void storeu(void* ptr, const vuint4& v) { _mm_storeu_si128((__m128i*)ptr,v); }
+
+#if defined(__AVX512VL__)
+ static __forceinline vuint4 load (const vboolf4& mask, const void* ptr) { return _mm_mask_load_epi32 (_mm_setzero_si128(),mask,ptr); }
+ static __forceinline vuint4 loadu(const vboolf4& mask, const void* ptr) { return _mm_mask_loadu_epi32(_mm_setzero_si128(),mask,ptr); }
+
+ static __forceinline void store (const vboolf4& mask, void* ptr, const vuint4& v) { _mm_mask_store_epi32 (ptr,mask,v); }
+ static __forceinline void storeu(const vboolf4& mask, void* ptr, const vuint4& v) { _mm_mask_storeu_epi32(ptr,mask,v); }
+#elif defined(__AVX__)
+ static __forceinline vuint4 load (const vbool4& mask, const void* a) { return _mm_castps_si128(_mm_maskload_ps((float*)a,mask)); }
+ static __forceinline vuint4 loadu(const vbool4& mask, const void* a) { return _mm_castps_si128(_mm_maskload_ps((float*)a,mask)); }
+
+ static __forceinline void store (const vboolf4& mask, void* ptr, const vuint4& i) { _mm_maskstore_ps((float*)ptr,(__m128i)mask,_mm_castsi128_ps(i)); }
+ static __forceinline void storeu(const vboolf4& mask, void* ptr, const vuint4& i) { _mm_maskstore_ps((float*)ptr,(__m128i)mask,_mm_castsi128_ps(i)); }
+#else
+ static __forceinline vuint4 load (const vbool4& mask, const void* a) { return _mm_and_si128(_mm_load_si128 ((__m128i*)a),mask); }
+ static __forceinline vuint4 loadu(const vbool4& mask, const void* a) { return _mm_and_si128(_mm_loadu_si128((__m128i*)a),mask); }
+
+ static __forceinline void store (const vboolf4& mask, void* ptr, const vuint4& i) { store (ptr,select(mask,i,load (ptr))); }
+ static __forceinline void storeu(const vboolf4& mask, void* ptr, const vuint4& i) { storeu(ptr,select(mask,i,loadu(ptr))); }
+#endif
+
+#if defined(__aarch64__)
+ static __forceinline vuint4 load(const uint8_t* ptr) {
+ return _mm_load4epu8_epi32(((__m128i*)ptr));
+ }
+ static __forceinline vuint4 loadu(const uint8_t* ptr) {
+ return _mm_load4epu8_epi32(((__m128i*)ptr));
+ }
+#elif defined(__SSE4_1__)
+ static __forceinline vuint4 load(const uint8_t* ptr) {
+ return _mm_cvtepu8_epi32(_mm_loadl_epi64((__m128i*)ptr));
+ }
+
+ static __forceinline vuint4 loadu(const uint8_t* ptr) {
+ return _mm_cvtepu8_epi32(_mm_loadl_epi64((__m128i*)ptr));
+ }
+
+#endif
+
+ static __forceinline vuint4 load(const unsigned short* ptr) {
+#if defined(__aarch64__)
+ return _mm_load4epu16_epi32(((__m128i*)ptr));
+#elif defined (__SSE4_1__)
+ return _mm_cvtepu16_epi32(_mm_loadu_si128((__m128i*)ptr));
+#else
+ return vuint4(ptr[0],ptr[1],ptr[2],ptr[3]);
+#endif
+ }
+
+ static __forceinline void store_uint8(uint8_t* ptr, const vuint4& v) {
+#if defined(__aarch64__)
+ uint32x4_t x = uint32x4_t(v.v);
+ uint16x4_t y = vqmovn_u32(x);
+ uint8x8_t z = vqmovn_u16(vcombine_u16(y, y));
+ vst1_lane_u32((uint32_t *)ptr, uint32x2_t(z), 0);
+#elif defined(__SSE4_1__)
+ __m128i x = v;
+ x = _mm_packus_epi32(x, x);
+ x = _mm_packus_epi16(x, x);
+ *(unsigned*)ptr = _mm_cvtsi128_si32(x);
+#else
+ for (size_t i=0;i<4;i++)
+ ptr[i] = (uint8_t)v[i];
+#endif
+ }
+
+ static __forceinline void store_uint8(unsigned short* ptr, const vuint4& v) {
+#if defined(__aarch64__)
+ uint32x4_t x = (uint32x4_t)v.v;
+ uint16x4_t y = vqmovn_u32(x);
+ vst1_u16(ptr, y);
+#else
+ for (size_t i=0;i<4;i++)
+ ptr[i] = (unsigned short)v[i];
+#endif
+ }
+
+ static __forceinline vuint4 load_nt(void* ptr) {
+#if (defined(__aarch64__)) || defined(__SSE4_1__)
+ return _mm_stream_load_si128((__m128i*)ptr);
+#else
+ return _mm_load_si128((__m128i*)ptr);
+#endif
+ }
+
+ static __forceinline void store_nt(void* ptr, const vuint4& v) {
+#if !defined(__aarch64__) && defined(__SSE4_1__)
+ _mm_stream_ps((float*)ptr, _mm_castsi128_ps(v));
+#else
+ _mm_store_si128((__m128i*)ptr,v);
+#endif
+ }
+
+ template<int scale = 4>
+ static __forceinline vuint4 gather(const unsigned int* ptr, const vint4& index) {
+#if defined(__AVX2__) && !defined(__aarch64__)
+ return _mm_i32gather_epi32((const int*)ptr, index, scale);
+#else
+ return vuint4(
+ *(unsigned int*)(((int8_t*)ptr)+scale*index[0]),
+ *(unsigned int*)(((int8_t*)ptr)+scale*index[1]),
+ *(unsigned int*)(((int8_t*)ptr)+scale*index[2]),
+ *(unsigned int*)(((int8_t*)ptr)+scale*index[3]));
+#endif
+ }
+
+ template<int scale = 4>
+ static __forceinline vuint4 gather(const vboolf4& mask, const unsigned int* ptr, const vint4& index) {
+ vuint4 r = zero;
+#if defined(__AVX512VL__)
+ return _mm_mmask_i32gather_epi32(r, mask, index, ptr, scale);
+#elif defined(__AVX2__) && !defined(__aarch64__)
+ return _mm_mask_i32gather_epi32(r, (const int*)ptr, index, mask, scale);
+#else
+ if (likely(mask[0])) r[0] = *(unsigned int*)(((int8_t*)ptr)+scale*index[0]);
+ if (likely(mask[1])) r[1] = *(unsigned int*)(((int8_t*)ptr)+scale*index[1]);
+ if (likely(mask[2])) r[2] = *(unsigned int*)(((int8_t*)ptr)+scale*index[2]);
+ if (likely(mask[3])) r[3] = *(unsigned int*)(((int8_t*)ptr)+scale*index[3]);
+ return r;
+#endif
+ }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Array Access
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline const unsigned int& operator [](size_t index) const { assert(index < 4); return i[index]; }
+ __forceinline unsigned int& operator [](size_t index) { assert(index < 4); return i[index]; }
+
+ friend __forceinline vuint4 select(const vboolf4& m, const vuint4& t, const vuint4& f) {
+#if defined(__AVX512VL__)
+ return _mm_mask_blend_epi32(m, (__m128i)f, (__m128i)t);
+#elif defined(__SSE4_1__)
+ return _mm_castps_si128(_mm_blendv_ps(_mm_castsi128_ps(f), _mm_castsi128_ps(t), m));
+#else
+ return _mm_or_si128(_mm_and_si128(m, t), _mm_andnot_si128(m, f));
+#endif
+ }
+ };
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Unary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+#if defined(__AVX512VL__)
+ __forceinline vboolf4 asBool(const vuint4& a) { return _mm_movepi32_mask(a); }
+#else
+ __forceinline vboolf4 asBool(const vuint4& a) { return _mm_castsi128_ps(a); }
+#endif
+
+ __forceinline vuint4 operator +(const vuint4& a) { return a; }
+ __forceinline vuint4 operator -(const vuint4& a) { return _mm_sub_epi32(_mm_setzero_si128(), a); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Binary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vuint4 operator +(const vuint4& a, const vuint4& b) { return _mm_add_epi32(a, b); }
+ __forceinline vuint4 operator +(const vuint4& a, unsigned int b) { return a + vuint4(b); }
+ __forceinline vuint4 operator +(unsigned int a, const vuint4& b) { return vuint4(a) + b; }
+
+ __forceinline vuint4 operator -(const vuint4& a, const vuint4& b) { return _mm_sub_epi32(a, b); }
+ __forceinline vuint4 operator -(const vuint4& a, unsigned int b) { return a - vuint4(b); }
+ __forceinline vuint4 operator -(unsigned int a, const vuint4& b) { return vuint4(a) - b; }
+
+//#if defined(__SSE4_1__)
+// __forceinline vuint4 operator *(const vuint4& a, const vuint4& b) { return _mm_mullo_epu32(a, b); }
+//#else
+// __forceinline vuint4 operator *(const vuint4& a, const vuint4& b) { return vuint4(a[0]*b[0],a[1]*b[1],a[2]*b[2],a[3]*b[3]); }
+//#endif
+// __forceinline vuint4 operator *(const vuint4& a, unsigned int b) { return a * vuint4(b); }
+// __forceinline vuint4 operator *(unsigned int a, const vuint4& b) { return vuint4(a) * b; }
+
+ __forceinline vuint4 operator &(const vuint4& a, const vuint4& b) { return _mm_and_si128(a, b); }
+ __forceinline vuint4 operator &(const vuint4& a, unsigned int b) { return a & vuint4(b); }
+ __forceinline vuint4 operator &(unsigned int a, const vuint4& b) { return vuint4(a) & b; }
+
+ __forceinline vuint4 operator |(const vuint4& a, const vuint4& b) { return _mm_or_si128(a, b); }
+ __forceinline vuint4 operator |(const vuint4& a, unsigned int b) { return a | vuint4(b); }
+ __forceinline vuint4 operator |(unsigned int a, const vuint4& b) { return vuint4(a) | b; }
+
+ __forceinline vuint4 operator ^(const vuint4& a, const vuint4& b) { return _mm_xor_si128(a, b); }
+ __forceinline vuint4 operator ^(const vuint4& a, unsigned int b) { return a ^ vuint4(b); }
+ __forceinline vuint4 operator ^(unsigned int a, const vuint4& b) { return vuint4(a) ^ b; }
+
+ __forceinline vuint4 operator <<(const vuint4& a, unsigned int n) { return _mm_slli_epi32(a, n); }
+ __forceinline vuint4 operator >>(const vuint4& a, unsigned int n) { return _mm_srli_epi32(a, n); }
+
+ __forceinline vuint4 sll (const vuint4& a, unsigned int b) { return _mm_slli_epi32(a, b); }
+ __forceinline vuint4 sra (const vuint4& a, unsigned int b) { return _mm_srai_epi32(a, b); }
+ __forceinline vuint4 srl (const vuint4& a, unsigned int b) { return _mm_srli_epi32(a, b); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Assignment Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vuint4& operator +=(vuint4& a, const vuint4& b) { return a = a + b; }
+ __forceinline vuint4& operator +=(vuint4& a, unsigned int b) { return a = a + b; }
+
+ __forceinline vuint4& operator -=(vuint4& a, const vuint4& b) { return a = a - b; }
+ __forceinline vuint4& operator -=(vuint4& a, unsigned int b) { return a = a - b; }
+
+//#if defined(__SSE4_1__)
+// __forceinline vuint4& operator *=(vuint4& a, const vuint4& b) { return a = a * b; }
+// __forceinline vuint4& operator *=(vuint4& a, unsigned int b) { return a = a * b; }
+//#endif
+
+ __forceinline vuint4& operator &=(vuint4& a, const vuint4& b) { return a = a & b; }
+ __forceinline vuint4& operator &=(vuint4& a, unsigned int b) { return a = a & b; }
+
+ __forceinline vuint4& operator |=(vuint4& a, const vuint4& b) { return a = a | b; }
+ __forceinline vuint4& operator |=(vuint4& a, unsigned int b) { return a = a | b; }
+
+ __forceinline vuint4& operator <<=(vuint4& a, unsigned int b) { return a = a << b; }
+ __forceinline vuint4& operator >>=(vuint4& a, unsigned int b) { return a = a >> b; }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Comparison Operators + Select
+ ////////////////////////////////////////////////////////////////////////////////
+
+#if defined(__AVX512VL__)
+ __forceinline vboolf4 operator ==(const vuint4& a, const vuint4& b) { return _mm_cmp_epu32_mask(a,b,_MM_CMPINT_EQ); }
+ __forceinline vboolf4 operator !=(const vuint4& a, const vuint4& b) { return _mm_cmp_epu32_mask(a,b,_MM_CMPINT_NE); }
+ //__forceinline vboolf4 operator < (const vuint4& a, const vuint4& b) { return _mm_cmp_epu32_mask(a,b,_MM_CMPINT_LT); }
+ //__forceinline vboolf4 operator >=(const vuint4& a, const vuint4& b) { return _mm_cmp_epu32_mask(a,b,_MM_CMPINT_GE); }
+ //__forceinline vboolf4 operator > (const vuint4& a, const vuint4& b) { return _mm_cmp_epu32_mask(a,b,_MM_CMPINT_GT); }
+ //__forceinline vboolf4 operator <=(const vuint4& a, const vuint4& b) { return _mm_cmp_epu32_mask(a,b,_MM_CMPINT_LE); }
+#else
+ __forceinline vboolf4 operator ==(const vuint4& a, const vuint4& b) { return _mm_castsi128_ps(_mm_cmpeq_epi32(a, b)); }
+ __forceinline vboolf4 operator !=(const vuint4& a, const vuint4& b) { return !(a == b); }
+ //__forceinline vboolf4 operator < (const vuint4& a, const vuint4& b) { return _mm_castsi128_ps(_mm_cmplt_epu32(a, b)); }
+ //__forceinline vboolf4 operator >=(const vuint4& a, const vuint4& b) { return !(a < b); }
+ //__forceinline vboolf4 operator > (const vuint4& a, const vuint4& b) { return _mm_castsi128_ps(_mm_cmpgt_epu32(a, b)); }
+ //__forceinline vboolf4 operator <=(const vuint4& a, const vuint4& b) { return !(a > b); }
+#endif
+
+ __forceinline vboolf4 operator ==(const vuint4& a, unsigned int b) { return a == vuint4(b); }
+ __forceinline vboolf4 operator ==(unsigned int a, const vuint4& b) { return vuint4(a) == b; }
+
+ __forceinline vboolf4 operator !=(const vuint4& a, unsigned int b) { return a != vuint4(b); }
+ __forceinline vboolf4 operator !=(unsigned int a, const vuint4& b) { return vuint4(a) != b; }
+
+ //__forceinline vboolf4 operator < (const vuint4& a, unsigned int b) { return a < vuint4(b); }
+ //__forceinline vboolf4 operator < (unsigned int a, const vuint4& b) { return vuint4(a) < b; }
+
+ //__forceinline vboolf4 operator >=(const vuint4& a, unsigned int b) { return a >= vuint4(b); }
+ //__forceinline vboolf4 operator >=(unsigned int a, const vuint4& b) { return vuint4(a) >= b; }
+
+ //__forceinline vboolf4 operator > (const vuint4& a, unsigned int b) { return a > vuint4(b); }
+ //__forceinline vboolf4 operator > (unsigned int a, const vuint4& b) { return vuint4(a) > b; }
+
+ //__forceinline vboolf4 operator <=(const vuint4& a, unsigned int b) { return a <= vuint4(b); }
+ //__forceinline vboolf4 operator <=(unsigned int a, const vuint4& b) { return vuint4(a) <= b; }
+
+ __forceinline vboolf4 eq(const vuint4& a, const vuint4& b) { return a == b; }
+ __forceinline vboolf4 ne(const vuint4& a, const vuint4& b) { return a != b; }
+ //__forceinline vboolf4 lt(const vuint4& a, const vuint4& b) { return a < b; }
+ //__forceinline vboolf4 ge(const vuint4& a, const vuint4& b) { return a >= b; }
+ //__forceinline vboolf4 gt(const vuint4& a, const vuint4& b) { return a > b; }
+ //__forceinline vboolf4 le(const vuint4& a, const vuint4& b) { return a <= b; }
+
+#if defined(__AVX512VL__)
+ __forceinline vboolf4 eq(const vboolf4& mask, const vuint4& a, const vuint4& b) { return _mm_mask_cmp_epu32_mask(mask, a, b, _MM_CMPINT_EQ); }
+ __forceinline vboolf4 ne(const vboolf4& mask, const vuint4& a, const vuint4& b) { return _mm_mask_cmp_epu32_mask(mask, a, b, _MM_CMPINT_NE); }
+ //__forceinline vboolf4 lt(const vboolf4& mask, const vuint4& a, const vuint4& b) { return _mm_mask_cmp_epu32_mask(mask, a, b, _MM_CMPINT_LT); }
+ //__forceinline vboolf4 ge(const vboolf4& mask, const vuint4& a, const vuint4& b) { return _mm_mask_cmp_epu32_mask(mask, a, b, _MM_CMPINT_GE); }
+ //__forceinline vboolf4 gt(const vboolf4& mask, const vuint4& a, const vuint4& b) { return _mm_mask_cmp_epu32_mask(mask, a, b, _MM_CMPINT_GT); }
+ //__forceinline vboolf4 le(const vboolf4& mask, const vuint4& a, const vuint4& b) { return _mm_mask_cmp_epu32_mask(mask, a, b, _MM_CMPINT_LE); }
+#else
+ __forceinline vboolf4 eq(const vboolf4& mask, const vuint4& a, const vuint4& b) { return mask & (a == b); }
+ __forceinline vboolf4 ne(const vboolf4& mask, const vuint4& a, const vuint4& b) { return mask & (a != b); }
+ //__forceinline vboolf4 lt(const vboolf4& mask, const vuint4& a, const vuint4& b) { return mask & (a < b); }
+ //__forceinline vboolf4 ge(const vboolf4& mask, const vuint4& a, const vuint4& b) { return mask & (a >= b); }
+ //__forceinline vboolf4 gt(const vboolf4& mask, const vuint4& a, const vuint4& b) { return mask & (a > b); }
+ //__forceinline vboolf4 le(const vboolf4& mask, const vuint4& a, const vuint4& b) { return mask & (a <= b); }
+#endif
+
+ template<int mask>
+ __forceinline vuint4 select(const vuint4& t, const vuint4& f) {
+#if defined(__SSE4_1__)
+ return _mm_castps_si128(_mm_blend_ps(_mm_castsi128_ps(f), _mm_castsi128_ps(t), mask));
+#else
+ return select(vboolf4(mask), t, f);
+#endif
+ }
+
+/*#if defined(__SSE4_1__)
+ __forceinline vuint4 min(const vuint4& a, const vuint4& b) { return _mm_min_epu32(a, b); }
+ __forceinline vuint4 max(const vuint4& a, const vuint4& b) { return _mm_max_epu32(a, b); }
+
+#else
+ __forceinline vuint4 min(const vuint4& a, const vuint4& b) { return select(a < b,a,b); }
+ __forceinline vuint4 max(const vuint4& a, const vuint4& b) { return select(a < b,b,a); }
+#endif
+
+ __forceinline vuint4 min(const vuint4& a, unsigned int b) { return min(a,vuint4(b)); }
+ __forceinline vuint4 min(unsigned int a, const vuint4& b) { return min(vuint4(a),b); }
+ __forceinline vuint4 max(const vuint4& a, unsigned int b) { return max(a,vuint4(b)); }
+ __forceinline vuint4 max(unsigned int a, const vuint4& b) { return max(vuint4(a),b); }*/
+
+ ////////////////////////////////////////////////////////////////////////////////
+ // Movement/Shifting/Shuffling Functions
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vuint4 unpacklo(const vuint4& a, const vuint4& b) { return _mm_castps_si128(_mm_unpacklo_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b))); }
+ __forceinline vuint4 unpackhi(const vuint4& a, const vuint4& b) { return _mm_castps_si128(_mm_unpackhi_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b))); }
+
+#if defined(__aarch64__)
+ template<int i0, int i1, int i2, int i3>
+ __forceinline vuint4 shuffle(const vuint4& v) {
+ return vreinterpretq_s32_u8(vqtbl1q_u8( (uint8x16_t)v.v, _MN_SHUFFLE(i0, i1, i2, i3)));
+ }
+ template<int i0, int i1, int i2, int i3>
+ __forceinline vuint4 shuffle(const vuint4& a, const vuint4& b) {
+ return vreinterpretq_s32_u8(vqtbl2q_u8( (uint8x16x2_t){(uint8x16_t)a.v, (uint8x16_t)b.v}, _MF_SHUFFLE(i0, i1, i2, i3)));
+ }
+#else
+ template<int i0, int i1, int i2, int i3>
+ __forceinline vuint4 shuffle(const vuint4& v) {
+ return _mm_shuffle_epi32(v, _MM_SHUFFLE(i3, i2, i1, i0));
+ }
+ template<int i0, int i1, int i2, int i3>
+ __forceinline vuint4 shuffle(const vuint4& a, const vuint4& b) {
+ return _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b), _MM_SHUFFLE(i3, i2, i1, i0)));
+ }
+#endif
+#if defined(__SSE3__)
+ template<> __forceinline vuint4 shuffle<0, 0, 2, 2>(const vuint4& v) { return _mm_castps_si128(_mm_moveldup_ps(_mm_castsi128_ps(v))); }
+ template<> __forceinline vuint4 shuffle<1, 1, 3, 3>(const vuint4& v) { return _mm_castps_si128(_mm_movehdup_ps(_mm_castsi128_ps(v))); }
+ template<> __forceinline vuint4 shuffle<0, 1, 0, 1>(const vuint4& v) { return _mm_castpd_si128(_mm_movedup_pd (_mm_castsi128_pd(v))); }
+#endif
+
+ template<int i>
+ __forceinline vuint4 shuffle(const vuint4& v) {
+ return shuffle<i,i,i,i>(v);
+ }
+
+#if defined(__aarch64__)
+ template<int src> __forceinline unsigned int extract(const vuint4& b);
+ template<int dst> __forceinline vuint4 insert(const vuint4& a, const unsigned b);
+#elif defined(__SSE4_1__)
+ template<int src> __forceinline unsigned int extract(const vuint4& b) { return _mm_extract_epi32(b, src); }
+ template<int dst> __forceinline vuint4 insert(const vuint4& a, const unsigned b) { return _mm_insert_epi32(a, b, dst); }
+#else
+ template<int src> __forceinline unsigned int extract(const vuint4& b) { return b[src&3]; }
+ template<int dst> __forceinline vuint4 insert(const vuint4& a, const unsigned b) { vuint4 c = a; c[dst&3] = b; return c; }
+#endif
+
+#if defined(__aarch64__)
+ template<> __forceinline unsigned int extract<0>(const vuint4& b) {
+ return b[0];
+ }
+ template<> __forceinline unsigned int extract<1>(const vuint4& b) {
+ return b[1];
+ }
+ template<> __forceinline unsigned int extract<2>(const vuint4& b) {
+ return b[2];
+ }
+ template<> __forceinline unsigned int extract<3>(const vuint4& b) {
+ return b[3];
+ }
+
+ template<> __forceinline vuint4 insert<0>(const vuint4& a, unsigned b){
+ vuint4 c = a;
+ c[0] = b;
+ return c;
+ }
+ template<> __forceinline vuint4 insert<1>(const vuint4& a, unsigned b){
+ vuint4 c = a;
+ c[1] = b;
+ return c;
+ }
+ template<> __forceinline vuint4 insert<2>(const vuint4& a, unsigned b){
+ vuint4 c = a;
+ c[2] = b;
+ return c;
+ }
+ template<> __forceinline vuint4 insert<3>(const vuint4& a, unsigned b){
+ vuint4 c = a;
+ c[3] = b;
+ return c;
+ }
+
+ __forceinline unsigned int toScalar(const vuint4& v) {
+ return v[0];
+ }
+#else
+ template<> __forceinline unsigned int extract<0>(const vuint4& b) { return _mm_cvtsi128_si32(b); }
+
+ __forceinline unsigned int toScalar(const vuint4& v) { return _mm_cvtsi128_si32(v); }
+#endif
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Reductions
+ ////////////////////////////////////////////////////////////////////////////////
+
+#if 0
+#if defined(__SSE4_1__)
+
+ __forceinline vuint4 vreduce_min(const vuint4& v) { vuint4 h = min(shuffle<1,0,3,2>(v),v); return min(shuffle<2,3,0,1>(h),h); }
+ __forceinline vuint4 vreduce_max(const vuint4& v) { vuint4 h = max(shuffle<1,0,3,2>(v),v); return max(shuffle<2,3,0,1>(h),h); }
+ __forceinline vuint4 vreduce_add(const vuint4& v) { vuint4 h = shuffle<1,0,3,2>(v) + v ; return shuffle<2,3,0,1>(h) + h ; }
+
+ __forceinline unsigned int reduce_min(const vuint4& v) { return toScalar(vreduce_min(v)); }
+ __forceinline unsigned int reduce_max(const vuint4& v) { return toScalar(vreduce_max(v)); }
+ __forceinline unsigned int reduce_add(const vuint4& v) { return toScalar(vreduce_add(v)); }
+
+ __forceinline size_t select_min(const vuint4& v) { return bsf(movemask(v == vreduce_min(v))); }
+ __forceinline size_t select_max(const vuint4& v) { return bsf(movemask(v == vreduce_max(v))); }
+
+ //__forceinline size_t select_min(const vboolf4& valid, const vuint4& v) { const vuint4 a = select(valid,v,vuint4(pos_inf)); return bsf(movemask(valid & (a == vreduce_min(a)))); }
+ //__forceinline size_t select_max(const vboolf4& valid, const vuint4& v) { const vuint4 a = select(valid,v,vuint4(neg_inf)); return bsf(movemask(valid & (a == vreduce_max(a)))); }
+
+#else
+
+ __forceinline unsigned int reduce_min(const vuint4& v) { return min(v[0],v[1],v[2],v[3]); }
+ __forceinline unsigned int reduce_max(const vuint4& v) { return max(v[0],v[1],v[2],v[3]); }
+ __forceinline unsigned int reduce_add(const vuint4& v) { return v[0]+v[1]+v[2]+v[3]; }
+
+#endif
+#endif
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Output Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline embree_ostream operator <<(embree_ostream cout, const vuint4& a) {
+ return cout << "<" << a[0] << ", " << a[1] << ", " << a[2] << ", " << a[3] << ">";
+ }
+}
+
diff --git a/thirdparty/embree-aarch64/common/simd/vuint8_avx.h b/thirdparty/embree-aarch64/common/simd/vuint8_avx.h
new file mode 100644
index 0000000000..d4e86ae92d
--- /dev/null
+++ b/thirdparty/embree-aarch64/common/simd/vuint8_avx.h
@@ -0,0 +1,379 @@
+// Copyright 2009-2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+namespace embree
+{
+ /* 8-wide AVX integer type */
+ template<>
+ struct vuint<8>
+ {
+ ALIGNED_STRUCT_(32);
+
+ typedef vboolf8 Bool;
+ typedef vuint8 Int;
+ typedef vfloat8 Float;
+
+ enum { size = 8 }; // number of SIMD elements
+ union { // data
+ __m256i v;
+ struct { __m128i vl,vh; };
+ unsigned int i[8];
+ };
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constructors, Assignment & Cast Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vuint() {}
+ __forceinline vuint(const vuint8& a) { v = a.v; }
+ __forceinline vuint8& operator =(const vuint8& a) { v = a.v; return *this; }
+
+ __forceinline vuint(__m256i a) : v(a) {}
+ __forceinline operator const __m256i&() const { return v; }
+ __forceinline operator __m256i&() { return v; }
+
+ __forceinline explicit vuint(const vuint4& a) : v(_mm256_insertf128_si256(_mm256_castsi128_si256(a),a,1)) {}
+ __forceinline vuint(const vuint4& a, const vuint4& b) : v(_mm256_insertf128_si256(_mm256_castsi128_si256(a),b,1)) {}
+ __forceinline vuint(const __m128i& a, const __m128i& b) : vl(a), vh(b) {}
+
+ __forceinline explicit vuint(const unsigned int* a) : v(_mm256_castps_si256(_mm256_loadu_ps((const float*)a))) {}
+ __forceinline vuint(unsigned int a) : v(_mm256_set1_epi32(a)) {}
+ __forceinline vuint(unsigned int a, unsigned int b) : v(_mm256_set_epi32(b, a, b, a, b, a, b, a)) {}
+ __forceinline vuint(unsigned int a, unsigned int b, unsigned int c, unsigned int d) : v(_mm256_set_epi32(d, c, b, a, d, c, b, a)) {}
+ __forceinline vuint(unsigned int a, unsigned int b, unsigned int c, unsigned int d, unsigned int e, unsigned int f, unsigned int g, unsigned int vh) : v(_mm256_set_epi32(vh, g, f, e, d, c, b, a)) {}
+
+ __forceinline explicit vuint(__m256 a) : v(_mm256_cvtps_epi32(a)) {}
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constants
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vuint(ZeroTy) : v(_mm256_setzero_si256()) {}
+ __forceinline vuint(OneTy) : v(_mm256_set1_epi32(1)) {}
+ __forceinline vuint(PosInfTy) : v(_mm256_set1_epi32(0xFFFFFFFF)) {}
+ __forceinline vuint(StepTy) : v(_mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0)) {}
+ __forceinline vuint(UndefinedTy) : v(_mm256_undefined_si256()) {}
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Loads and Stores
+ ////////////////////////////////////////////////////////////////////////////////
+
+ static __forceinline vuint8 load (const void* a) { return _mm256_castps_si256(_mm256_load_ps((float*)a)); }
+ static __forceinline vuint8 loadu(const void* a) { return _mm256_castps_si256(_mm256_loadu_ps((float*)a)); }
+
+ static __forceinline vuint8 load (const vboolf8& mask, const void* a) { return _mm256_castps_si256(_mm256_maskload_ps((float*)a,mask)); }
+ static __forceinline vuint8 loadu(const vboolf8& mask, const void* a) { return _mm256_castps_si256(_mm256_maskload_ps((float*)a,mask)); }
+
+ static __forceinline void store (void* ptr, const vuint8& f) { _mm256_store_ps((float*)ptr,_mm256_castsi256_ps(f)); }
+ static __forceinline void storeu(void* ptr, const vuint8& f) { _mm256_storeu_ps((float*)ptr,_mm256_castsi256_ps(f)); }
+
+#if !defined(__aarch64__)
+ static __forceinline void store (const vboolf8& mask, void* ptr, const vuint8& f) { _mm256_maskstore_ps((float*)ptr,(__m256i)mask,_mm256_castsi256_ps(f)); }
+ static __forceinline void storeu(const vboolf8& mask, void* ptr, const vuint8& f) { _mm256_maskstore_ps((float*)ptr,(__m256i)mask,_mm256_castsi256_ps(f)); }
+#else
+ static __forceinline void store (const vboolf8& mask, void* ptr, const vuint8& f) { _mm256_maskstore_ps((float*)ptr,(__m256i)mask.v,_mm256_castsi256_ps(f)); }
+ static __forceinline void storeu(const vboolf8& mask, void* ptr, const vuint8& f) { _mm256_maskstore_ps((float*)ptr,(__m256i)mask.v,_mm256_castsi256_ps(f)); }
+#endif
+ static __forceinline void store_nt(void* ptr, const vuint8& v) {
+ _mm256_stream_ps((float*)ptr,_mm256_castsi256_ps(v));
+ }
+
+ static __forceinline vuint8 load(const uint8_t* ptr) {
+ vuint4 il = vuint4::load(ptr+0);
+ vuint4 ih = vuint4::load(ptr+4);
+ return vuint8(il,ih);
+ }
+
+ static __forceinline vuint8 loadu(const uint8_t* ptr) {
+ vuint4 il = vuint4::loadu(ptr+0);
+ vuint4 ih = vuint4::loadu(ptr+4);
+ return vuint8(il,ih);
+ }
+
+ static __forceinline vuint8 load(const unsigned short* ptr) {
+ vuint4 il = vuint4::load(ptr+0);
+ vuint4 ih = vuint4::load(ptr+4);
+ return vuint8(il,ih);
+ }
+
+ static __forceinline vuint8 loadu(const unsigned short* ptr) {
+ vuint4 il = vuint4::loadu(ptr+0);
+ vuint4 ih = vuint4::loadu(ptr+4);
+ return vuint8(il,ih);
+ }
+
+ static __forceinline void store(uint8_t* ptr, const vuint8& i) {
+ vuint4 il(i.vl);
+ vuint4 ih(i.vh);
+ vuint4::store(ptr + 0,il);
+ vuint4::store(ptr + 4,ih);
+ }
+
+ static __forceinline void store(unsigned short* ptr, const vuint8& v) {
+ for (size_t i=0;i<8;i++)
+ ptr[i] = (unsigned short)v[i];
+ }
+
+ template<int scale = 4>
+ static __forceinline vuint8 gather(const unsigned int* ptr, const vint8& index) {
+ return vuint8(
+ *(unsigned int*)(((int8_t*)ptr)+scale*index[0]),
+ *(unsigned int*)(((int8_t*)ptr)+scale*index[1]),
+ *(unsigned int*)(((int8_t*)ptr)+scale*index[2]),
+ *(unsigned int*)(((int8_t*)ptr)+scale*index[3]),
+ *(unsigned int*)(((int8_t*)ptr)+scale*index[4]),
+ *(unsigned int*)(((int8_t*)ptr)+scale*index[5]),
+ *(unsigned int*)(((int8_t*)ptr)+scale*index[6]),
+ *(unsigned int*)(((int8_t*)ptr)+scale*index[7]));
+ }
+
+ template<int scale = 4>
+ static __forceinline vuint8 gather(const vboolf8& mask, const unsigned int* ptr, const vint8& index) {
+ vuint8 r = zero;
+ if (likely(mask[0])) r[0] = *(unsigned int*)(((int8_t*)ptr)+scale*index[0]);
+ if (likely(mask[1])) r[1] = *(unsigned int*)(((int8_t*)ptr)+scale*index[1]);
+ if (likely(mask[2])) r[2] = *(unsigned int*)(((int8_t*)ptr)+scale*index[2]);
+ if (likely(mask[3])) r[3] = *(unsigned int*)(((int8_t*)ptr)+scale*index[3]);
+ if (likely(mask[4])) r[4] = *(unsigned int*)(((int8_t*)ptr)+scale*index[4]);
+ if (likely(mask[5])) r[5] = *(unsigned int*)(((int8_t*)ptr)+scale*index[5]);
+ if (likely(mask[6])) r[6] = *(unsigned int*)(((int8_t*)ptr)+scale*index[6]);
+ if (likely(mask[7])) r[7] = *(unsigned int*)(((int8_t*)ptr)+scale*index[7]);
+ return r;
+ }
+
+ template<int scale = 4>
+ static __forceinline void scatter(void* ptr, const vint8& ofs, const vuint8& v)
+ {
+ *(unsigned int*)(((int8_t*)ptr)+scale*ofs[0]) = v[0];
+ *(unsigned int*)(((int8_t*)ptr)+scale*ofs[1]) = v[1];
+ *(unsigned int*)(((int8_t*)ptr)+scale*ofs[2]) = v[2];
+ *(unsigned int*)(((int8_t*)ptr)+scale*ofs[3]) = v[3];
+ *(unsigned int*)(((int8_t*)ptr)+scale*ofs[4]) = v[4];
+ *(unsigned int*)(((int8_t*)ptr)+scale*ofs[5]) = v[5];
+ *(unsigned int*)(((int8_t*)ptr)+scale*ofs[6]) = v[6];
+ *(unsigned int*)(((int8_t*)ptr)+scale*ofs[7]) = v[7];
+ }
+
+ template<int scale = 4>
+ static __forceinline void scatter(const vboolf8& mask, void* ptr, const vint8& ofs, const vuint8& v)
+ {
+ if (likely(mask[0])) *(unsigned int*)(((int8_t*)ptr)+scale*ofs[0]) = v[0];
+ if (likely(mask[1])) *(unsigned int*)(((int8_t*)ptr)+scale*ofs[1]) = v[1];
+ if (likely(mask[2])) *(unsigned int*)(((int8_t*)ptr)+scale*ofs[2]) = v[2];
+ if (likely(mask[3])) *(unsigned int*)(((int8_t*)ptr)+scale*ofs[3]) = v[3];
+ if (likely(mask[4])) *(unsigned int*)(((int8_t*)ptr)+scale*ofs[4]) = v[4];
+ if (likely(mask[5])) *(unsigned int*)(((int8_t*)ptr)+scale*ofs[5]) = v[5];
+ if (likely(mask[6])) *(unsigned int*)(((int8_t*)ptr)+scale*ofs[6]) = v[6];
+ if (likely(mask[7])) *(unsigned int*)(((int8_t*)ptr)+scale*ofs[7]) = v[7];
+ }
+
+
+ static __forceinline vuint8 broadcast64(const long long& a) { return _mm256_set1_epi64x(a); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Array Access
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline const unsigned int& operator [](size_t index) const { assert(index < 8); return i[index]; }
+ __forceinline unsigned int& operator [](size_t index) { assert(index < 8); return i[index]; }
+ };
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Unary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboolf8 asBool(const vuint8& a) { return _mm256_castsi256_ps(a); }
+
+ __forceinline vuint8 operator +(const vuint8& a) { return a; }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Binary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vuint8 operator +(const vuint8& a, const vuint8& b) { return vuint8(_mm_add_epi32(a.vl, b.vl), _mm_add_epi32(a.vh, b.vh)); }
+ __forceinline vuint8 operator +(const vuint8& a, unsigned int b) { return a + vuint8(b); }
+ __forceinline vuint8 operator +(unsigned int a, const vuint8& b) { return vuint8(a) + b; }
+
+ __forceinline vuint8 operator -(const vuint8& a, const vuint8& b) { return vuint8(_mm_sub_epi32(a.vl, b.vl), _mm_sub_epi32(a.vh, b.vh)); }
+ __forceinline vuint8 operator -(const vuint8& a, unsigned int b) { return a - vuint8(b); }
+ __forceinline vuint8 operator -(unsigned int a, const vuint8& b) { return vuint8(a) - b; }
+
+ //__forceinline vuint8 operator *(const vuint8& a, const vuint8& b) { return vuint8(_mm_mullo_epu32(a.vl, b.vl), _mm_mullo_epu32(a.vh, b.vh)); }
+ //__forceinline vuint8 operator *(const vuint8& a, unsigned int b) { return a * vuint8(b); }
+ //__forceinline vuint8 operator *(unsigned int a, const vuint8& b) { return vuint8(a) * b; }
+
+ __forceinline vuint8 operator &(const vuint8& a, const vuint8& b) { return _mm256_castps_si256(_mm256_and_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b))); }
+ __forceinline vuint8 operator &(const vuint8& a, unsigned int b) { return a & vuint8(b); }
+ __forceinline vuint8 operator &(unsigned int a, const vuint8& b) { return vuint8(a) & b; }
+
+ __forceinline vuint8 operator |(const vuint8& a, const vuint8& b) { return _mm256_castps_si256(_mm256_or_ps (_mm256_castsi256_ps(a), _mm256_castsi256_ps(b))); }
+ __forceinline vuint8 operator |(const vuint8& a, unsigned int b) { return a | vuint8(b); }
+ __forceinline vuint8 operator |(unsigned int a, const vuint8& b) { return vuint8(a) | b; }
+
+ __forceinline vuint8 operator ^(const vuint8& a, const vuint8& b) { return _mm256_castps_si256(_mm256_xor_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b))); }
+ __forceinline vuint8 operator ^(const vuint8& a, unsigned int b) { return a ^ vuint8(b); }
+ __forceinline vuint8 operator ^(unsigned int a, const vuint8& b) { return vuint8(a) ^ b; }
+
+ __forceinline vuint8 operator <<(const vuint8& a, unsigned int n) { return vuint8(_mm_slli_epi32(a.vl, n), _mm_slli_epi32(a.vh, n)); }
+ __forceinline vuint8 operator >>(const vuint8& a, unsigned int n) { return vuint8(_mm_srai_epi32(a.vl, n), _mm_srli_epi32(a.vh, n)); }
+
+ __forceinline vuint8 sll (const vuint8& a, unsigned int b) { return vuint8(_mm_slli_epi32(a.vl, b), _mm_slli_epi32(a.vh, b)); }
+ __forceinline vuint8 sra (const vuint8& a, unsigned int b) { return vuint8(_mm_srai_epi32(a.vl, b), _mm_srai_epi32(a.vh, b)); }
+ __forceinline vuint8 srl (const vuint8& a, unsigned int b) { return vuint8(_mm_srli_epi32(a.vl, b), _mm_srli_epi32(a.vh, b)); }
+
+ __forceinline vuint8 min(const vuint8& a, const vuint8& b) { return vuint8(_mm_min_epu32(a.vl, b.vl), _mm_min_epu32(a.vh, b.vh)); }
+ __forceinline vuint8 min(const vuint8& a, unsigned int b) { return min(a,vuint8(b)); }
+ __forceinline vuint8 min(unsigned int a, const vuint8& b) { return min(vuint8(a),b); }
+
+ __forceinline vuint8 max(const vuint8& a, const vuint8& b) { return vuint8(_mm_max_epu32(a.vl, b.vl), _mm_max_epu32(a.vh, b.vh)); }
+ __forceinline vuint8 max(const vuint8& a, unsigned int b) { return max(a,vuint8(b)); }
+ __forceinline vuint8 max(unsigned int a, const vuint8& b) { return max(vuint8(a),b); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Assignment Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vuint8& operator +=(vuint8& a, const vuint8& b) { return a = a + b; }
+ __forceinline vuint8& operator +=(vuint8& a, unsigned int b) { return a = a + b; }
+
+ __forceinline vuint8& operator -=(vuint8& a, const vuint8& b) { return a = a - b; }
+ __forceinline vuint8& operator -=(vuint8& a, unsigned int b) { return a = a - b; }
+
+ //__forceinline vuint8& operator *=(vuint8& a, const vuint8& b) { return a = a * b; }
+ //__forceinline vuint8& operator *=(vuint8& a, unsigned int b) { return a = a * b; }
+
+ __forceinline vuint8& operator &=(vuint8& a, const vuint8& b) { return a = a & b; }
+ __forceinline vuint8& operator &=(vuint8& a, unsigned int b) { return a = a & b; }
+
+ __forceinline vuint8& operator |=(vuint8& a, const vuint8& b) { return a = a | b; }
+ __forceinline vuint8& operator |=(vuint8& a, unsigned int b) { return a = a | b; }
+
+ __forceinline vuint8& operator <<=(vuint8& a, unsigned int b) { return a = a << b; }
+ __forceinline vuint8& operator >>=(vuint8& a, unsigned int b) { return a = a >> b; }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Comparison Operators + Select
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vboolf8 operator ==(const vuint8& a, const vuint8& b) { return vboolf8(_mm_castsi128_ps(_mm_cmpeq_epi32 (a.vl, b.vl)),
+ _mm_castsi128_ps(_mm_cmpeq_epi32 (a.vh, b.vh))); }
+ __forceinline vboolf8 operator ==(const vuint8& a, unsigned int b) { return a == vuint8(b); }
+ __forceinline vboolf8 operator ==(unsigned int a, const vuint8& b) { return vuint8(a) == b; }
+
+ __forceinline vboolf8 operator !=(const vuint8& a, const vuint8& b) { return !(a == b); }
+ __forceinline vboolf8 operator !=(const vuint8& a, unsigned int b) { return a != vuint8(b); }
+ __forceinline vboolf8 operator !=(unsigned int a, const vuint8& b) { return vuint8(a) != b; }
+
+ //__forceinline vboolf8 operator < (const vuint8& a, const vuint8& b) { return vboolf8(_mm_castsi128_ps(_mm_cmplt_epu32 (a.vl, b.vl)),
+ // _mm_castsi128_ps(_mm_cmplt_epu32 (a.vh, b.vh))); }
+ //__forceinline vboolf8 operator < (const vuint8& a, unsigned int b) { return a < vuint8(b); }
+ //__forceinline vboolf8 operator < (unsigned int a, const vuint8& b) { return vuint8(a) < b; }
+
+ //__forceinline vboolf8 operator >=(const vuint8& a, const vuint8& b) { return !(a < b); }
+ //__forceinline vboolf8 operator >=(const vuint8& a, unsigned int b) { return a >= vuint8(b); }
+ //__forceinline vboolf8 operator >=(unsigned int a, const vuint8& b) { return vuint8(a) >= b; }
+
+ //__forceinline vboolf8 operator > (const vuint8& a, const vuint8& b) { return vboolf8(_mm_castsi128_ps(_mm_cmpgt_epu32 (a.vl, b.vl)),
+ // _mm_castsi128_ps(_mm_cmpgt_epu32 (a.vh, b.vh))); }
+ //__forceinline vboolf8 operator > (const vuint8& a, unsigned int b) { return a > vuint8(b); }
+ //__forceinline vboolf8 operator > (unsigned int a, const vuint8& b) { return vuint8(a) > b; }
+
+ //__forceinline vboolf8 operator <=(const vuint8& a, const vuint8& b) { return !(a > b); }
+ //__forceinline vboolf8 operator <=(const vuint8& a, unsigned int b) { return a <= vuint8(b); }
+ //__forceinline vboolf8 operator <=(unsigned int a, const vuint8& b) { return vuint8(a) <= b; }
+
+ __forceinline vboolf8 eq(const vuint8& a, const vuint8& b) { return a == b; }
+ __forceinline vboolf8 ne(const vuint8& a, const vuint8& b) { return a != b; }
+
+ __forceinline vboolf8 eq(const vboolf8& mask, const vuint8& a, const vuint8& b) { return mask & (a == b); }
+ __forceinline vboolf8 ne(const vboolf8& mask, const vuint8& a, const vuint8& b) { return mask & (a != b); }
+
+ __forceinline vuint8 select(const vboolf8& m, const vuint8& t, const vuint8& f) {
+ return _mm256_castps_si256(_mm256_blendv_ps(_mm256_castsi256_ps(f), _mm256_castsi256_ps(t), m));
+ }
+
+ __forceinline vuint8 notand(const vboolf8& m, const vuint8& f) {
+ return _mm256_castps_si256(_mm256_andnot_ps(m, _mm256_castsi256_ps(f)));
+ }
+
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Movement/Shifting/Shuffling Functions
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vuint8 unpacklo(const vuint8& a, const vuint8& b) { return _mm256_castps_si256(_mm256_unpacklo_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b))); }
+ __forceinline vuint8 unpackhi(const vuint8& a, const vuint8& b) { return _mm256_castps_si256(_mm256_unpackhi_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b))); }
+
+ template<int i>
+ __forceinline vuint8 shuffle(const vuint8& v) {
+ return _mm256_castps_si256(_mm256_permute_ps(_mm256_castsi256_ps(v), _MM_SHUFFLE(i, i, i, i)));
+ }
+
+ template<int i0, int i1>
+ __forceinline vuint8 shuffle4(const vuint8& v) {
+ return _mm256_permute2f128_si256(v, v, (i1 << 4) | (i0 << 0));
+ }
+
+ template<int i0, int i1>
+ __forceinline vuint8 shuffle4(const vuint8& a, const vuint8& b) {
+ return _mm256_permute2f128_si256(a, b, (i1 << 4) | (i0 << 0));
+ }
+
+ template<int i0, int i1, int i2, int i3>
+ __forceinline vuint8 shuffle(const vuint8& v) {
+ return _mm256_castps_si256(_mm256_permute_ps(_mm256_castsi256_ps(v), _MM_SHUFFLE(i3, i2, i1, i0)));
+ }
+
+ template<int i0, int i1, int i2, int i3>
+ __forceinline vuint8 shuffle(const vuint8& a, const vuint8& b) {
+ return _mm256_castps_si256(_mm256_shuffle_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b), _MM_SHUFFLE(i3, i2, i1, i0)));
+ }
+
+ template<> __forceinline vuint8 shuffle<0, 0, 2, 2>(const vuint8& v) { return _mm256_castps_si256(_mm256_moveldup_ps(_mm256_castsi256_ps(v))); }
+ template<> __forceinline vuint8 shuffle<1, 1, 3, 3>(const vuint8& v) { return _mm256_castps_si256(_mm256_movehdup_ps(_mm256_castsi256_ps(v))); }
+ template<> __forceinline vuint8 shuffle<0, 1, 0, 1>(const vuint8& v) { return _mm256_castps_si256(_mm256_castpd_ps(_mm256_movedup_pd(_mm256_castps_pd(_mm256_castsi256_ps(v))))); }
+
+ __forceinline vuint8 broadcast(const unsigned int* ptr) { return _mm256_castps_si256(_mm256_broadcast_ss((const float*)ptr)); }
+ template<int i> __forceinline vuint8 insert4(const vuint8& a, const vuint4& b) { return _mm256_insertf128_si256(a, b, i); }
+ template<int i> __forceinline vuint4 extract4(const vuint8& a) { return _mm256_extractf128_si256(a, i); }
+ template<> __forceinline vuint4 extract4<0>(const vuint8& a) { return _mm256_castsi256_si128(a); }
+
+ __forceinline int toScalar(const vuint8& v) { return _mm_cvtsi128_si32(_mm256_castsi256_si128(v)); }
+
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Reductions
+ ////////////////////////////////////////////////////////////////////////////////
+
+ //__forceinline vuint8 vreduce_min2(const vuint8& v) { return min(v,shuffle<1,0,3,2>(v)); }
+ //__forceinline vuint8 vreduce_min4(const vuint8& v) { vuint8 v1 = vreduce_min2(v); return min(v1,shuffle<2,3,0,1>(v1)); }
+ //__forceinline vuint8 vreduce_min (const vuint8& v) { vuint8 v1 = vreduce_min4(v); return min(v1,shuffle4<1,0>(v1)); }
+
+ //__forceinline vuint8 vreduce_max2(const vuint8& v) { return max(v,shuffle<1,0,3,2>(v)); }
+ //__forceinline vuint8 vreduce_max4(const vuint8& v) { vuint8 v1 = vreduce_max2(v); return max(v1,shuffle<2,3,0,1>(v1)); }
+ //__forceinline vuint8 vreduce_max (const vuint8& v) { vuint8 v1 = vreduce_max4(v); return max(v1,shuffle4<1,0>(v1)); }
+
+ __forceinline vuint8 vreduce_add2(const vuint8& v) { return v + shuffle<1,0,3,2>(v); }
+ __forceinline vuint8 vreduce_add4(const vuint8& v) { vuint8 v1 = vreduce_add2(v); return v1 + shuffle<2,3,0,1>(v1); }
+ __forceinline vuint8 vreduce_add (const vuint8& v) { vuint8 v1 = vreduce_add4(v); return v1 + shuffle4<1,0>(v1); }
+
+ //__forceinline int reduce_min(const vuint8& v) { return toScalar(vreduce_min(v)); }
+ //__forceinline int reduce_max(const vuint8& v) { return toScalar(vreduce_max(v)); }
+ __forceinline int reduce_add(const vuint8& v) { return toScalar(vreduce_add(v)); }
+
+ //__forceinline size_t select_min(const vuint8& v) { return bsf(movemask(v == vreduce_min(v))); }
+ //__forceinline size_t select_max(const vuint8& v) { return bsf(movemask(v == vreduce_max(v))); }
+
+ //__forceinline size_t select_min(const vboolf8& valid, const vuint8& v) { const vuint8 a = select(valid,v,vuint8(pos_inf)); return bsf(movemask(valid & (a == vreduce_min(a)))); }
+ //__forceinline size_t select_max(const vboolf8& valid, const vuint8& v) { const vuint8 a = select(valid,v,vuint8(neg_inf)); return bsf(movemask(valid & (a == vreduce_max(a)))); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Output Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline embree_ostream operator <<(embree_ostream cout, const vuint8& a) {
+ return cout << "<" << a[0] << ", " << a[1] << ", " << a[2] << ", " << a[3] << ", " << a[4] << ", " << a[5] << ", " << a[6] << ", " << a[7] << ">";
+ }
+}
diff --git a/thirdparty/embree-aarch64/common/simd/vuint8_avx2.h b/thirdparty/embree-aarch64/common/simd/vuint8_avx2.h
new file mode 100644
index 0000000000..b2a965448d
--- /dev/null
+++ b/thirdparty/embree-aarch64/common/simd/vuint8_avx2.h
@@ -0,0 +1,439 @@
+// Copyright 2009-2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+namespace embree
+{
+ /* 8-wide AVX integer type */
+ template<>
+ struct vuint<8>
+ {
+ ALIGNED_STRUCT_(32);
+
+ typedef vboolf8 Bool;
+ typedef vuint8 Int;
+ typedef vfloat8 Float;
+
+ enum { size = 8 }; // number of SIMD elements
+ union { // data
+ __m256i v;
+ unsigned int i[8];
+ };
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constructors, Assignment & Cast Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vuint() {}
+ __forceinline vuint(const vuint8& a) { v = a.v; }
+ __forceinline vuint8& operator =(const vuint8& a) { v = a.v; return *this; }
+
+ __forceinline vuint(__m256i a) : v(a) {}
+ __forceinline operator const __m256i&() const { return v; }
+ __forceinline operator __m256i&() { return v; }
+
+ __forceinline explicit vuint(const vuint4& a) : v(_mm256_insertf128_si256(_mm256_castsi128_si256(a),a,1)) {}
+ __forceinline vuint(const vuint4& a, const vuint4& b) : v(_mm256_insertf128_si256(_mm256_castsi128_si256(a),b,1)) {}
+ __forceinline vuint(const __m128i& a, const __m128i& b) : v(_mm256_insertf128_si256(_mm256_castsi128_si256(a),b,1)) {}
+
+ __forceinline explicit vuint(const unsigned int* a) : v(_mm256_castps_si256(_mm256_loadu_ps((const float*)a))) {}
+ __forceinline vuint(unsigned int a) : v(_mm256_set1_epi32(a)) {}
+ __forceinline vuint(unsigned int a, unsigned int b) : v(_mm256_set_epi32(b, a, b, a, b, a, b, a)) {}
+ __forceinline vuint(unsigned int a, unsigned int b, unsigned int c, unsigned int d) : v(_mm256_set_epi32(d, c, b, a, d, c, b, a)) {}
+ __forceinline vuint(unsigned int a, unsigned int b, unsigned int c, unsigned int d, unsigned int e, unsigned int f, unsigned int g, unsigned int h) : v(_mm256_set_epi32(h, g, f, e, d, c, b, a)) {}
+
+ __forceinline explicit vuint(__m256 a) : v(_mm256_cvtps_epi32(a)) {}
+
+#if defined(__AVX512VL__)
+ __forceinline explicit vuint(const vboolf8& a) : v(_mm256_movm_epi32(a)) {}
+#else
+ __forceinline explicit vuint(const vboolf8& a) : v(_mm256_castps_si256((__m256)a)) {}
+#endif
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Constants
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vuint(ZeroTy) : v(_mm256_setzero_si256()) {}
+ __forceinline vuint(OneTy) : v(_mm256_set1_epi32(1)) {}
+ __forceinline vuint(PosInfTy) : v(_mm256_set1_epi32(pos_inf)) {}
+ __forceinline vuint(NegInfTy) : v(_mm256_set1_epi32(neg_inf)) {}
+ __forceinline vuint(StepTy) : v(_mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0)) {}
+ __forceinline vuint(UndefinedTy) : v(_mm256_undefined_si256()) {}
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Loads and Stores
+ ////////////////////////////////////////////////////////////////////////////////
+
+ static __forceinline vuint8 load(const uint8_t* ptr) { return _mm256_cvtepu8_epi32(_mm_loadl_epi64((__m128i*)ptr)); }
+ static __forceinline vuint8 loadu(const uint8_t* ptr) { return _mm256_cvtepu8_epi32(_mm_loadl_epi64((__m128i*)ptr)); }
+ static __forceinline vuint8 load(const unsigned short* ptr) { return _mm256_cvtepu16_epi32(_mm_load_si128((__m128i*)ptr)); }
+ static __forceinline vuint8 loadu(const unsigned short* ptr) { return _mm256_cvtepu16_epi32(_mm_loadu_si128((__m128i*)ptr)); }
+
+ static __forceinline vuint8 load(const void* ptr) { return _mm256_load_si256((__m256i*)ptr); }
+ static __forceinline vuint8 loadu(const void* ptr) { return _mm256_loadu_si256((__m256i*)ptr); }
+
+ static __forceinline void store (void* ptr, const vuint8& v) { _mm256_store_si256((__m256i*)ptr,v); }
+ static __forceinline void storeu(void* ptr, const vuint8& v) { _mm256_storeu_ps((float*)ptr,_mm256_castsi256_ps(v)); }
+
+#if defined(__AVX512VL__)
+
+ static __forceinline vuint8 compact(const vboolf8& mask, vuint8 &v) {
+ return _mm256_mask_compress_epi32(v, mask, v);
+ }
+ static __forceinline vuint8 compact(const vboolf8& mask, vuint8 &a, const vuint8& b) {
+ return _mm256_mask_compress_epi32(a, mask, b);
+ }
+
+ static __forceinline vuint8 load (const vboolf8& mask, const void* ptr) { return _mm256_mask_load_epi32 (_mm256_setzero_si256(),mask,ptr); }
+ static __forceinline vuint8 loadu(const vboolf8& mask, const void* ptr) { return _mm256_mask_loadu_epi32(_mm256_setzero_si256(),mask,ptr); }
+
+ static __forceinline void store (const vboolf8& mask, void* ptr, const vuint8& v) { _mm256_mask_store_epi32 (ptr,mask,v); }
+ static __forceinline void storeu(const vboolf8& mask, void* ptr, const vuint8& v) { _mm256_mask_storeu_epi32(ptr,mask,v); }
+#else
+ static __forceinline vuint8 load (const vboolf8& mask, const void* ptr) { return _mm256_castps_si256(_mm256_maskload_ps((float*)ptr,mask)); }
+ static __forceinline vuint8 loadu(const vboolf8& mask, const void* ptr) { return _mm256_castps_si256(_mm256_maskload_ps((float*)ptr,mask)); }
+
+ static __forceinline void store (const vboolf8& mask, void* ptr, const vuint8& v) { _mm256_maskstore_epi32((int*)ptr,mask,v); }
+ static __forceinline void storeu(const vboolf8& mask, void* ptr, const vuint8& v) { _mm256_maskstore_epi32((int*)ptr,mask,v); }
+#endif
+
+ static __forceinline vuint8 load_nt(void* ptr) {
+ return _mm256_stream_load_si256((__m256i*)ptr);
+ }
+
+ static __forceinline void store_nt(void* ptr, const vuint8& v) {
+ _mm256_stream_ps((float*)ptr,_mm256_castsi256_ps(v));
+ }
+
+ static __forceinline void store(uint8_t* ptr, const vuint8& i)
+ {
+ for (size_t j=0; j<8; j++)
+ ptr[j] = i[j];
+ }
+
+ static __forceinline void store(unsigned short* ptr, const vuint8& v) {
+ for (size_t i=0;i<8;i++)
+ ptr[i] = (unsigned short)v[i];
+ }
+
+ template<int scale = 4>
+ static __forceinline vuint8 gather(const unsigned int *const ptr, const vint8& index) {
+ return _mm256_i32gather_epi32((const int*) ptr, index, scale);
+ }
+
+ template<int scale = 4>
+ static __forceinline vuint8 gather(const vboolf8& mask, const unsigned int *const ptr, const vint8& index) {
+ vuint8 r = zero;
+#if defined(__AVX512VL__)
+ return _mm256_mmask_i32gather_epi32(r, mask, index, (const int*) ptr, scale);
+#else
+ return _mm256_mask_i32gather_epi32(r, (const int*) ptr, index, mask, scale);
+#endif
+ }
+
+ template<int scale = 4>
+ static __forceinline void scatter(void* ptr, const vint8& ofs, const vuint8& v)
+ {
+#if defined(__AVX512VL__)
+ _mm256_i32scatter_epi32((int*)ptr, ofs, v, scale);
+#else
+ *(unsigned int*)(((int8_t*)ptr) + scale * ofs[0]) = v[0];
+ *(unsigned int*)(((int8_t*)ptr) + scale * ofs[1]) = v[1];
+ *(unsigned int*)(((int8_t*)ptr) + scale * ofs[2]) = v[2];
+ *(unsigned int*)(((int8_t*)ptr) + scale * ofs[3]) = v[3];
+ *(unsigned int*)(((int8_t*)ptr) + scale * ofs[4]) = v[4];
+ *(unsigned int*)(((int8_t*)ptr) + scale * ofs[5]) = v[5];
+ *(unsigned int*)(((int8_t*)ptr) + scale * ofs[6]) = v[6];
+ *(unsigned int*)(((int8_t*)ptr) + scale * ofs[7]) = v[7];
+#endif
+ }
+
+ template<int scale = 4>
+ static __forceinline void scatter(const vboolf8& mask, void* ptr, const vint8& ofs, const vuint8& v)
+ {
+#if defined(__AVX512VL__)
+ _mm256_mask_i32scatter_epi32((int*)ptr, mask, ofs, v, scale);
+#else
+ if (likely(mask[0])) *(unsigned int*)(((int8_t*)ptr)+scale*ofs[0]) = v[0];
+ if (likely(mask[1])) *(unsigned int*)(((int8_t*)ptr)+scale*ofs[1]) = v[1];
+ if (likely(mask[2])) *(unsigned int*)(((int8_t*)ptr)+scale*ofs[2]) = v[2];
+ if (likely(mask[3])) *(unsigned int*)(((int8_t*)ptr)+scale*ofs[3]) = v[3];
+ if (likely(mask[4])) *(unsigned int*)(((int8_t*)ptr)+scale*ofs[4]) = v[4];
+ if (likely(mask[5])) *(unsigned int*)(((int8_t*)ptr)+scale*ofs[5]) = v[5];
+ if (likely(mask[6])) *(unsigned int*)(((int8_t*)ptr)+scale*ofs[6]) = v[6];
+ if (likely(mask[7])) *(unsigned int*)(((int8_t*)ptr)+scale*ofs[7]) = v[7];
+#endif
+ }
+
+ static __forceinline vuint8 broadcast64(const long long &a) { return _mm256_set1_epi64x(a); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Array Access
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline const unsigned int& operator [](size_t index) const { assert(index < 8); return i[index]; }
+ __forceinline unsigned int& operator [](size_t index) { assert(index < 8); return i[index]; }
+ };
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Unary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+#if defined(__AVX512VL__)
+ __forceinline vboolf8 asBool(const vuint8& a) { return _mm256_movepi32_mask(a); }
+#else
+ __forceinline vboolf8 asBool(const vuint8& a) { return _mm256_castsi256_ps(a); }
+#endif
+
+ __forceinline vuint8 operator +(const vuint8& a) { return a; }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Binary Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vuint8 operator +(const vuint8& a, const vuint8& b) { return _mm256_add_epi32(a, b); }
+ __forceinline vuint8 operator +(const vuint8& a, unsigned int b) { return a + vuint8(b); }
+ __forceinline vuint8 operator +(unsigned int a, const vuint8& b) { return vuint8(a) + b; }
+
+ __forceinline vuint8 operator -(const vuint8& a, const vuint8& b) { return _mm256_sub_epi32(a, b); }
+ __forceinline vuint8 operator -(const vuint8& a, unsigned int b) { return a - vuint8(b); }
+ __forceinline vuint8 operator -(unsigned int a, const vuint8& b) { return vuint8(a) - b; }
+
+ //__forceinline vuint8 operator *(const vuint8& a, const vuint8& b) { return _mm256_mullo_epu32(a, b); }
+ //__forceinline vuint8 operator *(const vuint8& a, unsigned int b) { return a * vuint8(b); }
+ //__forceinline vuint8 operator *(unsigned int a, const vuint8& b) { return vuint8(a) * b; }
+
+ __forceinline vuint8 operator &(const vuint8& a, const vuint8& b) { return _mm256_and_si256(a, b); }
+ __forceinline vuint8 operator &(const vuint8& a, unsigned int b) { return a & vuint8(b); }
+ __forceinline vuint8 operator &(unsigned int a, const vuint8& b) { return vuint8(a) & b; }
+
+ __forceinline vuint8 operator |(const vuint8& a, const vuint8& b) { return _mm256_or_si256(a, b); }
+ __forceinline vuint8 operator |(const vuint8& a, unsigned int b) { return a | vuint8(b); }
+ __forceinline vuint8 operator |(unsigned int a, const vuint8& b) { return vuint8(a) | b; }
+
+ __forceinline vuint8 operator ^(const vuint8& a, const vuint8& b) { return _mm256_xor_si256(a, b); }
+ __forceinline vuint8 operator ^(const vuint8& a, unsigned int b) { return a ^ vuint8(b); }
+ __forceinline vuint8 operator ^(unsigned int a, const vuint8& b) { return vuint8(a) ^ b; }
+
+ __forceinline vuint8 operator <<(const vuint8& a, unsigned int n) { return _mm256_slli_epi32(a, n); }
+ __forceinline vuint8 operator >>(const vuint8& a, unsigned int n) { return _mm256_srli_epi32(a, n); }
+
+ __forceinline vuint8 operator <<(const vuint8& a, const vuint8& n) { return _mm256_sllv_epi32(a, n); }
+ __forceinline vuint8 operator >>(const vuint8& a, const vuint8& n) { return _mm256_srlv_epi32(a, n); }
+
+ __forceinline vuint8 sll(const vuint8& a, unsigned int b) { return _mm256_slli_epi32(a, b); }
+ __forceinline vuint8 sra(const vuint8& a, unsigned int b) { return _mm256_srai_epi32(a, b); }
+ __forceinline vuint8 srl(const vuint8& a, unsigned int b) { return _mm256_srli_epi32(a, b); }
+
+ __forceinline vuint8 sll(const vuint8& a, const vuint8& b) { return _mm256_sllv_epi32(a, b); }
+ __forceinline vuint8 sra(const vuint8& a, const vuint8& b) { return _mm256_srav_epi32(a, b); }
+ __forceinline vuint8 srl(const vuint8& a, const vuint8& b) { return _mm256_srlv_epi32(a, b); }
+
+ __forceinline vuint8 min(const vuint8& a, const vuint8& b) { return _mm256_min_epu32(a, b); }
+ __forceinline vuint8 min(const vuint8& a, unsigned int b) { return min(a,vuint8(b)); }
+ __forceinline vuint8 min(unsigned int a, const vuint8& b) { return min(vuint8(a),b); }
+
+ __forceinline vuint8 max(const vuint8& a, const vuint8& b) { return _mm256_max_epu32(a, b); }
+ __forceinline vuint8 max(const vuint8& a, unsigned int b) { return max(a,vuint8(b)); }
+ __forceinline vuint8 max(unsigned int a, const vuint8& b) { return max(vuint8(a),b); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Assignment Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vuint8& operator +=(vuint8& a, const vuint8& b) { return a = a + b; }
+ __forceinline vuint8& operator +=(vuint8& a, unsigned int b) { return a = a + b; }
+
+ __forceinline vuint8& operator -=(vuint8& a, const vuint8& b) { return a = a - b; }
+ __forceinline vuint8& operator -=(vuint8& a, unsigned int b) { return a = a - b; }
+
+ //__forceinline vuint8& operator *=(vuint8& a, const vuint8& b) { return a = a * b; }
+ //__forceinline vuint8& operator *=(vuint8& a, unsigned int b) { return a = a * b; }
+
+ __forceinline vuint8& operator &=(vuint8& a, const vuint8& b) { return a = a & b; }
+ __forceinline vuint8& operator &=(vuint8& a, unsigned int b) { return a = a & b; }
+
+ __forceinline vuint8& operator |=(vuint8& a, const vuint8& b) { return a = a | b; }
+ __forceinline vuint8& operator |=(vuint8& a, unsigned int b) { return a = a | b; }
+
+ __forceinline vuint8& operator <<=(vuint8& a, const unsigned int b) { return a = a << b; }
+ __forceinline vuint8& operator >>=(vuint8& a, const unsigned int b) { return a = a >> b; }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Comparison Operators + Select
+ ////////////////////////////////////////////////////////////////////////////////
+
+#if defined(__AVX512VL__)
+ __forceinline vboolf8 operator ==(const vuint8& a, const vuint8& b) { return _mm256_cmp_epu32_mask(a,b,_MM_CMPINT_EQ); }
+ __forceinline vboolf8 operator !=(const vuint8& a, const vuint8& b) { return _mm256_cmp_epu32_mask(a,b,_MM_CMPINT_NE); }
+ __forceinline vboolf8 operator < (const vuint8& a, const vuint8& b) { return _mm256_cmp_epu32_mask(a,b,_MM_CMPINT_LT); }
+ __forceinline vboolf8 operator >=(const vuint8& a, const vuint8& b) { return _mm256_cmp_epu32_mask(a,b,_MM_CMPINT_GE); }
+ __forceinline vboolf8 operator > (const vuint8& a, const vuint8& b) { return _mm256_cmp_epu32_mask(a,b,_MM_CMPINT_GT); }
+ __forceinline vboolf8 operator <=(const vuint8& a, const vuint8& b) { return _mm256_cmp_epu32_mask(a,b,_MM_CMPINT_LE); }
+
+ __forceinline vuint8 select(const vboolf8& m, const vuint8& t, const vuint8& f) {
+ return _mm256_mask_blend_epi32(m, (__m256i)f, (__m256i)t);
+ }
+#else
+ __forceinline vboolf8 operator ==(const vuint8& a, const vuint8& b) { return _mm256_castsi256_ps(_mm256_cmpeq_epi32(a, b)); }
+ __forceinline vboolf8 operator !=(const vuint8& a, const vuint8& b) { return !(a == b); }
+ //__forceinline vboolf8 operator < (const vuint8& a, const vuint8& b) { return _mm256_castsi256_ps(_mm256_cmpgt_epu32(b, a)); }
+ //__forceinline vboolf8 operator >=(const vuint8& a, const vuint8& b) { return !(a < b); }
+ //__forceinline vboolf8 operator > (const vuint8& a, const vuint8& b) { return _mm256_castsi256_ps(_mm256_cmpgt_epu32(a, b)); }
+ //__forceinline vboolf8 operator <=(const vuint8& a, const vuint8& b) { return !(a > b); }
+
+ __forceinline vuint8 select(const vboolf8& m, const vuint8& t, const vuint8& f) {
+ return _mm256_castps_si256(_mm256_blendv_ps(_mm256_castsi256_ps(f), _mm256_castsi256_ps(t), m));
+ }
+#endif
+
+ template<int mask>
+ __forceinline vuint8 select(const vuint8& t, const vuint8& f) {
+ return _mm256_blend_epi32(f, t, mask);
+ }
+
+ __forceinline vboolf8 operator ==(const vuint8& a, unsigned int b) { return a == vuint8(b); }
+ __forceinline vboolf8 operator ==(unsigned int a, const vuint8& b) { return vuint8(a) == b; }
+
+ __forceinline vboolf8 operator !=(const vuint8& a, unsigned int b) { return a != vuint8(b); }
+ __forceinline vboolf8 operator !=(unsigned int a, const vuint8& b) { return vuint8(a) != b; }
+
+ //__forceinline vboolf8 operator < (const vuint8& a, unsigned int b) { return a < vuint8(b); }
+ //__forceinline vboolf8 operator < (unsigned int a, const vuint8& b) { return vuint8(a) < b; }
+
+ //__forceinline vboolf8 operator >=(const vuint8& a, unsigned int b) { return a >= vuint8(b); }
+ //__forceinline vboolf8 operator >=(unsigned int a, const vuint8& b) { return vuint8(a) >= b; }
+
+ //__forceinline vboolf8 operator > (const vuint8& a, unsigned int b) { return a > vuint8(b); }
+ //__forceinline vboolf8 operator > (unsigned int a, const vuint8& b) { return vuint8(a) > b; }
+
+ //__forceinline vboolf8 operator <=(const vuint8& a, unsigned int b) { return a <= vuint8(b); }
+ //__forceinline vboolf8 operator <=(unsigned int a, const vuint8& b) { return vuint8(a) <= b; }
+
+ __forceinline vboolf8 eq(const vuint8& a, const vuint8& b) { return a == b; }
+ __forceinline vboolf8 ne(const vuint8& a, const vuint8& b) { return a != b; }
+ //__forceinline vboolf8 lt(const vuint8& a, const vuint8& b) { return a < b; }
+ //__forceinline vboolf8 ge(const vuint8& a, const vuint8& b) { return a >= b; }
+ //__forceinline vboolf8 gt(const vuint8& a, const vuint8& b) { return a > b; }
+ //__forceinline vboolf8 le(const vuint8& a, const vuint8& b) { return a <= b; }
+
+#if defined(__AVX512VL__)
+ __forceinline vboolf8 eq(const vboolf8& mask, const vuint8& a, const vuint8& b) { return _mm256_mask_cmp_epu32_mask(mask, a, b, _MM_CMPINT_EQ); }
+ __forceinline vboolf8 ne(const vboolf8& mask, const vuint8& a, const vuint8& b) { return _mm256_mask_cmp_epu32_mask(mask, a, b, _MM_CMPINT_NE); }
+ __forceinline vboolf8 lt(const vboolf8& mask, const vuint8& a, const vuint8& b) { return _mm256_mask_cmp_epu32_mask(mask, a, b, _MM_CMPINT_LT); }
+ __forceinline vboolf8 ge(const vboolf8& mask, const vuint8& a, const vuint8& b) { return _mm256_mask_cmp_epu32_mask(mask, a, b, _MM_CMPINT_GE); }
+ __forceinline vboolf8 gt(const vboolf8& mask, const vuint8& a, const vuint8& b) { return _mm256_mask_cmp_epu32_mask(mask, a, b, _MM_CMPINT_GT); }
+ __forceinline vboolf8 le(const vboolf8& mask, const vuint8& a, const vuint8& b) { return _mm256_mask_cmp_epu32_mask(mask, a, b, _MM_CMPINT_LE); }
+#else
+ __forceinline vboolf8 eq(const vboolf8& mask, const vuint8& a, const vuint8& b) { return mask & (a == b); }
+ __forceinline vboolf8 ne(const vboolf8& mask, const vuint8& a, const vuint8& b) { return mask & (a != b); }
+ //__forceinline vboolf8 lt(const vboolf8& mask, const vuint8& a, const vuint8& b) { return mask & (a < b); }
+ //__forceinline vboolf8 ge(const vboolf8& mask, const vuint8& a, const vuint8& b) { return mask & (a >= b); }
+ //__forceinline vboolf8 gt(const vboolf8& mask, const vuint8& a, const vuint8& b) { return mask & (a > b); }
+ //__forceinline vboolf8 le(const vboolf8& mask, const vuint8& a, const vuint8& b) { return mask & (a <= b); }
+#endif
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Movement/Shifting/Shuffling Functions
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline vuint8 unpacklo(const vuint8& a, const vuint8& b) { return _mm256_unpacklo_epi32(a, b); }
+ __forceinline vuint8 unpackhi(const vuint8& a, const vuint8& b) { return _mm256_unpackhi_epi32(a, b); }
+
+ template<int i>
+ __forceinline vuint8 shuffle(const vuint8& v) {
+ return _mm256_castps_si256(_mm256_permute_ps(_mm256_castsi256_ps(v), _MM_SHUFFLE(i, i, i, i)));
+ }
+
+ template<int i0, int i1>
+ __forceinline vuint8 shuffle4(const vuint8& v) {
+ return _mm256_permute2f128_si256(v, v, (i1 << 4) | (i0 << 0));
+ }
+
+ template<int i0, int i1>
+ __forceinline vuint8 shuffle4(const vuint8& a, const vuint8& b) {
+ return _mm256_permute2f128_si256(a, b, (i1 << 4) | (i0 << 0));
+ }
+
+ template<int i0, int i1, int i2, int i3>
+ __forceinline vuint8 shuffle(const vuint8& v) {
+ return _mm256_castps_si256(_mm256_permute_ps(_mm256_castsi256_ps(v), _MM_SHUFFLE(i3, i2, i1, i0)));
+ }
+
+ template<int i0, int i1, int i2, int i3>
+ __forceinline vuint8 shuffle(const vuint8& a, const vuint8& b) {
+ return _mm256_castps_si256(_mm256_shuffle_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b), _MM_SHUFFLE(i3, i2, i1, i0)));
+ }
+
+ template<> __forceinline vuint8 shuffle<0, 0, 2, 2>(const vuint8& v) { return _mm256_castps_si256(_mm256_moveldup_ps(_mm256_castsi256_ps(v))); }
+ template<> __forceinline vuint8 shuffle<1, 1, 3, 3>(const vuint8& v) { return _mm256_castps_si256(_mm256_movehdup_ps(_mm256_castsi256_ps(v))); }
+ template<> __forceinline vuint8 shuffle<0, 1, 0, 1>(const vuint8& v) { return _mm256_castps_si256(_mm256_castpd_ps(_mm256_movedup_pd(_mm256_castps_pd(_mm256_castsi256_ps(v))))); }
+
+ __forceinline vuint8 broadcast(const unsigned int* ptr) { return _mm256_castps_si256(_mm256_broadcast_ss((const float*)ptr)); }
+
+ template<int i> __forceinline vuint8 insert4(const vuint8& a, const vuint4& b) { return _mm256_insertf128_si256(a, b, i); }
+ template<int i> __forceinline vuint4 extract4(const vuint8& a) { return _mm256_extractf128_si256(a, i); }
+ template<> __forceinline vuint4 extract4<0>(const vuint8& a) { return _mm256_castsi256_si128(a); }
+
+ __forceinline int toScalar(const vuint8& v) { return _mm_cvtsi128_si32(_mm256_castsi256_si128(v)); }
+
+#if !defined(__aarch64__)
+
+ __forceinline vuint8 permute(const vuint8& v, const __m256i& index) {
+ return _mm256_permutevar8x32_epi32(v, index);
+ }
+
+ __forceinline vuint8 shuffle(const vuint8& v, const __m256i& index) {
+ return _mm256_castps_si256(_mm256_permutevar_ps(_mm256_castsi256_ps(v), index));
+ }
+
+ template<int i>
+ __forceinline vuint8 align_shift_right(const vuint8& a, const vuint8& b) {
+#if defined(__AVX512VL__)
+ return _mm256_alignr_epi32(a, b, i);
+#else
+ return _mm256_alignr_epi8(a, b, 4*i);
+#endif
+ }
+
+#endif
+
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Reductions
+ ////////////////////////////////////////////////////////////////////////////////
+
+ //__forceinline vuint8 vreduce_min2(const vuint8& v) { return min(v,shuffle<1,0,3,2>(v)); }
+ //__forceinline vuint8 vreduce_min4(const vuint8& v) { vuint8 v1 = vreduce_min2(v); return min(v1,shuffle<2,3,0,1>(v1)); }
+ //__forceinline vuint8 vreduce_min (const vuint8& v) { vuint8 v1 = vreduce_min4(v); return min(v1,shuffle4<1,0>(v1)); }
+
+ //__forceinline vuint8 vreduce_max2(const vuint8& v) { return max(v,shuffle<1,0,3,2>(v)); }
+ //__forceinline vuint8 vreduce_max4(const vuint8& v) { vuint8 v1 = vreduce_max2(v); return max(v1,shuffle<2,3,0,1>(v1)); }
+ //__forceinline vuint8 vreduce_max (const vuint8& v) { vuint8 v1 = vreduce_max4(v); return max(v1,shuffle4<1,0>(v1)); }
+
+ __forceinline vuint8 vreduce_add2(const vuint8& v) { return v + shuffle<1,0,3,2>(v); }
+ __forceinline vuint8 vreduce_add4(const vuint8& v) { vuint8 v1 = vreduce_add2(v); return v1 + shuffle<2,3,0,1>(v1); }
+ __forceinline vuint8 vreduce_add (const vuint8& v) { vuint8 v1 = vreduce_add4(v); return v1 + shuffle4<1,0>(v1); }
+
+ //__forceinline int reduce_min(const vuint8& v) { return toScalar(vreduce_min(v)); }
+ //__forceinline int reduce_max(const vuint8& v) { return toScalar(vreduce_max(v)); }
+ __forceinline int reduce_add(const vuint8& v) { return toScalar(vreduce_add(v)); }
+
+ //__forceinline size_t select_min(const vuint8& v) { return bsf(movemask(v == vreduce_min(v))); }
+ //__forceinline size_t select_max(const vuint8& v) { return bsf(movemask(v == vreduce_max(v))); }
+
+ //__forceinline size_t select_min(const vboolf8& valid, const vuint8& v) { const vuint8 a = select(valid,v,vuint8(pos_inf)); return bsf(movemask(valid & (a == vreduce_min(a)))); }
+ //__forceinline size_t select_max(const vboolf8& valid, const vuint8& v) { const vuint8 a = select(valid,v,vuint8(neg_inf)); return bsf(movemask(valid & (a == vreduce_max(a)))); }
+
+ __forceinline vuint8 assign(const vuint4& a) { return _mm256_castsi128_si256(a); }
+
+ ////////////////////////////////////////////////////////////////////////////////
+ /// Output Operators
+ ////////////////////////////////////////////////////////////////////////////////
+
+ __forceinline embree_ostream operator <<(embree_ostream cout, const vuint8& a) {
+ return cout << "<" << a[0] << ", " << a[1] << ", " << a[2] << ", " << a[3] << ", " << a[4] << ", " << a[5] << ", " << a[6] << ", " << a[7] << ">";
+ }
+}