25 #ifndef _GLIBCXX_EXPERIMENTAL_SIMD_H
26 #define _GLIBCXX_EXPERIMENTAL_SIMD_H
28 #if __cplusplus >= 201703L
30 #include "simd_detail.h"
34 #ifdef _GLIBCXX_DEBUG_UB
42 #if _GLIBCXX_SIMD_X86INTRIN
43 #include <x86intrin.h>
44 #elif _GLIBCXX_SIMD_HAVE_NEON
68 _GLIBCXX_SIMD_BEGIN_NAMESPACE
70 #if !_GLIBCXX_SIMD_X86INTRIN
71 using __m128 [[__gnu__::__vector_size__(16)]] = float;
72 using __m128d [[__gnu__::__vector_size__(16)]] = double;
73 using __m128i [[__gnu__::__vector_size__(16)]] =
long long;
74 using __m256 [[__gnu__::__vector_size__(32)]] = float;
75 using __m256d [[__gnu__::__vector_size__(32)]] = double;
76 using __m256i [[__gnu__::__vector_size__(32)]] =
long long;
77 using __m512 [[__gnu__::__vector_size__(64)]] = float;
78 using __m512d [[__gnu__::__vector_size__(64)]] = double;
79 using __m512i [[__gnu__::__vector_size__(64)]] =
long long;
101 template <
int _UsedBytes>
104 template <
int _UsedBytes>
105 struct _VecBltnBtmsk;
107 template <
typename _Tp,
int _Np>
108 using _VecN = _VecBuiltin<
sizeof(_Tp) * _Np>;
110 template <
int _UsedBytes = 16>
111 using _Sse = _VecBuiltin<_UsedBytes>;
113 template <
int _UsedBytes = 32>
114 using _Avx = _VecBuiltin<_UsedBytes>;
116 template <
int _UsedBytes = 64>
117 using _Avx512 = _VecBltnBtmsk<_UsedBytes>;
119 template <
int _UsedBytes = 16>
120 using _Neon = _VecBuiltin<_UsedBytes>;
123 using __sse = _Sse<>;
124 using __avx = _Avx<>;
125 using __avx512 = _Avx512<>;
126 using __neon = _Neon<>;
127 using __neon128 = _Neon<16>;
128 using __neon64 = _Neon<8>;
131 template <
typename _Tp,
size_t _Np,
typename...>
135 using fixed_size = _Fixed<_Np>;
137 using scalar = _Scalar;
142 template <
typename _Tp>
145 template <
typename _Tp>
148 template <
typename _Tp,
typename _Abi>
151 template <
typename _Tp,
typename _Abi>
154 template <
typename _Tp,
typename _Abi>
159 struct element_aligned_tag
161 template <
typename _Tp,
typename _Up =
typename _Tp::value_type>
162 static constexpr
size_t _S_alignment =
alignof(_Up);
164 template <
typename _Tp,
typename _Up>
165 _GLIBCXX_SIMD_INTRINSIC
static constexpr _Up*
170 struct vector_aligned_tag
172 template <
typename _Tp,
typename _Up =
typename _Tp::value_type>
173 static constexpr
size_t _S_alignment
174 = std::__bit_ceil(
sizeof(_Up) *
_Tp::size());
176 template <
typename _Tp,
typename _Up>
177 _GLIBCXX_SIMD_INTRINSIC
static constexpr _Up*
180 return static_cast<_Up*
>(
181 __builtin_assume_aligned(__ptr, _S_alignment<_Tp, _Up>));
185 template <
size_t _Np>
struct overaligned_tag
187 template <
typename _Tp,
typename _Up =
typename _Tp::value_type>
188 static constexpr
size_t _S_alignment = _Np;
190 template <
typename _Tp,
typename _Up>
191 _GLIBCXX_SIMD_INTRINSIC
static constexpr _Up*
193 {
return static_cast<_Up*
>(__builtin_assume_aligned(__ptr, _Np)); }
196 inline constexpr element_aligned_tag element_aligned = {};
198 inline constexpr vector_aligned_tag vector_aligned = {};
200 template <
size_t _Np>
201 inline constexpr overaligned_tag<_Np> overaligned = {};
205 using _SizeConstant = integral_constant<size_t, _X>;
209 template <
typename _Fp,
size_t... _I>
210 _GLIBCXX_SIMD_INTRINSIC constexpr
void
211 __execute_on_index_sequence(_Fp&& __f, index_sequence<_I...>)
212 { ((void)__f(_SizeConstant<_I>()), ...); }
214 template <
typename _Fp>
215 _GLIBCXX_SIMD_INTRINSIC constexpr
void
216 __execute_on_index_sequence(_Fp&&, index_sequence<>)
219 template <
size_t _Np,
typename _Fp>
220 _GLIBCXX_SIMD_INTRINSIC constexpr
void
221 __execute_n_times(_Fp&& __f)
223 __execute_on_index_sequence(
static_cast<_Fp&&
>(__f),
224 make_index_sequence<_Np>{});
229 template <
typename _R,
typename _Fp,
size_t... _I>
230 _GLIBCXX_SIMD_INTRINSIC constexpr _R
231 __execute_on_index_sequence_with_return(_Fp&& __f, index_sequence<_I...>)
232 {
return _R{__f(_SizeConstant<_I>())...}; }
234 template <
size_t _Np,
typename _R,
typename _Fp>
235 _GLIBCXX_SIMD_INTRINSIC constexpr _R
236 __generate_from_n_evaluations(_Fp&& __f)
238 return __execute_on_index_sequence_with_return<_R>(
239 static_cast<_Fp&&
>(__f), make_index_sequence<_Np>{});
244 template <
size_t... _I,
typename _F0,
typename _FArgs>
245 _GLIBCXX_SIMD_INTRINSIC constexpr
auto
246 __call_with_n_evaluations(index_sequence<_I...>, _F0&& __f0, _FArgs&& __fargs)
247 {
return __f0(__fargs(_SizeConstant<_I>())...); }
249 template <
size_t _Np,
typename _F0,
typename _FArgs>
250 _GLIBCXX_SIMD_INTRINSIC constexpr
auto
251 __call_with_n_evaluations(_F0&& __f0, _FArgs&& __fargs)
253 return __call_with_n_evaluations(make_index_sequence<_Np>{},
254 static_cast<_F0&&
>(__f0),
255 static_cast<_FArgs&&
>(__fargs));
260 template <
size_t _First = 0,
size_t... _It,
typename _Tp,
typename _Fp>
261 _GLIBCXX_SIMD_INTRINSIC constexpr
auto
262 __call_with_subscripts(_Tp&& __x, index_sequence<_It...>, _Fp&& __fun)
263 {
return __fun(__x[_First + _It]...); }
265 template <
size_t _Np,
size_t _First = 0,
typename _Tp,
typename _Fp>
266 _GLIBCXX_SIMD_INTRINSIC constexpr
auto
267 __call_with_subscripts(_Tp&& __x, _Fp&& __fun)
269 return __call_with_subscripts<_First>(
static_cast<_Tp&&
>(__x),
270 make_index_sequence<_Np>(),
271 static_cast<_Fp&&
>(__fun));
278 using _UChar =
unsigned char;
279 using _SChar =
signed char;
280 using _UShort =
unsigned short;
281 using _UInt =
unsigned int;
282 using _ULong =
unsigned long;
283 using _ULLong =
unsigned long long;
284 using _LLong =
long long;
288 template <
typename _T0,
typename...>
289 struct __first_of_pack
290 {
using type = _T0; };
292 template <
typename... _Ts>
293 using __first_of_pack_t =
typename __first_of_pack<_Ts...>::type;
297 template <
typename _Tp>
298 typename _Tp::value_type
299 __value_type_or_identity_impl(
int);
301 template <
typename _Tp>
303 __value_type_or_identity_impl(
float);
305 template <
typename _Tp>
306 using __value_type_or_identity_t
307 = decltype(__value_type_or_identity_impl<_Tp>(
int()));
311 template <
typename _Tp>
312 struct __is_vectorizable :
public is_arithmetic<_Tp> {};
315 struct __is_vectorizable<bool> :
public false_type {};
317 template <
typename _Tp>
318 inline constexpr
bool __is_vectorizable_v = __is_vectorizable<_Tp>::value;
321 template <
typename _Tp,
typename = enable_if_t<__is_vectorizable_v<_Tp>>>
322 using _Vectorizable = _Tp;
326 template <
typename _Ptr,
typename _ValueType>
327 struct __is_possible_loadstore_conversion
328 : conjunction<__is_vectorizable<_Ptr>, __is_vectorizable<_ValueType>> {};
331 struct __is_possible_loadstore_conversion<bool, bool> :
true_type {};
334 template <
typename _Ptr,
typename _ValueType,
336 __is_possible_loadstore_conversion<_Ptr, _ValueType>::value>>
337 using _LoadStorePtr = _Ptr;
341 template <
typename _Tp,
typename =
void_t<>>
344 template <
typename _Tp>
345 inline constexpr
bool __is_bitmask_v = __is_bitmask<_Tp>::value;
348 template <
typename _Tp>
349 struct __is_bitmask<_Tp,
350 void_t<decltype(declval<unsigned&>() = declval<_Tp>() & 1u)>>
355 #pragma GCC diagnostic push
356 #pragma GCC diagnostic ignored "-Wpedantic"
357 template <
size_t _Bytes>
361 if constexpr (_Bytes ==
sizeof(
int))
364 else if constexpr (_Bytes ==
sizeof(
char))
367 else if constexpr (_Bytes ==
sizeof(_SChar))
370 else if constexpr (_Bytes ==
sizeof(
short))
373 else if constexpr (_Bytes ==
sizeof(
long))
376 else if constexpr (_Bytes ==
sizeof(_LLong))
378 #ifdef __SIZEOF_INT128__
379 else if constexpr (_Bytes ==
sizeof(__int128))
382 else if constexpr (_Bytes %
sizeof(
int) == 0)
384 constexpr
size_t _Np = _Bytes /
sizeof(int);
389 _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
392 return __generate_from_n_evaluations<_Np, _Ip>(
393 [&](
auto __i) {
return __rhs._M_data[__i] & _M_data[__i]; });
396 _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
397 operator|(_Ip __rhs)
const
399 return __generate_from_n_evaluations<_Np, _Ip>(
400 [&](
auto __i) {
return __rhs._M_data[__i] | _M_data[__i]; });
403 _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
404 operator^(_Ip __rhs)
const
406 return __generate_from_n_evaluations<_Np, _Ip>(
407 [&](
auto __i) {
return __rhs._M_data[__i] ^ _M_data[__i]; });
410 _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
413 return __generate_from_n_evaluations<_Np, _Ip>(
414 [&](
auto __i) {
return ~_M_data[__i]; });
420 static_assert(_Bytes != _Bytes,
"this should be unreachable");
422 #pragma GCC diagnostic pop
424 template <
typename _Tp>
425 using __int_for_sizeof_t = decltype(__int_for_sizeof<
sizeof(_Tp)>());
427 template <
size_t _Np>
428 using __int_with_sizeof_t = decltype(__int_for_sizeof<_Np>());
432 template <
typename _Tp>
436 struct __is_fixed_size_abi<simd_abi::fixed_size<_Np>> :
true_type {};
438 template <
typename _Tp>
439 inline constexpr
bool __is_fixed_size_abi_v = __is_fixed_size_abi<_Tp>::value;
443 constexpr
inline bool __have_mmx = _GLIBCXX_SIMD_HAVE_MMX;
444 constexpr
inline bool __have_sse = _GLIBCXX_SIMD_HAVE_SSE;
445 constexpr
inline bool __have_sse2 = _GLIBCXX_SIMD_HAVE_SSE2;
446 constexpr
inline bool __have_sse3 = _GLIBCXX_SIMD_HAVE_SSE3;
447 constexpr
inline bool __have_ssse3 = _GLIBCXX_SIMD_HAVE_SSSE3;
448 constexpr
inline bool __have_sse4_1 = _GLIBCXX_SIMD_HAVE_SSE4_1;
449 constexpr
inline bool __have_sse4_2 = _GLIBCXX_SIMD_HAVE_SSE4_2;
450 constexpr
inline bool __have_xop = _GLIBCXX_SIMD_HAVE_XOP;
451 constexpr
inline bool __have_avx = _GLIBCXX_SIMD_HAVE_AVX;
452 constexpr
inline bool __have_avx2 = _GLIBCXX_SIMD_HAVE_AVX2;
453 constexpr
inline bool __have_bmi = _GLIBCXX_SIMD_HAVE_BMI1;
454 constexpr
inline bool __have_bmi2 = _GLIBCXX_SIMD_HAVE_BMI2;
455 constexpr
inline bool __have_lzcnt = _GLIBCXX_SIMD_HAVE_LZCNT;
456 constexpr
inline bool __have_sse4a = _GLIBCXX_SIMD_HAVE_SSE4A;
457 constexpr
inline bool __have_fma = _GLIBCXX_SIMD_HAVE_FMA;
458 constexpr
inline bool __have_fma4 = _GLIBCXX_SIMD_HAVE_FMA4;
459 constexpr
inline bool __have_f16c = _GLIBCXX_SIMD_HAVE_F16C;
460 constexpr
inline bool __have_popcnt = _GLIBCXX_SIMD_HAVE_POPCNT;
461 constexpr
inline bool __have_avx512f = _GLIBCXX_SIMD_HAVE_AVX512F;
462 constexpr
inline bool __have_avx512dq = _GLIBCXX_SIMD_HAVE_AVX512DQ;
463 constexpr
inline bool __have_avx512vl = _GLIBCXX_SIMD_HAVE_AVX512VL;
464 constexpr
inline bool __have_avx512bw = _GLIBCXX_SIMD_HAVE_AVX512BW;
465 constexpr
inline bool __have_avx512dq_vl = __have_avx512dq && __have_avx512vl;
466 constexpr
inline bool __have_avx512bw_vl = __have_avx512bw && __have_avx512vl;
468 constexpr
inline bool __have_neon = _GLIBCXX_SIMD_HAVE_NEON;
469 constexpr
inline bool __have_neon_a32 = _GLIBCXX_SIMD_HAVE_NEON_A32;
470 constexpr
inline bool __have_neon_a64 = _GLIBCXX_SIMD_HAVE_NEON_A64;
471 constexpr
inline bool __support_neon_float =
472 #if defined __GCC_IEC_559
474 #elif defined __FAST_MATH__
480 #ifdef __POWER9_VECTOR__
481 constexpr
inline bool __have_power9vec =
true;
483 constexpr
inline bool __have_power9vec =
false;
485 #if defined __POWER8_VECTOR__
486 constexpr
inline bool __have_power8vec =
true;
488 constexpr
inline bool __have_power8vec = __have_power9vec;
491 constexpr
inline bool __have_power_vsx =
true;
493 constexpr
inline bool __have_power_vsx = __have_power8vec;
495 #if defined __ALTIVEC__
496 constexpr
inline bool __have_power_vmx =
true;
498 constexpr
inline bool __have_power_vmx = __have_power_vsx;
503 template <
typename _Abi>
506 {
return is_same_v<simd_abi::scalar, _Abi>; }
510 template <
template <
int>
class _Abi,
int _Bytes>
512 __abi_bytes_impl(_Abi<_Bytes>*)
515 template <
typename _Tp>
517 __abi_bytes_impl(_Tp*)
520 template <
typename _Abi>
521 inline constexpr
int __abi_bytes_v
522 = __abi_bytes_impl(
static_cast<_Abi*
>(
nullptr));
526 template <
typename _Abi>
528 __is_builtin_bitmask_abi()
529 {
return is_same_v<simd_abi::_VecBltnBtmsk<__abi_bytes_v<_Abi>>, _Abi>; }
533 template <
typename _Abi>
537 constexpr
auto _Bytes = __abi_bytes_v<_Abi>;
538 return _Bytes <= 16 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
543 template <
typename _Abi>
547 constexpr
auto _Bytes = __abi_bytes_v<_Abi>;
548 return _Bytes > 16 && _Bytes <= 32
549 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
554 template <
typename _Abi>
558 constexpr
auto _Bytes = __abi_bytes_v<_Abi>;
559 return _Bytes <= 64 && is_same_v<simd_abi::_Avx512<_Bytes>, _Abi>;
564 template <
typename _Abi>
568 constexpr
auto _Bytes = __abi_bytes_v<_Abi>;
569 return _Bytes <= 16 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
574 template <
typename,
typename _Up>
575 struct __make_dependent
576 {
using type = _Up; };
578 template <
typename _Tp,
typename _Up>
579 using __make_dependent_t =
typename __make_dependent<_Tp, _Up>::type;
585 template <
typename... _Args>
586 [[noreturn]] _GLIBCXX_SIMD_ALWAYS_INLINE
void
587 __invoke_ub([[maybe_unused]]
const char* __msg,
588 [[maybe_unused]]
const _Args&... __args)
590 #ifdef _GLIBCXX_DEBUG_UB
591 __builtin_fprintf(stderr, __msg, __args...);
594 __builtin_unreachable();
600 template <
typename _Tp>
601 struct __assert_unreachable
602 { static_assert(!is_same_v<_Tp, _Tp>,
"this should be unreachable"); };
606 template <typename _Tp, typename _Ap, size_t _Np = simd_size<_Tp, _Ap>::value>
608 __size_or_zero_dispatch(
int)
611 template <
typename _Tp,
typename _Ap>
613 __size_or_zero_dispatch(
float)
616 template <
typename _Tp,
typename _Ap>
617 inline constexpr
size_t __size_or_zero_v
618 = __size_or_zero_dispatch<_Tp, _Ap>(0);
622 inline constexpr
size_t
623 __div_roundup(
size_t __a,
size_t __b)
624 {
return (__a + __b - 1) / __b; }
633 _GLIBCXX_SIMD_INTRINSIC constexpr _ExactBool(
bool __b) : _M_data(__b) {}
635 _ExactBool(
int) =
delete;
637 _GLIBCXX_SIMD_INTRINSIC constexpr
operator bool()
const {
return _M_data; }
647 template <
typename _Tp>
648 using __may_alias [[__gnu__::__may_alias__]] = _Tp;
653 struct _UnsupportedBase
655 _UnsupportedBase() =
delete;
656 _UnsupportedBase(
const _UnsupportedBase&) =
delete;
657 _UnsupportedBase&
operator=(
const _UnsupportedBase&) =
delete;
658 ~_UnsupportedBase() =
delete;
671 struct _InvalidTraits
674 using _SimdBase = _UnsupportedBase;
675 using _MaskBase = _UnsupportedBase;
677 static constexpr
size_t _S_full_size = 0;
678 static constexpr
bool _S_is_partial =
false;
680 static constexpr
size_t _S_simd_align = 1;
682 struct _SimdMember {};
683 struct _SimdCastType;
685 static constexpr
size_t _S_mask_align = 1;
687 struct _MaskMember {};
688 struct _MaskCastType;
693 template <
typename _Tp,
typename _Abi,
typename =
void_t<>>
694 struct _SimdTraits : _InvalidTraits {};
702 inline constexpr
struct _PrivateInit {} __private_init = {};
704 inline constexpr
struct _BitsetInit {} __bitset_init = {};
708 template <
typename _From,
typename _To,
bool = is_arithmetic_v<_From>,
709 bool = is_arithmetic_v<_To>>
710 struct __is_narrowing_conversion;
714 template <
typename _From,
typename _To>
715 struct __is_narrowing_conversion<_From, _To, true, true>
716 :
public __bool_constant<(
717 __digits_v<_From> > __digits_v<_To>
718 || __finite_max_v<_From> > __finite_max_v<_To>
719 || __finite_min_v<_From> < __finite_min_v<_To>
720 || (is_signed_v<_From> && is_unsigned_v<_To>))> {};
722 template <
typename _Tp>
723 struct __is_narrowing_conversion<_Tp, bool, true, true>
727 struct __is_narrowing_conversion<bool, bool, true, true>
730 template <
typename _Tp>
731 struct __is_narrowing_conversion<_Tp, _Tp, true, true>
734 template <
typename _From,
typename _To>
735 struct __is_narrowing_conversion<_From, _To, false, true>
736 :
public negation<is_convertible<_From, _To>> {};
740 template <
typename _From,
typename _To,
bool = (sizeof(_From) < sizeof(_To))>
741 struct __converts_to_higher_
integer_rank : public true_type {};
744 template <
typename _From,
typename _To>
745 struct __converts_to_higher_integer_rank<_From, _To, false>
746 :
public is_same<decltype(declval<_From>() + declval<_To>()), _To> {};
750 template <
typename _Tp,
typename _Ap>
751 _GLIBCXX_SIMD_INTRINSIC constexpr
const auto&
752 __data(
const simd<_Tp, _Ap>& __x);
754 template <
typename _Tp,
typename _Ap>
755 _GLIBCXX_SIMD_INTRINSIC constexpr
auto&
756 __data(simd<_Tp, _Ap>& __x);
758 template <
typename _Tp,
typename _Ap>
759 _GLIBCXX_SIMD_INTRINSIC constexpr
const auto&
760 __data(
const simd_mask<_Tp, _Ap>& __x);
762 template <
typename _Tp,
typename _Ap>
763 _GLIBCXX_SIMD_INTRINSIC constexpr
auto&
764 __data(simd_mask<_Tp, _Ap>& __x);
768 template <
typename _FromT,
typename _FromA,
typename _ToT,
typename _ToA,
770 struct _SimdConverter;
772 template <
typename _Tp,
typename _Ap>
773 struct _SimdConverter<_Tp, _Ap, _Tp, _Ap, void>
775 template <
typename _Up>
776 _GLIBCXX_SIMD_INTRINSIC
const _Up&
777 operator()(
const _Up& __x)
783 template <
typename _V>
784 _GLIBCXX_SIMD_INTRINSIC constexpr
auto
785 __to_value_type_or_member_type(
const _V& __x) -> decltype(__data(__x))
786 {
return __data(__x); }
788 template <
typename _V>
789 _GLIBCXX_SIMD_INTRINSIC constexpr
const typename _V::value_type&
790 __to_value_type_or_member_type(
const typename _V::value_type& __x)
795 template <
size_t _Size>
796 struct __bool_storage_member_type;
798 template <
size_t _Size>
799 using __bool_storage_member_type_t =
800 typename __bool_storage_member_type<_Size>::type;
810 template <
typename _Tp,
typename... _Abis>
815 template <
typename _Tp,
int _Np>
816 struct __fixed_size_storage;
818 template <
typename _Tp,
int _Np>
819 using __fixed_size_storage_t =
typename __fixed_size_storage<_Tp, _Np>::type;
823 template <
typename _Tp,
size_t _Size,
typename =
void_t<>>
826 template <
typename _Tp>
827 using _SimdWrapper8 = _SimdWrapper<_Tp, 8 /
sizeof(_Tp)>;
828 template <
typename _Tp>
829 using _SimdWrapper16 = _SimdWrapper<_Tp, 16 /
sizeof(_Tp)>;
830 template <
typename _Tp>
831 using _SimdWrapper32 = _SimdWrapper<_Tp, 32 /
sizeof(_Tp)>;
832 template <
typename _Tp>
833 using _SimdWrapper64 = _SimdWrapper<_Tp, 64 /
sizeof(_Tp)>;
837 template <
typename _Tp>
840 template <
typename _Tp,
size_t _Np>
841 struct __is_simd_wrapper<_SimdWrapper<_Tp, _Np>> :
true_type {};
843 template <
typename _Tp>
844 inline constexpr
bool __is_simd_wrapper_v = __is_simd_wrapper<_Tp>::value;
851 template <
typename _Tp,
typename _Fp>
853 _S_bit_iteration(_Tp __mask, _Fp&& __f)
855 static_assert(
sizeof(_ULLong) >=
sizeof(_Tp));
856 conditional_t<
sizeof(_Tp) <=
sizeof(_UInt), _UInt, _ULLong> __k;
857 if constexpr (is_convertible_v<_Tp, decltype(__k)>)
860 __k = __mask.to_ullong();
863 __f(std::__countr_zero(__k));
873 template <
typename _Tp =
void>
875 { constexpr _Tp operator()(_Tp __a)
const {
return ++__a; } };
878 struct __increment<void>
880 template <
typename _Tp>
882 operator()(_Tp __a)
const
886 template <
typename _Tp =
void>
888 { constexpr _Tp operator()(_Tp __a)
const {
return --__a; } };
891 struct __decrement<void>
893 template <
typename _Tp>
895 operator()(_Tp __a)
const
901 template <
typename _From,
typename _To,
903 __is_narrowing_conversion<__remove_cvref_t<_From>, _To>>::value>>
904 using _ValuePreserving = _From;
906 template <
typename _From,
typename _To,
907 typename _DecayedFrom = __remove_cvref_t<_From>,
909 is_convertible<_From, _To>,
911 is_same<_DecayedFrom, _To>, is_same<_DecayedFrom, int>,
912 conjunction<is_same<_DecayedFrom, _UInt>, is_unsigned<_To>>,
913 negation<__is_narrowing_conversion<_DecayedFrom, _To>>>>::value>>
914 using _ValuePreservingOrInt = _From;
918 template <
typename _Tp,
size_t _Bytes,
typename =
void_t<>>
919 struct __intrinsic_type;
921 template <
typename _Tp,
size_t _Size>
922 using __intrinsic_type_t =
923 typename __intrinsic_type<_Tp, _Size *
sizeof(_Tp)>::type;
925 template <
typename _Tp>
926 using __intrinsic_type2_t =
typename __intrinsic_type<_Tp, 2>::type;
927 template <
typename _Tp>
928 using __intrinsic_type4_t =
typename __intrinsic_type<_Tp, 4>::type;
929 template <
typename _Tp>
930 using __intrinsic_type8_t =
typename __intrinsic_type<_Tp, 8>::type;
931 template <
typename _Tp>
932 using __intrinsic_type16_t =
typename __intrinsic_type<_Tp, 16>::type;
933 template <
typename _Tp>
934 using __intrinsic_type32_t =
typename __intrinsic_type<_Tp, 32>::type;
935 template <
typename _Tp>
936 using __intrinsic_type64_t =
typename __intrinsic_type<_Tp, 64>::type;
940 template <
size_t _Np,
bool _Sanitized = false>
943 template <
size_t _Np,
bool _Sanitized>
944 struct __is_bitmask<_BitMask<_Np, _Sanitized>, void> :
true_type {};
946 template <
size_t _Np>
947 using _SanitizedBitMask = _BitMask<_Np, true>;
949 template <
size_t _Np,
bool _Sanitized>
952 static_assert(_Np > 0);
954 static constexpr
size_t _NBytes = __div_roundup(_Np, __CHAR_BIT__);
958 sizeof(_ULLong), std::__bit_ceil(_NBytes))>>>;
960 static constexpr
int _S_array_size = __div_roundup(_NBytes,
sizeof(_Tp));
962 _Tp _M_bits[_S_array_size];
964 static constexpr
int _S_unused_bits
965 = _Np == 1 ? 0 : _S_array_size *
sizeof(_Tp) * __CHAR_BIT__ - _Np;
967 static constexpr _Tp _S_bitmask = +_Tp(~_Tp()) >> _S_unused_bits;
969 constexpr _BitMask() noexcept = default;
971 constexpr _BitMask(
unsigned long long __x) noexcept
972 : _M_bits{
static_cast<_Tp
>(__x)} {}
974 _BitMask(bitset<_Np> __x) noexcept : _BitMask(__x.to_ullong()) {}
976 constexpr _BitMask(
const _BitMask&) noexcept =
default;
978 template <
bool _RhsSanitized,
typename =
enable_if_t<_RhsSanitized ==
false
979 && _Sanitized ==
true>>
980 constexpr _BitMask(
const _BitMask<_Np, _RhsSanitized>& __rhs) noexcept
981 : _BitMask(__rhs._M_sanitized()) {}
983 constexpr
operator _SimdWrapper<bool, _Np>() const noexcept
985 static_assert(_S_array_size == 1);
991 _M_to_bits() const noexcept
993 static_assert(_S_array_size == 1);
998 constexpr
unsigned long long
999 to_ullong() const noexcept
1001 static_assert(_S_array_size == 1);
1006 constexpr
unsigned long
1007 to_ulong() const noexcept
1009 static_assert(_S_array_size == 1);
1013 constexpr bitset<_Np>
1014 _M_to_bitset() const noexcept
1016 static_assert(_S_array_size == 1);
1020 constexpr decltype(
auto)
1021 _M_sanitized() const noexcept
1023 if constexpr (_Sanitized)
1025 else if constexpr (_Np == 1)
1026 return _SanitizedBitMask<_Np>(_M_bits[0]);
1029 _SanitizedBitMask<_Np> __r = {};
1030 for (
int __i = 0; __i < _S_array_size; ++__i)
1031 __r._M_bits[__i] = _M_bits[__i];
1032 if constexpr (_S_unused_bits > 0)
1033 __r._M_bits[_S_array_size - 1] &= _S_bitmask;
1038 template <
size_t _Mp,
bool _LSanitized>
1039 constexpr _BitMask<_Np + _Mp, _Sanitized>
1040 _M_prepend(_BitMask<_Mp, _LSanitized> __lsb)
const noexcept
1042 constexpr
size_t _RN = _Np + _Mp;
1043 using _Rp = _BitMask<_RN, _Sanitized>;
1044 if constexpr (_Rp::_S_array_size == 1)
1046 _Rp __r{{_M_bits[0]}};
1047 __r._M_bits[0] <<= _Mp;
1048 __r._M_bits[0] |= __lsb._M_sanitized()._M_bits[0];
1052 __assert_unreachable<_Rp>();
1058 template <
size_t _DropLsb,
size_t _NewSize = _Np - _DropLsb>
1060 _M_extract() const noexcept
1062 static_assert(_Np > _DropLsb);
1063 static_assert(_DropLsb + _NewSize <=
sizeof(_ULLong) * __CHAR_BIT__,
1064 "not implemented for bitmasks larger than one ullong");
1065 if constexpr (_NewSize == 1)
1067 return _SanitizedBitMask<1>(_M_bits[0] & (_Tp(1) << _DropLsb));
1069 return _BitMask<_NewSize,
1070 ((_NewSize + _DropLsb == sizeof(_Tp) * __CHAR_BIT__
1071 && _NewSize + _DropLsb <= _Np)
1072 || ((_Sanitized || _Np == sizeof(_Tp) * __CHAR_BIT__)
1073 && _NewSize + _DropLsb >= _Np))>(_M_bits[0]
1079 all() const noexcept
1081 if constexpr (_Np == 1)
1083 else if constexpr (!_Sanitized)
1084 return _M_sanitized().all();
1087 constexpr _Tp __allbits = ~_Tp();
1088 for (
int __i = 0; __i < _S_array_size - 1; ++__i)
1089 if (_M_bits[__i] != __allbits)
1091 return _M_bits[_S_array_size - 1] == _S_bitmask;
1098 any() const noexcept
1100 if constexpr (_Np == 1)
1102 else if constexpr (!_Sanitized)
1103 return _M_sanitized().any();
1106 for (
int __i = 0; __i < _S_array_size - 1; ++__i)
1107 if (_M_bits[__i] != 0)
1109 return _M_bits[_S_array_size - 1] != 0;
1115 none() const noexcept
1117 if constexpr (_Np == 1)
1119 else if constexpr (!_Sanitized)
1120 return _M_sanitized().none();
1123 for (
int __i = 0; __i < _S_array_size - 1; ++__i)
1124 if (_M_bits[__i] != 0)
1126 return _M_bits[_S_array_size - 1] == 0;
1133 count() const noexcept
1135 if constexpr (_Np == 1)
1137 else if constexpr (!_Sanitized)
1138 return _M_sanitized().none();
1141 int __result = __builtin_popcountll(_M_bits[0]);
1142 for (
int __i = 1; __i < _S_array_size; ++__i)
1143 __result += __builtin_popcountll(_M_bits[__i]);
1150 operator[](
size_t __i)
const noexcept
1152 if constexpr (_Np == 1)
1154 else if constexpr (_S_array_size == 1)
1155 return (_M_bits[0] >> __i) & 1;
1158 const size_t __j = __i / (
sizeof(_Tp) * __CHAR_BIT__);
1159 const size_t __shift = __i % (
sizeof(_Tp) * __CHAR_BIT__);
1160 return (_M_bits[__j] >> __shift) & 1;
1164 template <
size_t __i>
1166 operator[](_SizeConstant<__i>)
const noexcept
1168 static_assert(__i < _Np);
1169 constexpr
size_t __j = __i / (
sizeof(_Tp) * __CHAR_BIT__);
1170 constexpr
size_t __shift = __i % (
sizeof(_Tp) * __CHAR_BIT__);
1171 return static_cast<bool>(_M_bits[__j] & (_Tp(1) << __shift));
1176 set(
size_t __i,
bool __x) noexcept
1178 if constexpr (_Np == 1)
1180 else if constexpr (_S_array_size == 1)
1182 _M_bits[0] &= ~_Tp(_Tp(1) << __i);
1183 _M_bits[0] |= _Tp(_Tp(__x) << __i);
1187 const size_t __j = __i / (
sizeof(_Tp) * __CHAR_BIT__);
1188 const size_t __shift = __i % (
sizeof(_Tp) * __CHAR_BIT__);
1189 _M_bits[__j] &= ~_Tp(_Tp(1) << __shift);
1190 _M_bits[__j] |= _Tp(_Tp(__x) << __shift);
1194 template <
size_t __i>
1196 set(_SizeConstant<__i>,
bool __x) noexcept
1198 static_assert(__i < _Np);
1199 if constexpr (_Np == 1)
1203 constexpr
size_t __j = __i / (
sizeof(_Tp) * __CHAR_BIT__);
1204 constexpr
size_t __shift = __i % (
sizeof(_Tp) * __CHAR_BIT__);
1205 constexpr _Tp __mask = ~_Tp(_Tp(1) << __shift);
1206 _M_bits[__j] &= __mask;
1207 _M_bits[__j] |= _Tp(_Tp(__x) << __shift);
1213 operator~() const noexcept
1215 if constexpr (_Np == 1)
1219 _BitMask __result{};
1220 for (
int __i = 0; __i < _S_array_size - 1; ++__i)
1221 __result._M_bits[__i] = ~_M_bits[__i];
1222 if constexpr (_Sanitized)
1223 __result._M_bits[_S_array_size - 1]
1224 = _M_bits[_S_array_size - 1] ^ _S_bitmask;
1226 __result._M_bits[_S_array_size - 1] = ~_M_bits[_S_array_size - 1];
1232 operator^=(
const _BitMask& __b) & noexcept
1234 __execute_n_times<_S_array_size>(
1235 [&](
auto __i) { _M_bits[__i] ^= __b._M_bits[__i]; });
1240 operator|=(
const _BitMask& __b) & noexcept
1242 __execute_n_times<_S_array_size>(
1243 [&](
auto __i) { _M_bits[__i] |= __b._M_bits[__i]; });
1248 operator&=(
const _BitMask& __b) & noexcept
1250 __execute_n_times<_S_array_size>(
1251 [&](
auto __i) { _M_bits[__i] &= __b._M_bits[__i]; });
1255 friend constexpr _BitMask
1256 operator^(
const _BitMask& __a,
const _BitMask& __b) noexcept
1263 friend constexpr _BitMask
1264 operator|(
const _BitMask& __a,
const _BitMask& __b) noexcept
1271 friend constexpr _BitMask
1272 operator&(
const _BitMask& __a,
const _BitMask& __b) noexcept
1279 _GLIBCXX_SIMD_INTRINSIC
1281 _M_is_constprop()
const
1283 if constexpr (_S_array_size == 0)
1284 return __builtin_constant_p(_M_bits[0]);
1287 for (
int __i = 0; __i < _S_array_size; ++__i)
1288 if (!__builtin_constant_p(_M_bits[__i]))
1299 template <
typename _Tp =
void>
1300 static inline constexpr
int __min_vector_size = 2 *
sizeof(_Tp);
1302 #if _GLIBCXX_SIMD_HAVE_NEON
1304 inline constexpr
int __min_vector_size<void> = 8;
1307 inline constexpr
int __min_vector_size<void> = 16;
1312 template <
typename _Tp,
size_t _Np,
typename =
void>
1313 struct __vector_type_n {};
1316 template <
typename _Tp>
1317 struct __vector_type_n<_Tp, 0, void> {};
1320 template <
typename _Tp>
1321 struct __vector_type_n<_Tp, 1,
enable_if_t<__is_vectorizable_v<_Tp>>>
1322 {
using type = _Tp; };
1325 template <
typename _Tp,
size_t _Np>
1326 struct __vector_type_n<_Tp, _Np,
1327 enable_if_t<__is_vectorizable_v<_Tp> && _Np >= 2>>
1329 static constexpr
size_t _S_Np2 = std::__bit_ceil(_Np *
sizeof(_Tp));
1331 static constexpr
size_t _S_Bytes =
1337 _S_Np2 < __min_vector_size<_Tp> ? __min_vector_size<_Tp>
1340 using type [[__gnu__::__vector_size__(_S_Bytes)]] = _Tp;
1343 template <
typename _Tp,
size_t _Bytes,
size_t = _Bytes % sizeof(_Tp)>
1344 struct __vector_type;
1346 template <
typename _Tp,
size_t _Bytes>
1347 struct __vector_type<_Tp, _Bytes, 0>
1348 : __vector_type_n<_Tp, _Bytes / sizeof(_Tp)> {};
1350 template <
typename _Tp,
size_t _Size>
1351 using __vector_type_t =
typename __vector_type_n<_Tp, _Size>::type;
1353 template <
typename _Tp>
1354 using __vector_type2_t =
typename __vector_type<_Tp, 2>::type;
1355 template <
typename _Tp>
1356 using __vector_type4_t =
typename __vector_type<_Tp, 4>::type;
1357 template <
typename _Tp>
1358 using __vector_type8_t =
typename __vector_type<_Tp, 8>::type;
1359 template <
typename _Tp>
1360 using __vector_type16_t =
typename __vector_type<_Tp, 16>::type;
1361 template <
typename _Tp>
1362 using __vector_type32_t =
typename __vector_type<_Tp, 32>::type;
1363 template <
typename _Tp>
1364 using __vector_type64_t =
typename __vector_type<_Tp, 64>::type;
1368 template <
typename _Tp,
typename =
void_t<>>
1371 template <
typename _Tp>
1372 struct __is_vector_type<
1373 _Tp,
void_t<typename __vector_type<
1375 : is_same<_Tp, typename __vector_type<
1376 remove_reference_t<decltype(declval<_Tp>()[0])>,
1377 sizeof(_Tp)>::type> {};
1379 template <
typename _Tp>
1380 inline constexpr
bool __is_vector_type_v = __is_vector_type<_Tp>::value;
1384 template <
typename _Tp,
typename =
void_t<>>
1385 struct _VectorTraitsImpl;
1387 template <
typename _Tp>
1388 struct _VectorTraitsImpl<_Tp,
enable_if_t<__is_vector_type_v<_Tp>>>
1391 using value_type = remove_reference_t<decltype(declval<_Tp>()[0])>;
1392 static constexpr
int _S_full_size =
sizeof(_Tp) /
sizeof(value_type);
1393 using _Wrapper = _SimdWrapper<value_type, _S_full_size>;
1394 template <
typename _Up,
int _W = _S_full_size>
1395 static constexpr
bool _S_is
1396 = is_same_v<value_type, _Up> && _W == _S_full_size;
1399 template <
typename _Tp,
size_t _Np>
1400 struct _VectorTraitsImpl<_SimdWrapper<_Tp, _Np>,
1401 void_t<__vector_type_t<_Tp, _Np>>>
1403 using type = __vector_type_t<_Tp, _Np>;
1404 using value_type = _Tp;
1405 static constexpr
int _S_full_size =
sizeof(type) /
sizeof(value_type);
1406 using _Wrapper = _SimdWrapper<_Tp, _Np>;
1407 static constexpr
bool _S_is_partial = (_Np == _S_full_size);
1408 static constexpr
int _S_partial_width = _Np;
1409 template <
typename _Up,
int _W = _S_full_size>
1410 static constexpr
bool _S_is
1411 = is_same_v<value_type, _Up>&& _W == _S_full_size;
1414 template <typename _Tp, typename = typename _VectorTraitsImpl<_Tp>::type>
1415 using _VectorTraits = _VectorTraitsImpl<_Tp>;
1419 template <
typename _V>
1420 _GLIBCXX_SIMD_INTRINSIC constexpr
auto
1423 if constexpr (__is_vector_type_v<_V>)
1425 else if constexpr (is_simd<_V>::value || is_simd_mask<_V>::value)
1426 return __data(__x)._M_data;
1427 else if constexpr (__is_vectorizable_v<_V>)
1428 return __vector_type_t<_V, 2>{__x};
1435 template <
size_t _Np = 0,
typename _V>
1436 _GLIBCXX_SIMD_INTRINSIC constexpr
auto
1437 __as_wrapper(_V __x)
1439 if constexpr (__is_vector_type_v<_V>)
1440 return _SimdWrapper<typename _VectorTraits<_V>::value_type,
1441 (_Np > 0 ? _Np : _VectorTraits<_V>::_S_full_size)>(__x);
1442 else if constexpr (is_simd<_V>::value || is_simd_mask<_V>::value)
1449 static_assert(_V::_S_size == _Np);
1456 template <
typename _To,
typename _From>
1457 _GLIBCXX_SIMD_INTRINSIC constexpr _To
1458 __intrin_bitcast(_From __v)
1460 static_assert(__is_vector_type_v<_From> && __is_vector_type_v<_To>);
1461 if constexpr (
sizeof(_To) ==
sizeof(_From))
1462 return reinterpret_cast<_To
>(__v);
1463 else if constexpr (
sizeof(_From) >
sizeof(_To))
1464 if constexpr (
sizeof(_To) >= 16)
1465 return reinterpret_cast<const __may_alias<_To>&>(__v);
1469 __builtin_memcpy(&__r, &__v,
sizeof(_To));
1472 #if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1473 else if constexpr (__have_avx &&
sizeof(_From) == 16 &&
sizeof(_To) == 32)
1474 return reinterpret_cast<_To>(__builtin_ia32_ps256_ps(
1475 reinterpret_cast<__vector_type_t<
float, 4>>(__v)));
1476 else if constexpr (__have_avx512f && sizeof(_From) == 16
1477 && sizeof(_To) == 64)
1478 return reinterpret_cast<_To>(__builtin_ia32_ps512_ps(
1479 reinterpret_cast<__vector_type_t<
float, 4>>(__v)));
1480 else if constexpr (__have_avx512f && sizeof(_From) == 32
1481 && sizeof(_To) == 64)
1482 return reinterpret_cast<_To>(__builtin_ia32_ps512_256ps(
1483 reinterpret_cast<__vector_type_t<
float, 8>>(__v)));
1485 else if constexpr (
sizeof(__v) <= 8)
1486 return reinterpret_cast<_To>(
1487 __vector_type_t<__int_for_sizeof_t<_From>, sizeof(_To) / sizeof(_From)>{
1488 reinterpret_cast<__int_for_sizeof_t<_From>
>(__v)});
1491 static_assert(
sizeof(_To) >
sizeof(_From));
1493 __builtin_memcpy(&__r, &__v,
sizeof(_From));
1500 template <
typename _To,
size_t _NN = 0,
typename _From,
1501 typename _FromVT = _VectorTraits<_From>,
1502 size_t _Np = _NN == 0 ?
sizeof(_From) /
sizeof(_To) : _NN>
1503 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_To, _Np>
1504 __vector_bitcast(_From __x)
1506 using _R = __vector_type_t<_To, _Np>;
1507 return __intrin_bitcast<_R>(__x);
1510 template <
typename _To,
size_t _NN = 0,
typename _Tp,
size_t _Nx,
1512 = _NN == 0 ?
sizeof(_SimdWrapper<_Tp, _Nx>) /
sizeof(_To) : _NN>
1513 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_To, _Np>
1514 __vector_bitcast(
const _SimdWrapper<_Tp, _Nx>& __x)
1516 static_assert(_Np > 1);
1517 return __intrin_bitcast<__vector_type_t<_To, _Np>>(__x._M_data);
1522 #ifdef _GLIBCXX_SIMD_WORKAROUND_PR85048
1523 template <
typename _To,
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1524 _To __convert_x86(_Tp);
1526 template <
typename _To,
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1527 _To __convert_x86(_Tp, _Tp);
1529 template <
typename _To,
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1530 _To __convert_x86(_Tp, _Tp, _Tp, _Tp);
1532 template <
typename _To,
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1533 _To __convert_x86(_Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp);
1535 template <
typename _To,
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1536 _To __convert_x86(_Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp,
1537 _Tp, _Tp, _Tp, _Tp);
1542 template <
typename _To,
typename _From>
1543 _GLIBCXX_SIMD_INTRINSIC constexpr _To
1544 __bit_cast(
const _From __x)
1547 static_assert(
sizeof(_To) ==
sizeof(_From));
1548 constexpr
bool __to_is_vectorizable
1549 = is_arithmetic_v<_To> || is_enum_v<_To>;
1550 constexpr
bool __from_is_vectorizable
1551 = is_arithmetic_v<_From> || is_enum_v<_From>;
1552 if constexpr (__is_vector_type_v<_To> && __is_vector_type_v<_From>)
1553 return reinterpret_cast<_To
>(__x);
1554 else if constexpr (__is_vector_type_v<_To> && __from_is_vectorizable)
1556 using _FV [[gnu::vector_size(
sizeof(_From))]] = _From;
1557 return reinterpret_cast<_To
>(_FV{__x});
1559 else if constexpr (__to_is_vectorizable && __from_is_vectorizable)
1561 using _TV [[gnu::vector_size(
sizeof(_To))]] = _To;
1562 using _FV [[gnu::vector_size(
sizeof(_From))]] = _From;
1563 return reinterpret_cast<_TV
>(_FV{__x})[0];
1565 else if constexpr (__to_is_vectorizable && __is_vector_type_v<_From>)
1567 using _TV [[gnu::vector_size(
sizeof(_To))]] = _To;
1568 return reinterpret_cast<_TV
>(__x)[0];
1573 __builtin_memcpy(
reinterpret_cast<char*
>(&__r),
1574 reinterpret_cast<const char*
>(&__x),
sizeof(_To));
1581 template <
typename _Tp,
typename _TVT = _VectorTraits<_Tp>,
1583 = __
intrinsic_type_t<
typename _TVT::value_type, _TVT::_S_full_size>>
1584 _GLIBCXX_SIMD_INTRINSIC constexpr _R
1585 __to_intrin(_Tp __x)
1587 static_assert(
sizeof(__x) <=
sizeof(_R),
1588 "__to_intrin may never drop values off the end");
1589 if constexpr (
sizeof(__x) ==
sizeof(_R))
1590 return reinterpret_cast<_R
>(__as_vector(__x));
1593 using _Up = __int_for_sizeof_t<_Tp>;
1594 return reinterpret_cast<_R
>(
1595 __vector_type_t<_Up,
sizeof(_R) /
sizeof(_Up)>{__bit_cast<_Up>(__x)});
1601 template <
typename _Tp,
typename... _Args>
1602 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp,
sizeof...(_Args)>
1603 __make_vector(
const _Args&... __args)
1605 return __vector_type_t<_Tp,
sizeof...(_Args)>{
static_cast<_Tp
>(__args)...};
1610 template <
size_t _Np,
typename _Tp>
1611 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1612 __vector_broadcast(_Tp __x)
1614 return __call_with_n_evaluations<_Np>(
1615 [](
auto... __xx) {
return __vector_type_t<_Tp, _Np>{__xx...}; },
1616 [&__x](int) {
return __x; });
1621 template <
typename _Tp,
size_t _Np,
typename _Gp,
size_t... _I>
1622 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1623 __generate_vector_impl(_Gp&& __gen, index_sequence<_I...>)
1625 return __vector_type_t<_Tp, _Np>{
1626 static_cast<_Tp
>(__gen(_SizeConstant<_I>()))...};
1629 template <
typename _V,
typename _VVT = _VectorTraits<_V>,
typename _Gp>
1630 _GLIBCXX_SIMD_INTRINSIC constexpr _V
1631 __generate_vector(_Gp&& __gen)
1633 if constexpr (__is_vector_type_v<_V>)
1634 return __generate_vector_impl<
typename _VVT::value_type,
1635 _VVT::_S_full_size>(
1636 static_cast<_Gp&&
>(__gen), make_index_sequence<_VVT::_S_full_size>());
1638 return __generate_vector_impl<
typename _VVT::value_type,
1639 _VVT::_S_partial_width>(
1640 static_cast<_Gp&&
>(__gen),
1641 make_index_sequence<_VVT::_S_partial_width>());
1644 template <
typename _Tp,
size_t _Np,
typename _Gp>
1645 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1646 __generate_vector(_Gp&& __gen)
1648 return __generate_vector_impl<_Tp, _Np>(
static_cast<_Gp&&
>(__gen),
1649 make_index_sequence<_Np>());
1654 template <
typename _TW>
1655 _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1656 __xor(_TW __a, _TW __b) noexcept
1658 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1660 using _Tp =
typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1661 _VectorTraitsImpl<_TW>>::value_type;
1662 if constexpr (is_floating_point_v<_Tp>)
1664 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1665 return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1666 ^ __vector_bitcast<_Ip>(__b));
1668 else if constexpr (__is_vector_type_v<_TW>)
1671 return __a._M_data ^ __b._M_data;
1679 template <
typename _TW>
1680 _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1681 __or(_TW __a, _TW __b) noexcept
1683 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1685 using _Tp =
typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1686 _VectorTraitsImpl<_TW>>::value_type;
1687 if constexpr (is_floating_point_v<_Tp>)
1689 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1690 return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1691 | __vector_bitcast<_Ip>(__b));
1693 else if constexpr (__is_vector_type_v<_TW>)
1696 return __a._M_data | __b._M_data;
1704 template <
typename _TW>
1705 _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1706 __and(_TW __a, _TW __b) noexcept
1708 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1710 using _Tp =
typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1711 _VectorTraitsImpl<_TW>>::value_type;
1712 if constexpr (is_floating_point_v<_Tp>)
1714 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1715 return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1716 & __vector_bitcast<_Ip>(__b));
1718 else if constexpr (__is_vector_type_v<_TW>)
1721 return __a._M_data & __b._M_data;
1729 #if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1730 static constexpr
struct
1732 _GLIBCXX_SIMD_INTRINSIC __v4sf
1733 operator()(__v4sf __a, __v4sf __b)
const noexcept
1734 {
return __builtin_ia32_andnps(__a, __b); }
1736 _GLIBCXX_SIMD_INTRINSIC __v2df
1737 operator()(__v2df __a, __v2df __b)
const noexcept
1738 {
return __builtin_ia32_andnpd(__a, __b); }
1740 _GLIBCXX_SIMD_INTRINSIC __v2di
1741 operator()(__v2di __a, __v2di __b)
const noexcept
1742 {
return __builtin_ia32_pandn128(__a, __b); }
1744 _GLIBCXX_SIMD_INTRINSIC __v8sf
1745 operator()(__v8sf __a, __v8sf __b)
const noexcept
1746 {
return __builtin_ia32_andnps256(__a, __b); }
1748 _GLIBCXX_SIMD_INTRINSIC __v4df
1749 operator()(__v4df __a, __v4df __b)
const noexcept
1750 {
return __builtin_ia32_andnpd256(__a, __b); }
1752 _GLIBCXX_SIMD_INTRINSIC __v4di
1753 operator()(__v4di __a, __v4di __b)
const noexcept
1755 if constexpr (__have_avx2)
1756 return __builtin_ia32_andnotsi256(__a, __b);
1758 return reinterpret_cast<__v4di
>(
1759 __builtin_ia32_andnpd256(
reinterpret_cast<__v4df
>(__a),
1760 reinterpret_cast<__v4df
>(__b)));
1763 _GLIBCXX_SIMD_INTRINSIC __v16sf
1764 operator()(__v16sf __a, __v16sf __b)
const noexcept
1766 if constexpr (__have_avx512dq)
1767 return _mm512_andnot_ps(__a, __b);
1769 return reinterpret_cast<__v16sf
>(
1770 _mm512_andnot_si512(
reinterpret_cast<__v8di
>(__a),
1771 reinterpret_cast<__v8di
>(__b)));
1774 _GLIBCXX_SIMD_INTRINSIC __v8df
1775 operator()(__v8df __a, __v8df __b)
const noexcept
1777 if constexpr (__have_avx512dq)
1778 return _mm512_andnot_pd(__a, __b);
1780 return reinterpret_cast<__v8df
>(
1781 _mm512_andnot_si512(
reinterpret_cast<__v8di
>(__a),
1782 reinterpret_cast<__v8di
>(__b)));
1785 _GLIBCXX_SIMD_INTRINSIC __v8di
1786 operator()(__v8di __a, __v8di __b)
const noexcept
1787 {
return _mm512_andnot_si512(__a, __b); }
1791 template <
typename _TW>
1792 _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1793 __andnot(_TW __a, _TW __b) noexcept
1795 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1797 using _TVT = conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1798 _VectorTraitsImpl<_TW>>;
1799 using _Tp =
typename _TVT::value_type;
1800 #if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1801 if constexpr (
sizeof(_TW) >= 16)
1803 const auto __ai = __to_intrin(__a);
1804 const auto __bi = __to_intrin(__b);
1805 if (!__builtin_is_constant_evaluated()
1806 && !(__builtin_constant_p(__ai) && __builtin_constant_p(__bi)))
1808 const auto __r = _S_x86_andnot(__ai, __bi);
1809 if constexpr (is_convertible_v<decltype(__r), _TW>)
1812 return reinterpret_cast<typename _TVT::type>(__r);
1816 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1817 return __vector_bitcast<_Tp>(~__vector_bitcast<_Ip>(__a)
1818 & __vector_bitcast<_Ip>(__b));
1826 template <
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1827 _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
1828 __not(_Tp __a) noexcept
1830 if constexpr (is_floating_point_v<typename _TVT::value_type>)
1831 return reinterpret_cast<typename _TVT::type
>(
1832 ~__vector_bitcast<unsigned>(__a));
1839 template <
typename _Tp,
typename _TVT = _VectorTraits<_Tp>,
1840 typename _R = __vector_type_t<
typename _TVT::value_type,
1841 _TVT::_S_full_size * 2>>
1843 __concat(_Tp a_, _Tp b_)
1845 #ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_1
1847 = conditional_t<is_floating_point_v<typename _TVT::value_type>, double,
1849 long long,
typename _TVT::value_type>>;
1850 constexpr
int input_width =
sizeof(_Tp) /
sizeof(_W);
1851 const auto __a = __vector_bitcast<_W>(a_);
1852 const auto __b = __vector_bitcast<_W>(b_);
1853 using _Up = __vector_type_t<_W,
sizeof(_R) /
sizeof(_W)>;
1855 constexpr
int input_width = _TVT::_S_full_size;
1856 const _Tp& __a = a_;
1857 const _Tp& __b = b_;
1860 if constexpr (input_width == 2)
1861 return reinterpret_cast<_R>(_Up{__a[0], __a[1], __b[0], __b[1]});
1862 else if constexpr (input_width == 4)
1863 return reinterpret_cast<_R>(
1864 _Up{__a[0], __a[1], __a[2], __a[3], __b[0], __b[1], __b[2], __b[3]});
1865 else if constexpr (input_width == 8)
1866 return reinterpret_cast<_R>(
1867 _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6], __a[7],
1868 __b[0], __b[1], __b[2], __b[3], __b[4], __b[5], __b[6], __b[7]});
1869 else if constexpr (input_width == 16)
1870 return reinterpret_cast<_R>(
1871 _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6],
1872 __a[7], __a[8], __a[9], __a[10], __a[11], __a[12], __a[13],
1873 __a[14], __a[15], __b[0], __b[1], __b[2], __b[3], __b[4],
1874 __b[5], __b[6], __b[7], __b[8], __b[9], __b[10], __b[11],
1875 __b[12], __b[13], __b[14], __b[15]});
1876 else if constexpr (input_width == 32)
1877 return reinterpret_cast<_R>(
1878 _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6],
1879 __a[7], __a[8], __a[9], __a[10], __a[11], __a[12], __a[13],
1880 __a[14], __a[15], __a[16], __a[17], __a[18], __a[19], __a[20],
1881 __a[21], __a[22], __a[23], __a[24], __a[25], __a[26], __a[27],
1882 __a[28], __a[29], __a[30], __a[31], __b[0], __b[1], __b[2],
1883 __b[3], __b[4], __b[5], __b[6], __b[7], __b[8], __b[9],
1884 __b[10], __b[11], __b[12], __b[13], __b[14], __b[15], __b[16],
1885 __b[17], __b[18], __b[19], __b[20], __b[21], __b[22], __b[23],
1886 __b[24], __b[25], __b[26], __b[27], __b[28], __b[29], __b[30],
1892 template <
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1893 struct _ZeroExtendProxy
1895 using value_type =
typename _TVT::value_type;
1896 static constexpr
size_t _Np = _TVT::_S_full_size;
1899 template <
typename _To,
typename _ToVT = _VectorTraits<_To>,
1901 = enable_if_t<is_same_v<
typename _ToVT::value_type, value_type>>>
1902 _GLIBCXX_SIMD_INTRINSIC
operator _To()
const
1904 constexpr
size_t _ToN = _ToVT::_S_full_size;
1905 if constexpr (_ToN == _Np)
1907 else if constexpr (_ToN == 2 * _Np)
1909 #ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_3
1910 if constexpr (__have_avx && _TVT::template _S_is<float, 4>)
1911 return __vector_bitcast<value_type>(
1912 _mm256_insertf128_ps(__m256(), __x, 0));
1913 else if constexpr (__have_avx && _TVT::template _S_is<double, 2>)
1914 return __vector_bitcast<value_type>(
1915 _mm256_insertf128_pd(__m256d(), __x, 0));
1916 else if constexpr (__have_avx2 && _Np *
sizeof(value_type) == 16)
1917 return __vector_bitcast<value_type>(
1918 _mm256_insertf128_si256(__m256i(), __to_intrin(__x), 0));
1919 else if constexpr (__have_avx512f && _TVT::template _S_is<
float, 8>)
1921 if constexpr (__have_avx512dq)
1922 return __vector_bitcast<value_type>(
1923 _mm512_insertf32x8(__m512(), __x, 0));
1925 return reinterpret_cast<__m512
>(
1926 _mm512_insertf64x4(__m512d(),
1927 reinterpret_cast<__m256d
>(__x), 0));
1929 else if constexpr (__have_avx512f
1930 && _TVT::template _S_is<double, 4>)
1931 return __vector_bitcast<value_type>(
1932 _mm512_insertf64x4(__m512d(), __x, 0));
1933 else if constexpr (__have_avx512f && _Np *
sizeof(value_type) == 32)
1934 return __vector_bitcast<value_type>(
1935 _mm512_inserti64x4(__m512i(), __to_intrin(__x), 0));
1937 return __concat(__x, _Tp());
1939 else if constexpr (_ToN == 4 * _Np)
1941 #ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_3
1942 if constexpr (__have_avx512dq && _TVT::template _S_is<double, 2>)
1944 return __vector_bitcast<value_type>(
1945 _mm512_insertf64x2(__m512d(), __x, 0));
1947 else if constexpr (__have_avx512f
1948 && is_floating_point_v<value_type>)
1950 return __vector_bitcast<value_type>(
1951 _mm512_insertf32x4(__m512(),
reinterpret_cast<__m128
>(__x),
1954 else if constexpr (__have_avx512f && _Np *
sizeof(value_type) == 16)
1956 return __vector_bitcast<value_type>(
1957 _mm512_inserti32x4(__m512i(), __to_intrin(__x), 0));
1960 return __concat(__concat(__x, _Tp()),
1961 __vector_type_t<value_type, _Np * 2>());
1963 else if constexpr (_ToN == 8 * _Np)
1964 return __concat(
operator __vector_type_t<value_type, _Np * 4>(),
1965 __vector_type_t<value_type, _Np * 4>());
1966 else if constexpr (_ToN == 16 * _Np)
1967 return __concat(
operator __vector_type_t<value_type, _Np * 8>(),
1968 __vector_type_t<value_type, _Np * 8>());
1970 __assert_unreachable<_Tp>();
1974 template <
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1975 _GLIBCXX_SIMD_INTRINSIC _ZeroExtendProxy<_Tp, _TVT>
1976 __zero_extend(_Tp __x)
1981 template <
int _Offset,
1984 typename _TVT = _VectorTraits<_Tp>,
1985 typename _R = __vector_type_t<
typename _TVT::value_type,
1986 _TVT::_S_full_size / _SplitBy>>
1987 _GLIBCXX_SIMD_INTRINSIC constexpr _R
1990 using value_type =
typename _TVT::value_type;
1991 #if _GLIBCXX_SIMD_X86INTRIN
1992 if constexpr (
sizeof(_Tp) == 64 && _SplitBy == 4 && _Offset > 0)
1994 if constexpr (__have_avx512dq && is_same_v<double, value_type>)
1995 return _mm512_extractf64x2_pd(__to_intrin(__in), _Offset);
1996 else if constexpr (is_floating_point_v<value_type>)
1997 return __vector_bitcast<value_type>(
1998 _mm512_extractf32x4_ps(__intrin_bitcast<__m512>(__in), _Offset));
2000 return reinterpret_cast<_R
>(
2001 _mm512_extracti32x4_epi32(__intrin_bitcast<__m512i>(__in),
2007 #ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_1
2009 is_floating_point_v<value_type>, double,
2011 static_assert(
sizeof(_R) %
sizeof(_W) == 0);
2012 constexpr
int __return_width =
sizeof(_R) /
sizeof(_W);
2013 using _Up = __vector_type_t<_W, __return_width>;
2014 const auto __x = __vector_bitcast<_W>(__in);
2016 constexpr
int __return_width = _TVT::_S_full_size / _SplitBy;
2018 const __vector_type_t<value_type, _TVT::_S_full_size>& __x
2021 constexpr
int _O = _Offset * __return_width;
2022 return __call_with_subscripts<__return_width, _O>(
2023 __x, [](
auto... __entries) {
2024 return reinterpret_cast<_R
>(_Up{__entries...});
2031 template <
typename _Tp,
2033 = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2034 _GLIBCXX_SIMD_INTRINSIC constexpr _R
2038 __builtin_memcpy(&__r, &__x, 8);
2042 template <
typename _Tp,
2044 = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2045 _GLIBCXX_SIMD_INTRINSIC constexpr _R
2048 static_assert(
sizeof(_Tp) == 16,
"use __hi64z if you meant it");
2050 __builtin_memcpy(&__r,
reinterpret_cast<const char*
>(&__x) + 8, 8);
2054 template <
typename _Tp,
2056 = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2057 _GLIBCXX_SIMD_INTRINSIC constexpr _R
2058 __hi64z([[maybe_unused]] _Tp __x)
2061 if constexpr (
sizeof(_Tp) == 16)
2062 __builtin_memcpy(&__r, reinterpret_cast<const
char*>(&__x) + 8, 8);
2068 template <typename _Tp>
2069 _GLIBCXX_SIMD_INTRINSIC constexpr auto
2071 {
return __extract<0,
sizeof(_Tp) / 16>(__x); }
2073 template <
typename _Tp>
2074 _GLIBCXX_SIMD_INTRINSIC constexpr
auto
2077 static_assert(
sizeof(__x) == 32);
2078 return __extract<1, 2>(__x);
2083 template <
typename _Tp>
2084 _GLIBCXX_SIMD_INTRINSIC constexpr
auto
2087 static_assert(
sizeof(__x) == 64);
2088 return __extract<0, 2>(__x);
2091 template <
typename _Tp>
2092 _GLIBCXX_SIMD_INTRINSIC constexpr
auto
2095 static_assert(
sizeof(__x) == 64);
2096 return __extract<1, 2>(__x);
2101 template <
typename _Tp>
2104 static_assert(__is_vector_type_v<_Tp>);
2108 template <
typename _Up,
typename _UVT = _VectorTraits<_Up>>
2109 _GLIBCXX_SIMD_INTRINSIC constexpr
operator _Up()
const
2110 {
return __intrin_bitcast<typename _UVT::type>(__x); }
2113 template <
typename _Tp>
2114 _GLIBCXX_SIMD_INTRINSIC constexpr _AutoCast<_Tp>
2115 __auto_bitcast(
const _Tp& __x)
2118 template <
typename _Tp,
size_t _Np>
2119 _GLIBCXX_SIMD_INTRINSIC constexpr
2120 _AutoCast<typename _SimdWrapper<_Tp, _Np>::_BuiltinType>
2121 __auto_bitcast(
const _SimdWrapper<_Tp, _Np>& __x)
2122 {
return {__x._M_data}; }
2127 #if _GLIBCXX_SIMD_HAVE_SSE_ABI
2129 #if _GLIBCXX_SIMD_HAVE_AVX512F && _GLIBCXX_SIMD_X86INTRIN
2130 template <
size_t _Size>
2131 struct __bool_storage_member_type
2133 static_assert((_Size & (_Size - 1)) != 0,
2134 "This trait may only be used for non-power-of-2 sizes. "
2135 "Power-of-2 sizes must be specialized.");
2137 typename __bool_storage_member_type<std::__bit_ceil(_Size)>::type;
2141 struct __bool_storage_member_type<1> {
using type = bool; };
2144 struct __bool_storage_member_type<2> {
using type = __mmask8; };
2147 struct __bool_storage_member_type<4> {
using type = __mmask8; };
2150 struct __bool_storage_member_type<8> {
using type = __mmask8; };
2153 struct __bool_storage_member_type<16> {
using type = __mmask16; };
2156 struct __bool_storage_member_type<32> {
using type = __mmask32; };
2159 struct __bool_storage_member_type<64> {
using type = __mmask64; };
2165 #if _GLIBCXX_SIMD_HAVE_SSE
2166 template <
typename _Tp,
size_t _Bytes>
2167 struct __intrinsic_type<_Tp, _Bytes,
2168 enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 64>>
2170 static_assert(!is_same_v<_Tp, long double>,
2171 "no __intrinsic_type support for long double on x86");
2173 static constexpr
size_t _S_VBytes = _Bytes <= 16 ? 16
2177 using type [[__gnu__::__vector_size__(_S_VBytes)]]
2178 = conditional_t<is_integral_v<_Tp>,
long long int, _Tp>;
2185 #if _GLIBCXX_SIMD_HAVE_NEON
2186 template <
typename _Tp,
size_t _Bytes>
2187 struct __intrinsic_type<_Tp, _Bytes,
2188 enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 16>>
2190 static constexpr
int _S_VBytes = _Bytes <= 8 ? 8 : 16;
2191 using _Ip = __int_for_sizeof_t<_Tp>;
2193 is_floating_point_v<_Tp>, _Tp,
2194 conditional_t<is_unsigned_v<_Tp>, make_unsigned_t<_Ip>, _Ip>>;
2195 using type [[__gnu__::__vector_size__(_S_VBytes)]] = _Up;
2202 template <
typename _Tp>
2203 struct __intrinsic_type_impl;
2205 #define _GLIBCXX_SIMD_PPC_INTRIN(_Tp) \
2207 struct __intrinsic_type_impl<_Tp> { using type = __vector _Tp; }
2208 _GLIBCXX_SIMD_PPC_INTRIN(
float);
2209 _GLIBCXX_SIMD_PPC_INTRIN(
double);
2210 _GLIBCXX_SIMD_PPC_INTRIN(
signed char);
2211 _GLIBCXX_SIMD_PPC_INTRIN(
unsigned char);
2212 _GLIBCXX_SIMD_PPC_INTRIN(
signed short);
2213 _GLIBCXX_SIMD_PPC_INTRIN(
unsigned short);
2214 _GLIBCXX_SIMD_PPC_INTRIN(
signed int);
2215 _GLIBCXX_SIMD_PPC_INTRIN(
unsigned int);
2216 _GLIBCXX_SIMD_PPC_INTRIN(
signed long);
2217 _GLIBCXX_SIMD_PPC_INTRIN(
unsigned long);
2218 _GLIBCXX_SIMD_PPC_INTRIN(
signed long long);
2219 _GLIBCXX_SIMD_PPC_INTRIN(
unsigned long long);
2220 #undef _GLIBCXX_SIMD_PPC_INTRIN
2222 template <
typename _Tp,
size_t _Bytes>
2223 struct __intrinsic_type<_Tp, _Bytes,
2224 enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 16>>
2226 static_assert(!is_same_v<_Tp, long double>,
2227 "no __intrinsic_type support for long double on PPC");
2229 static_assert(!is_same_v<_Tp, double>,
2230 "no __intrinsic_type support for double on PPC w/o VSX");
2232 #ifndef __POWER8_VECTOR__
2234 !(is_integral_v<_Tp> &&
sizeof(_Tp) > 4),
2235 "no __intrinsic_type support for integers larger than 4 Bytes "
2236 "on PPC w/o POWER8 vectors");
2239 is_floating_point_v<_Tp>, _Tp, __int_for_sizeof_t<_Tp>>>::type;
2245 template <
size_t _W
idth>
2246 struct _SimdWrapper<bool, _Width,
2247 void_t<typename __bool_storage_member_type<_Width>::type>>
2249 using _BuiltinType =
typename __bool_storage_member_type<_Width>::type;
2250 using value_type = bool;
2252 static constexpr
size_t _S_full_size =
sizeof(_BuiltinType) * __CHAR_BIT__;
2254 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<bool, _S_full_size>
2255 __as_full_vector()
const {
return _M_data; }
2257 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper() =
default;
2258 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper(_BuiltinType __k)
2261 _GLIBCXX_SIMD_INTRINSIC
operator const _BuiltinType&()
const
2264 _GLIBCXX_SIMD_INTRINSIC
operator _BuiltinType&()
2267 _GLIBCXX_SIMD_INTRINSIC _BuiltinType __intrin()
const
2270 _GLIBCXX_SIMD_INTRINSIC constexpr value_type operator[](
size_t __i)
const
2271 {
return _M_data & (_BuiltinType(1) << __i); }
2273 template <
size_t __i>
2274 _GLIBCXX_SIMD_INTRINSIC constexpr value_type
2275 operator[](_SizeConstant<__i>)
const
2276 {
return _M_data & (_BuiltinType(1) << __i); }
2278 _GLIBCXX_SIMD_INTRINSIC constexpr
void _M_set(
size_t __i, value_type __x)
2281 _M_data |= (_BuiltinType(1) << __i);
2283 _M_data &= ~(_BuiltinType(1) << __i);
2286 _GLIBCXX_SIMD_INTRINSIC
2287 constexpr
bool _M_is_constprop()
const
2288 {
return __builtin_constant_p(_M_data); }
2290 _GLIBCXX_SIMD_INTRINSIC constexpr
bool _M_is_constprop_none_of()
const
2292 if (__builtin_constant_p(_M_data))
2294 constexpr
int __nbits =
sizeof(_BuiltinType) * __CHAR_BIT__;
2295 constexpr _BuiltinType __active_mask
2296 = ~_BuiltinType() >> (__nbits - _Width);
2297 return (_M_data & __active_mask) == 0;
2302 _GLIBCXX_SIMD_INTRINSIC constexpr
bool _M_is_constprop_all_of()
const
2304 if (__builtin_constant_p(_M_data))
2306 constexpr
int __nbits =
sizeof(_BuiltinType) * __CHAR_BIT__;
2307 constexpr _BuiltinType __active_mask
2308 = ~_BuiltinType() >> (__nbits - _Width);
2309 return (_M_data & __active_mask) == __active_mask;
2314 _BuiltinType _M_data;
2318 template <
bool _MustZeroInitPadding,
typename _BuiltinType>
2319 struct _SimdWrapperBase;
2321 template <
typename _BuiltinType>
2322 struct _SimdWrapperBase<false, _BuiltinType>
2324 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapperBase() =
default;
2325 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapperBase(_BuiltinType __init)
2329 _BuiltinType _M_data;
2332 template <
typename _BuiltinType>
2333 struct _SimdWrapperBase<true, _BuiltinType>
2336 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapperBase() : _M_data() {}
2337 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapperBase(_BuiltinType __init)
2341 _BuiltinType _M_data;
2346 template <
typename _Tp,
size_t _W
idth>
2347 struct _SimdWrapper<
2349 void_t<__vector_type_t<_Tp, _Width>, __intrinsic_type_t<_Tp, _Width>>>
2350 : _SimdWrapperBase<__has_iec559_behavior<__signaling_NaN, _Tp>::value
2351 && sizeof(_Tp) * _Width
2352 == sizeof(__vector_type_t<_Tp, _Width>),
2353 __vector_type_t<_Tp, _Width>>
2356 = _SimdWrapperBase<__has_iec559_behavior<__signaling_NaN, _Tp>::value
2357 &&
sizeof(_Tp) * _Width
2358 ==
sizeof(__vector_type_t<_Tp, _Width>),
2359 __vector_type_t<_Tp, _Width>>;
2361 static_assert(__is_vectorizable_v<_Tp>);
2362 static_assert(_Width >= 2);
2364 using _BuiltinType = __vector_type_t<_Tp, _Width>;
2365 using value_type = _Tp;
2367 static inline constexpr
size_t _S_full_size
2368 =
sizeof(_BuiltinType) /
sizeof(value_type);
2369 static inline constexpr
int _S_size = _Width;
2370 static inline constexpr
bool _S_is_partial = _S_full_size != _S_size;
2372 using _Base::_M_data;
2374 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<_Tp, _S_full_size>
2375 __as_full_vector()
const
2378 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper(initializer_list<_Tp> __init)
2379 : _Base(__generate_from_n_evaluations<_Width, _BuiltinType>(
2380 [&](auto __i) {
return __init.begin()[__i.value]; })) {}
2382 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper() =
default;
2383 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper(
const _SimdWrapper&)
2385 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper(_SimdWrapper&&) =
default;
2387 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper&
2388 operator=(
const _SimdWrapper&) =
default;
2389 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper&
2392 template <
typename _V,
typename =
enable_if_t<disjunction_v<
2393 is_same<_V, __vector_type_t<_Tp, _Width>>,
2394 is_same<_V, __intrinsic_type_t<_Tp, _Width>>>>>
2395 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper(_V __x)
2397 : _Base(__vector_bitcast<_Tp, _Width>(__x)) {}
2399 template <
typename... _As,
2400 typename =
enable_if_t<((is_same_v<simd_abi::scalar, _As> && ...)
2401 &&
sizeof...(_As) <= _Width)>>
2402 _GLIBCXX_SIMD_INTRINSIC constexpr
2403 operator _SimdTuple<_Tp, _As...>()
const
2405 const auto& dd = _M_data;
2406 return __generate_from_n_evaluations<
sizeof...(_As),
2407 _SimdTuple<_Tp, _As...>>([&](
2408 auto __i) constexpr {
return dd[int(__i)]; });
2411 _GLIBCXX_SIMD_INTRINSIC constexpr
operator const _BuiltinType&()
const
2414 _GLIBCXX_SIMD_INTRINSIC constexpr
operator _BuiltinType&()
2417 _GLIBCXX_SIMD_INTRINSIC constexpr _Tp operator[](
size_t __i)
const
2418 {
return _M_data[__i]; }
2420 template <
size_t __i>
2421 _GLIBCXX_SIMD_INTRINSIC constexpr _Tp operator[](_SizeConstant<__i>)
const
2422 {
return _M_data[__i]; }
2424 _GLIBCXX_SIMD_INTRINSIC constexpr
void _M_set(
size_t __i, _Tp __x)
2425 { _M_data[__i] = __x; }
2427 _GLIBCXX_SIMD_INTRINSIC
2428 constexpr
bool _M_is_constprop()
const
2429 {
return __builtin_constant_p(_M_data); }
2431 _GLIBCXX_SIMD_INTRINSIC constexpr
bool _M_is_constprop_none_of()
const
2433 if (__builtin_constant_p(_M_data))
2436 if constexpr (is_floating_point_v<_Tp>)
2438 using _Ip = __int_for_sizeof_t<_Tp>;
2439 const auto __intdata = __vector_bitcast<_Ip>(_M_data);
2440 __execute_n_times<_Width>(
2441 [&](
auto __i) { __r &= __intdata[__i.value] == _Ip(); });
2444 __execute_n_times<_Width>(
2445 [&](
auto __i) { __r &= _M_data[__i.value] == _Tp(); });
2451 _GLIBCXX_SIMD_INTRINSIC constexpr
bool _M_is_constprop_all_of()
const
2453 if (__builtin_constant_p(_M_data))
2456 if constexpr (is_floating_point_v<_Tp>)
2458 using _Ip = __int_for_sizeof_t<_Tp>;
2459 const auto __intdata = __vector_bitcast<_Ip>(_M_data);
2460 __execute_n_times<_Width>(
2461 [&](
auto __i) { __r &= __intdata[__i.value] == ~_Ip(); });
2464 __execute_n_times<_Width>(
2465 [&](
auto __i) { __r &= _M_data[__i.value] == ~_Tp(); });
2475 template <
typename _Tp>
2477 __vectorized_sizeof()
2479 if constexpr (!__is_vectorizable_v<_Tp>)
2482 if constexpr (
sizeof(_Tp) <= 8)
2485 if constexpr (__have_avx512bw)
2487 if constexpr (__have_avx512f &&
sizeof(_Tp) >= 4)
2489 if constexpr (__have_avx2)
2491 if constexpr (__have_avx && is_floating_point_v<_Tp>)
2493 if constexpr (__have_sse2)
2495 if constexpr (__have_sse && is_same_v<_Tp,
float>)
2505 if constexpr (__have_power8vec
2506 || (__have_power_vmx && (sizeof(_Tp) < 8))
2507 || (__have_power_vsx && is_floating_point_v<_Tp>) )
2511 if constexpr (__have_neon_a64
2512 || (__have_neon_a32 && !is_same_v<_Tp,
double>) )
2514 if constexpr (__have_neon
2519 && (__support_neon_float || !is_floating_point_v<_Tp>))
2527 namespace simd_abi {
2529 template <
typename _Tp>
2530 inline constexpr
int max_fixed_size
2531 = (__have_avx512bw &&
sizeof(_Tp) == 1) ? 64 : 32;
2534 #if defined __x86_64__ || defined __aarch64__
2535 template <
typename _Tp>
2536 using compatible =
conditional_t<(
sizeof(_Tp) <= 8), _VecBuiltin<16>, scalar>;
2537 #elif defined __ARM_NEON
2540 template <
typename _Tp>
2543 && (__support_neon_float || !is_floating_point_v<_Tp>)),
2544 _VecBuiltin<16>, scalar>;
2547 using compatible = scalar;
2552 template <
typename _Tp>
2554 __determine_native_abi()
2556 constexpr
size_t __bytes = __vectorized_sizeof<_Tp>();
2557 if constexpr (__bytes ==
sizeof(_Tp))
2558 return static_cast<scalar*
>(
nullptr);
2559 else if constexpr (__have_avx512vl || (__have_avx512f && __bytes == 64))
2560 return static_cast<_VecBltnBtmsk<__bytes>*>(
nullptr);
2562 return static_cast<_VecBuiltin<__bytes>*>(
nullptr);
2565 template <typename _Tp, typename =
enable_if_t<__is_vectorizable_v<_Tp>>>
2570 #if defined _GLIBCXX_SIMD_DEFAULT_ABI
2571 template <
typename _Tp>
2572 using __default_abi = _GLIBCXX_SIMD_DEFAULT_ABI<_Tp>;
2574 template <
typename _Tp>
2575 using __default_abi = compatible<_Tp>;
2583 template <
typename _Tp,
typename =
void_t<>>
2586 template <
typename _Tp>
2587 struct is_abi_tag<_Tp,
void_t<typename _Tp::_IsValidAbiTag>>
2588 :
public _Tp::_IsValidAbiTag {};
2590 template <
typename _Tp>
2591 inline constexpr
bool is_abi_tag_v = is_abi_tag<_Tp>::value;
2594 template <
typename _Tp>
2597 template <
typename _Tp>
2598 inline constexpr
bool is_simd_v = is_simd<_Tp>::value;
2600 template <
typename _Tp>
2603 template <
typename _Tp>
2604 inline constexpr
bool is_simd_mask_v = is_simd_mask<_Tp>::value;
2607 template <
typename _Tp,
typename _Abi,
typename =
void>
2608 struct __simd_size_impl {};
2610 template <
typename _Tp,
typename _Abi>
2611 struct __simd_size_impl<
2613 enable_if_t<conjunction_v<__is_vectorizable<_Tp>, is_abi_tag<_Abi>>>>
2614 : _SizeConstant<_Abi::template _S_size<_Tp>> {};
2616 template <
typename _Tp,
typename _Abi = simd_abi::__default_abi<_Tp>>
2617 struct simd_size : __simd_size_impl<_Tp, _Abi> {};
2619 template <
typename _Tp,
typename _Abi = simd_abi::__default_abi<_Tp>>
2620 inline constexpr
size_t simd_size_v = simd_size<_Tp, _Abi>::value;
2623 template <
typename _Tp,
size_t _Np,
typename =
void>
2624 struct __deduce_impl;
2626 namespace simd_abi {
2635 template <
typename _Tp,
size_t _Np,
typename...>
2638 template <
typename _Tp,
size_t _Np,
typename... _Abis>
2639 using deduce_t =
typename deduce<_Tp, _Np, _Abis...>::type;
2644 template <
typename _Tp,
typename _V,
typename =
void>
2647 template <
typename _Tp,
typename _Up,
typename _Abi>
2649 _Tp, simd<_Up, _Abi>,
2650 void_t<simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>>
2653 = simd<_Tp, simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>;
2656 template <
typename _Tp,
typename _Up,
typename _Abi>
2658 _Tp, simd_mask<_Up, _Abi>,
2659 void_t<simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>>
2662 = simd_mask<_Tp, simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>;
2665 template <
typename _Tp,
typename _V>
2666 using rebind_simd_t =
typename rebind_simd<_Tp, _V>::type;
2669 template <
int _Np,
typename _V,
typename =
void>
2672 template <
int _Np,
typename _Tp,
typename _Abi>
2673 struct resize_simd<_Np, simd<_Tp, _Abi>,
2674 void_t<simd_abi::deduce_t<_Tp, _Np, _Abi>>>
2675 {
using type = simd<_Tp, simd_abi::deduce_t<_Tp, _Np, _Abi>>; };
2677 template <
int _Np,
typename _Tp,
typename _Abi>
2678 struct resize_simd<_Np, simd_mask<_Tp, _Abi>,
2679 void_t<simd_abi::deduce_t<_Tp, _Np, _Abi>>>
2680 {
using type = simd_mask<_Tp, simd_abi::deduce_t<_Tp, _Np, _Abi>>; };
2682 template <
int _Np,
typename _V>
2683 using resize_simd_t =
typename resize_simd<_Np, _V>::type;
2687 template <
typename _Tp,
typename _Up =
typename _Tp::value_type>
2688 struct memory_alignment
2689 :
public _SizeConstant<vector_aligned_tag::_S_alignment<_Tp, _Up>> {};
2691 template <
typename _Tp,
typename _Up =
typename _Tp::value_type>
2692 inline constexpr
size_t memory_alignment_v = memory_alignment<_Tp, _Up>::value;
2695 template <
typename _Tp,
typename _Abi = simd_abi::__default_abi<_Tp>>
2698 template <
typename _Tp,
typename _Abi>
2699 struct is_simd<simd<_Tp, _Abi>> :
public true_type {};
2701 template <
typename _Tp>
2702 using native_simd = simd<_Tp, simd_abi::native<_Tp>>;
2704 template <
typename _Tp,
int _Np>
2705 using fixed_size_simd = simd<_Tp, simd_abi::fixed_size<_Np>>;
2707 template <
typename _Tp,
size_t _Np>
2708 using __deduced_simd = simd<_Tp, simd_abi::deduce_t<_Tp, _Np>>;
2711 template <
typename _Tp,
typename _Abi = simd_abi::__default_abi<_Tp>>
2714 template <
typename _Tp,
typename _Abi>
2715 struct is_simd_mask<simd_mask<_Tp, _Abi>> :
public true_type {};
2717 template <
typename _Tp>
2718 using native_simd_mask = simd_mask<_Tp, simd_abi::native<_Tp>>;
2720 template <
typename _Tp,
int _Np>
2721 using fixed_size_simd_mask = simd_mask<_Tp, simd_abi::fixed_size<_Np>>;
2723 template <
typename _Tp,
size_t _Np>
2724 using __deduced_simd_mask = simd_mask<_Tp, simd_abi::deduce_t<_Tp, _Np>>;
2728 template <
typename _Tp,
typename _Up,
typename _Ap,
bool = is_simd_v<_Tp>,
2730 struct __static_simd_cast_return_type;
2732 template <
typename _Tp,
typename _A0,
typename _Up,
typename _Ap>
2733 struct __static_simd_cast_return_type<simd_mask<_Tp, _A0>, _Up, _Ap, false,
2735 : __static_simd_cast_return_type<simd<_Tp, _A0>, _Up, _Ap> {};
2737 template <
typename _Tp,
typename _Up,
typename _Ap>
2738 struct __static_simd_cast_return_type<
2739 _Tp, _Up, _Ap, true,
enable_if_t<_Tp::
size() == simd_size_v<_Up, _Ap>>>
2740 {
using type = _Tp; };
2742 template <
typename _Tp,
typename _Ap>
2743 struct __static_simd_cast_return_type<_Tp, _Tp, _Ap, false,
2744 #ifdef _GLIBCXX_SIMD_FIX_P2TS_ISSUE66
2750 {
using type = simd<_Tp, _Ap>; };
2752 template <
typename _Tp,
typename =
void>
2753 struct __safe_make_signed {
using type = _Tp;};
2755 template <
typename _Tp>
2756 struct __safe_make_signed<_Tp,
enable_if_t<is_integral_v<_Tp>>>
2759 using type = make_signed_t<make_unsigned_t<_Tp>>;
2762 template <
typename _Tp>
2763 using safe_make_signed_t =
typename __safe_make_signed<_Tp>::type;
2765 template <
typename _Tp,
typename _Up,
typename _Ap>
2766 struct __static_simd_cast_return_type<_Tp, _Up, _Ap, false,
2767 #ifdef _GLIBCXX_SIMD_FIX_P2TS_ISSUE66
2775 (is_integral_v<_Up> && is_integral_v<_Tp> &&
2776 #ifndef _GLIBCXX_SIMD_FIX_P2TS_ISSUE65
2777 is_signed_v<_Up> != is_signed_v<_Tp> &&
2779 is_same_v<safe_make_signed_t<_Up>, safe_make_signed_t<_Tp>>),
2780 simd<_Tp, _Ap>, fixed_size_simd<_Tp, simd_size_v<_Up, _Ap>>>;
2783 template <
typename _Tp,
typename _Up,
typename _Ap,
2785 =
typename __static_simd_cast_return_type<_Tp, _Up, _Ap>::type>
2786 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _R
2787 static_simd_cast(
const simd<_Up, _Ap>& __x)
2789 if constexpr (is_same<_R, simd<_Up, _Ap>>::value)
2793 _SimdConverter<_Up, _Ap, typename _R::value_type, typename _R::abi_type>
2795 return _R(__private_init, __c(__data(__x)));
2799 namespace __proposed {
2800 template <
typename _Tp,
typename _Up,
typename _Ap,
2802 =
typename __static_simd_cast_return_type<_Tp, _Up, _Ap>::type>
2803 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
typename _R::mask_type
2804 static_simd_cast(
const simd_mask<_Up, _Ap>& __x)
2806 using _RM =
typename _R::mask_type;
2807 return {__private_init, _RM::abi_type::_MaskImpl::template _S_convert<
2808 typename _RM::simd_type::value_type>(__x)};
2813 template <
typename _Tp,
typename _Up,
typename _Ap,
2814 typename _To = __value_type_or_identity_t<_Tp>>
2815 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
auto
2816 simd_cast(
const simd<_ValuePreserving<_Up, _To>, _Ap>& __x)
2817 -> decltype(static_simd_cast<_Tp>(__x))
2818 {
return static_simd_cast<_Tp>(__x); }
2820 namespace __proposed {
2821 template <
typename _Tp,
typename _Up,
typename _Ap,
2822 typename _To = __value_type_or_identity_t<_Tp>>
2823 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
auto
2824 simd_cast(
const simd_mask<_ValuePreserving<_Up, _To>, _Ap>& __x)
2825 -> decltype(static_simd_cast<_Tp>(__x))
2826 {
return static_simd_cast<_Tp>(__x); }
2831 namespace __proposed {
2858 template <
typename _Tp,
typename _Up,
typename _Ap>
2859 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
enable_if_t<
2860 conjunction_v<is_simd<_Tp>, is_same<typename _Tp::value_type, _Up>>, _Tp>
2861 resizing_simd_cast(
const simd<_Up, _Ap>& __x)
2863 if constexpr (is_same_v<typename _Tp::abi_type, _Ap>)
2865 else if constexpr (simd_size_v<_Up, _Ap> == 1)
2873 else if constexpr (sizeof(_Tp) == sizeof(__x)
2874 && !__is_fixed_size_abi_v<_Ap>)
2875 return {__private_init,
2876 __vector_bitcast<typename _Tp::value_type, _Tp::size()>(
2877 _Ap::_S_masked(__data(__x))._M_data)};
2881 __builtin_memcpy(&__data(__r), &__data(__x),
2888 template <
typename _Tp,
typename _Up,
typename _Ap>
2889 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
2890 enable_if_t<is_simd_mask_v<_Tp>, _Tp>
2891 resizing_simd_cast(
const simd_mask<_Up, _Ap>& __x)
2893 return {__private_init, _Tp::abi_type::_MaskImpl::template _S_convert<
2894 typename _Tp::simd_type::value_type>(__x)};
2900 template <
typename _Tp,
int _Np>
2901 _GLIBCXX_SIMD_INTRINSIC fixed_size_simd<_Tp, _Np>
2902 to_fixed_size(
const fixed_size_simd<_Tp, _Np>& __x)
2905 template <
typename _Tp,
int _Np>
2906 _GLIBCXX_SIMD_INTRINSIC fixed_size_simd_mask<_Tp, _Np>
2907 to_fixed_size(
const fixed_size_simd_mask<_Tp, _Np>& __x)
2910 template <
typename _Tp,
typename _Ap>
2911 _GLIBCXX_SIMD_INTRINSIC
auto
2912 to_fixed_size(
const simd<_Tp, _Ap>& __x)
2914 return simd<_Tp, simd_abi::fixed_size<simd_size_v<_Tp, _Ap>>>([&__x](
2915 auto __i) constexpr {
return __x[__i]; });
2918 template <
typename _Tp,
typename _Ap>
2919 _GLIBCXX_SIMD_INTRINSIC
auto
2920 to_fixed_size(
const simd_mask<_Tp, _Ap>& __x)
2923 fixed_size_simd_mask<_Tp, _Np> __r;
2924 __execute_n_times<_Np>([&](
auto __i) constexpr { __r[__i] = __x[__i]; });
2929 template <
typename _Tp,
int _Np>
2930 _GLIBCXX_SIMD_INTRINSIC
2932 to_native(
const fixed_size_simd<_Tp, _Np>& __x)
2934 alignas(memory_alignment_v<native_simd<_Tp>>) _Tp __mem[_Np];
2935 __x.copy_to(__mem, vector_aligned);
2936 return {__mem, vector_aligned};
2939 template <
typename _Tp,
size_t _Np>
2940 _GLIBCXX_SIMD_INTRINSIC
2942 to_native(
const fixed_size_simd_mask<_Tp, _Np>& __x)
2944 return native_simd_mask<_Tp>([&](
auto __i) constexpr {
return __x[__i]; });
2948 template <
typename _Tp,
size_t _Np>
2950 to_compatible(
const simd<_Tp, simd_abi::fixed_size<_Np>>& __x)
2952 alignas(memory_alignment_v<simd<_Tp>>) _Tp __mem[_Np];
2953 __x.copy_to(__mem, vector_aligned);
2954 return {__mem, vector_aligned};
2957 template <
typename _Tp,
size_t _Np>
2958 _GLIBCXX_SIMD_INTRINSIC
2960 to_compatible(
const simd_mask<_Tp, simd_abi::fixed_size<_Np>>& __x)
2961 {
return simd_mask<_Tp>([&](
auto __i) constexpr {
return __x[__i]; }); }
2967 template <
typename _M,
typename _Tp>
2968 class const_where_expression
2971 static_assert(is_same_v<_V, __remove_cvref_t<_Tp>>);
2973 struct _Wrapper {
using value_type = _V; };
2976 using _Impl =
typename _V::_Impl;
2979 typename conditional_t<is_arithmetic_v<_V>, _Wrapper, _V>::value_type;
2981 _GLIBCXX_SIMD_INTRINSIC
friend const _M&
2982 __get_mask(
const const_where_expression& __x)
2983 {
return __x._M_k; }
2985 _GLIBCXX_SIMD_INTRINSIC
friend const _Tp&
2986 __get_lvalue(
const const_where_expression& __x)
2987 {
return __x._M_value; }
2993 const_where_expression(
const const_where_expression&) =
delete;
2994 const_where_expression&
operator=(
const const_where_expression&) =
delete;
2996 _GLIBCXX_SIMD_INTRINSIC const_where_expression(
const _M& __kk,
const _Tp& dd)
2997 : _M_k(__kk), _M_value(const_cast<_Tp&>(dd)) {}
2999 _GLIBCXX_SIMD_INTRINSIC _V
3002 return {__private_init,
3003 _Impl::template _S_masked_unary<negate>(__data(_M_k),
3007 template <
typename _Up,
typename _Flags>
3008 [[nodiscard]] _GLIBCXX_SIMD_INTRINSIC _V
3009 copy_from(
const _LoadStorePtr<_Up, value_type>* __mem, _Flags)
const&&
3011 return {__private_init,
3012 _Impl::_S_masked_load(__data(_M_value), __data(_M_k),
3013 _Flags::template _S_apply<_V>(__mem))};
3016 template <
typename _Up,
typename _Flags>
3017 _GLIBCXX_SIMD_INTRINSIC
void
3018 copy_to(_LoadStorePtr<_Up, value_type>* __mem, _Flags)
const&&
3020 _Impl::_S_masked_store(__data(_M_value),
3021 _Flags::template _S_apply<_V>(__mem),
3027 template <
typename _Tp>
3028 class const_where_expression<bool, _Tp>
3033 static_assert(is_same_v<_V, __remove_cvref_t<_Tp>>);
3035 struct _Wrapper {
using value_type = _V; };
3039 typename conditional_t<is_arithmetic_v<_V>, _Wrapper, _V>::value_type;
3041 _GLIBCXX_SIMD_INTRINSIC
friend const _M&
3042 __get_mask(
const const_where_expression& __x)
3043 {
return __x._M_k; }
3045 _GLIBCXX_SIMD_INTRINSIC
friend const _Tp&
3046 __get_lvalue(
const const_where_expression& __x)
3047 {
return __x._M_value; }
3053 const_where_expression(
const const_where_expression&) =
delete;
3054 const_where_expression&
operator=(
const const_where_expression&) =
delete;
3056 _GLIBCXX_SIMD_INTRINSIC const_where_expression(
const bool __kk,
const _Tp& dd)
3057 : _M_k(__kk), _M_value(const_cast<_Tp&>(dd)) {}
3059 _GLIBCXX_SIMD_INTRINSIC _V
operator-() const&&
3060 {
return _M_k ? -_M_value : _M_value; }
3062 template <
typename _Up,
typename _Flags>
3063 [[nodiscard]] _GLIBCXX_SIMD_INTRINSIC _V
3064 copy_from(
const _LoadStorePtr<_Up, value_type>* __mem, _Flags)
const&&
3065 {
return _M_k ?
static_cast<_V
>(__mem[0]) : _M_value; }
3067 template <
typename _Up,
typename _Flags>
3068 _GLIBCXX_SIMD_INTRINSIC
void
3069 copy_to(_LoadStorePtr<_Up, value_type>* __mem, _Flags)
const&&
3072 __mem[0] = _M_value;
3077 template <
typename _M,
typename _Tp>
3078 class where_expression :
public const_where_expression<_M, _Tp>
3080 using _Impl =
typename const_where_expression<_M, _Tp>::_Impl;
3082 static_assert(!is_const<_Tp>::value,
3083 "where_expression may only be instantiated with __a non-const "
3086 using typename const_where_expression<_M, _Tp>::value_type;
3087 using const_where_expression<_M, _Tp>::_M_k;
3088 using const_where_expression<_M, _Tp>::_M_value;
3091 is_same<typename _M::abi_type, typename _Tp::abi_type>::value,
"");
3094 _GLIBCXX_SIMD_INTRINSIC
friend _Tp& __get_lvalue(where_expression& __x)
3095 {
return __x._M_value; }
3098 where_expression(
const where_expression&) =
delete;
3099 where_expression&
operator=(
const where_expression&) =
delete;
3101 _GLIBCXX_SIMD_INTRINSIC where_expression(
const _M& __kk, _Tp& dd)
3102 : const_where_expression<_M, _Tp>(__kk, dd) {}
3104 template <
typename _Up>
3105 _GLIBCXX_SIMD_INTRINSIC
void operator=(_Up&& __x) &&
3107 _Impl::_S_masked_assign(__data(_M_k), __data(_M_value),
3108 __to_value_type_or_member_type<_Tp>(
3109 static_cast<_Up&&
>(__x)));
3112 #define _GLIBCXX_SIMD_OP_(__op, __name) \
3113 template <typename _Up> \
3114 _GLIBCXX_SIMD_INTRINSIC void operator __op##=(_Up&& __x)&& \
3116 _Impl::template _S_masked_cassign( \
3117 __data(_M_k), __data(_M_value), \
3118 __to_value_type_or_member_type<_Tp>(static_cast<_Up&&>(__x)), \
3119 [](auto __impl, auto __lhs, auto __rhs) constexpr { \
3120 return __impl.__name(__lhs, __rhs); \
3124 _GLIBCXX_SIMD_OP_(+, _S_plus);
3125 _GLIBCXX_SIMD_OP_(-, _S_minus);
3126 _GLIBCXX_SIMD_OP_(*, _S_multiplies);
3127 _GLIBCXX_SIMD_OP_(/, _S_divides);
3128 _GLIBCXX_SIMD_OP_(%, _S_modulus);
3129 _GLIBCXX_SIMD_OP_(&, _S_bit_and);
3130 _GLIBCXX_SIMD_OP_(|, _S_bit_or);
3131 _GLIBCXX_SIMD_OP_(^, _S_bit_xor);
3132 _GLIBCXX_SIMD_OP_(<<, _S_shift_left);
3133 _GLIBCXX_SIMD_OP_(>>, _S_shift_right);
3134 #undef _GLIBCXX_SIMD_OP_
3136 _GLIBCXX_SIMD_INTRINSIC
void operator++() &&
3139 = _Impl::template _S_masked_unary<__increment>(__data(_M_k),
3143 _GLIBCXX_SIMD_INTRINSIC
void operator++(
int) &&
3146 = _Impl::template _S_masked_unary<__increment>(__data(_M_k),
3150 _GLIBCXX_SIMD_INTRINSIC
void operator--() &&
3153 = _Impl::template _S_masked_unary<__decrement>(__data(_M_k),
3157 _GLIBCXX_SIMD_INTRINSIC
void operator--(
int) &&
3160 = _Impl::template _S_masked_unary<__decrement>(__data(_M_k),
3165 template <
typename _Up,
typename _Flags>
3166 _GLIBCXX_SIMD_INTRINSIC
void
3167 copy_from(
const _LoadStorePtr<_Up, value_type>* __mem, _Flags) &&
3170 = _Impl::_S_masked_load(__data(_M_value), __data(_M_k),
3171 _Flags::template _S_apply<_Tp>(__mem));
3176 template <
typename _Tp>
3177 class where_expression<bool, _Tp> :
public const_where_expression<bool, _Tp>
3180 using typename const_where_expression<_M, _Tp>::value_type;
3181 using const_where_expression<_M, _Tp>::_M_k;
3182 using const_where_expression<_M, _Tp>::_M_value;
3185 where_expression(
const where_expression&) =
delete;
3186 where_expression&
operator=(
const where_expression&) =
delete;
3188 _GLIBCXX_SIMD_INTRINSIC where_expression(
const _M& __kk, _Tp& dd)
3189 : const_where_expression<_M, _Tp>(__kk, dd) {}
3191 #define _GLIBCXX_SIMD_OP_(__op) \
3192 template <typename _Up> \
3193 _GLIBCXX_SIMD_INTRINSIC void operator __op(_Up&& __x)&& \
3194 { if (_M_k) _M_value __op static_cast<_Up&&>(__x); }
3196 _GLIBCXX_SIMD_OP_(=)
3197 _GLIBCXX_SIMD_OP_(+=)
3198 _GLIBCXX_SIMD_OP_(-=)
3199 _GLIBCXX_SIMD_OP_(*=)
3200 _GLIBCXX_SIMD_OP_(/=)
3201 _GLIBCXX_SIMD_OP_(%=)
3202 _GLIBCXX_SIMD_OP_(&=)
3203 _GLIBCXX_SIMD_OP_(|=)
3204 _GLIBCXX_SIMD_OP_(^=)
3205 _GLIBCXX_SIMD_OP_(<<=)
3206 _GLIBCXX_SIMD_OP_(>>=)
3207 #undef _GLIBCXX_SIMD_OP_
3209 _GLIBCXX_SIMD_INTRINSIC
void operator++() &&
3210 {
if (_M_k) ++_M_value; }
3212 _GLIBCXX_SIMD_INTRINSIC
void operator++(
int) &&
3213 {
if (_M_k) ++_M_value; }
3215 _GLIBCXX_SIMD_INTRINSIC
void operator--() &&
3216 {
if (_M_k) --_M_value; }
3218 _GLIBCXX_SIMD_INTRINSIC
void operator--(
int) &&
3219 {
if (_M_k) --_M_value; }
3222 template <
typename _Up,
typename _Flags>
3223 _GLIBCXX_SIMD_INTRINSIC
void
3224 copy_from(
const _LoadStorePtr<_Up, value_type>* __mem, _Flags) &&
3225 {
if (_M_k) _M_value = __mem[0]; }
3229 template <
typename _Tp,
typename _Ap>
3230 _GLIBCXX_SIMD_INTRINSIC where_expression<simd_mask<_Tp, _Ap>, simd<_Tp, _Ap>>
3231 where(
const typename simd<_Tp, _Ap>::mask_type& __k, simd<_Tp, _Ap>& __value)
3232 {
return {__k, __value}; }
3234 template <
typename _Tp,
typename _Ap>
3235 _GLIBCXX_SIMD_INTRINSIC
3236 const_where_expression<simd_mask<_Tp, _Ap>, simd<_Tp, _Ap>>
3237 where(
const typename simd<_Tp, _Ap>::mask_type& __k,
3238 const simd<_Tp, _Ap>& __value)
3239 {
return {__k, __value}; }
3241 template <
typename _Tp,
typename _Ap>
3242 _GLIBCXX_SIMD_INTRINSIC
3243 where_expression<simd_mask<_Tp, _Ap>, simd_mask<_Tp, _Ap>>
3245 simd_mask<_Tp, _Ap>& __value)
3246 {
return {__k, __value}; }
3248 template <
typename _Tp,
typename _Ap>
3249 _GLIBCXX_SIMD_INTRINSIC
3250 const_where_expression<simd_mask<_Tp, _Ap>, simd_mask<_Tp, _Ap>>
3252 const simd_mask<_Tp, _Ap>& __value)
3253 {
return {__k, __value}; }
3255 template <
typename _Tp>
3256 _GLIBCXX_SIMD_INTRINSIC where_expression<bool, _Tp>
3257 where(_ExactBool __k, _Tp& __value)
3258 {
return {__k, __value}; }
3260 template <
typename _Tp>
3261 _GLIBCXX_SIMD_INTRINSIC const_where_expression<bool, _Tp>
3262 where(_ExactBool __k,
const _Tp& __value)
3263 {
return {__k, __value}; }
3265 template <
typename _Tp,
typename _Ap>
3266 void where(
bool __k, simd<_Tp, _Ap>& __value) =
delete;
3268 template <
typename _Tp,
typename _Ap>
3269 void where(
bool __k,
const simd<_Tp, _Ap>& __value) =
delete;
3272 namespace __proposed {
3273 template <
size_t _Np>
3276 const bitset<_Np> __bits;
3279 where_range(bitset<_Np> __b) : __bits(__b) {}
3286 _GLIBCXX_SIMD_INTRINSIC
void __next_bit()
3287 { __bit = __builtin_ctzl(__mask); }
3289 _GLIBCXX_SIMD_INTRINSIC
void __reset_lsb()
3292 __mask &= (__mask - 1);
3297 iterator(decltype(__mask) __m) : __mask(__m) { __next_bit(); }
3298 iterator(
const iterator&) =
default;
3299 iterator(iterator&&) =
default;
3301 _GLIBCXX_SIMD_ALWAYS_INLINE
size_t operator->()
const
3304 _GLIBCXX_SIMD_ALWAYS_INLINE
size_t operator*()
const
3307 _GLIBCXX_SIMD_ALWAYS_INLINE iterator& operator++()
3314 _GLIBCXX_SIMD_ALWAYS_INLINE iterator operator++(
int)
3316 iterator __tmp = *
this;
3322 _GLIBCXX_SIMD_ALWAYS_INLINE
bool operator==(
const iterator& __rhs)
const
3323 {
return __mask == __rhs.__mask; }
3325 _GLIBCXX_SIMD_ALWAYS_INLINE
bool operator!=(
const iterator& __rhs)
const
3326 {
return __mask != __rhs.__mask; }
3329 iterator
begin()
const
3330 {
return __bits.to_ullong(); }
3332 iterator
end()
const
3336 template <
typename _Tp,
typename _Ap>
3337 where_range<simd_size_v<_Tp, _Ap>>
3338 where(
const simd_mask<_Tp, _Ap>& __k)
3339 {
return __k.__to_bitset(); }
3345 template <
typename _Tp,
typename _Abi,
typename _BinaryOperation = plus<>>
3346 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3347 reduce(
const simd<_Tp, _Abi>& __v,
3348 _BinaryOperation __binary_op = _BinaryOperation())
3349 {
return _Abi::_SimdImpl::_S_reduce(__v, __binary_op); }
3351 template <
typename _M,
typename _V,
typename _BinaryOperation = plus<>>
3352 _GLIBCXX_SIMD_INTRINSIC
typename _V::value_type
3353 reduce(
const const_where_expression<_M, _V>& __x,
3354 typename _V::value_type __identity_element,
3355 _BinaryOperation __binary_op)
3357 if (__builtin_expect(none_of(__get_mask(__x)),
false))
3358 return __identity_element;
3360 _V __tmp = __identity_element;
3361 _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3362 __data(__get_lvalue(__x)));
3363 return reduce(__tmp, __binary_op);
3366 template <
typename _M,
typename _V>
3367 _GLIBCXX_SIMD_INTRINSIC
typename _V::value_type
3368 reduce(
const const_where_expression<_M, _V>& __x, plus<> __binary_op = {})
3369 {
return reduce(__x, 0, __binary_op); }
3371 template <
typename _M,
typename _V>
3372 _GLIBCXX_SIMD_INTRINSIC
typename _V::value_type
3373 reduce(
const const_where_expression<_M, _V>& __x, multiplies<> __binary_op)
3374 {
return reduce(__x, 1, __binary_op); }
3376 template <
typename _M,
typename _V>
3377 _GLIBCXX_SIMD_INTRINSIC
typename _V::value_type
3378 reduce(
const const_where_expression<_M, _V>& __x, bit_and<> __binary_op)
3379 {
return reduce(__x, ~
typename _V::value_type(), __binary_op); }
3381 template <
typename _M,
typename _V>
3382 _GLIBCXX_SIMD_INTRINSIC
typename _V::value_type
3383 reduce(
const const_where_expression<_M, _V>& __x, bit_or<> __binary_op)
3384 {
return reduce(__x, 0, __binary_op); }
3386 template <
typename _M,
typename _V>
3387 _GLIBCXX_SIMD_INTRINSIC
typename _V::value_type
3388 reduce(
const const_where_expression<_M, _V>& __x, bit_xor<> __binary_op)
3389 {
return reduce(__x, 0, __binary_op); }
3393 template <
typename _Tp,
typename _Ap>
3394 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3395 min(
const simd<_Tp, _Ap>& __a,
const simd<_Tp, _Ap>& __b)
3396 {
return {__private_init, _Ap::_SimdImpl::_S_min(__data(__a), __data(__b))}; }
3398 template <
typename _Tp,
typename _Ap>
3399 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3400 max(
const simd<_Tp, _Ap>& __a,
const simd<_Tp, _Ap>& __b)
3401 {
return {__private_init, _Ap::_SimdImpl::_S_max(__data(__a), __data(__b))}; }
3403 template <
typename _Tp,
typename _Ap>
3404 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3405 pair<simd<_Tp, _Ap>, simd<_Tp, _Ap>>
3406 minmax(
const simd<_Tp, _Ap>& __a,
const simd<_Tp, _Ap>& __b)
3408 const auto pair_of_members
3409 = _Ap::_SimdImpl::_S_minmax(__data(__a), __data(__b));
3410 return {simd<_Tp, _Ap>(__private_init, pair_of_members.first),
3411 simd<_Tp, _Ap>(__private_init, pair_of_members.second)};
3414 template <
typename _Tp,
typename _Ap>
3415 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3416 clamp(
const simd<_Tp, _Ap>& __v,
const simd<_Tp, _Ap>& __lo,
3417 const simd<_Tp, _Ap>& __hi)
3419 using _Impl =
typename _Ap::_SimdImpl;
3420 return {__private_init,
3421 _Impl::_S_min(__data(__hi),
3422 _Impl::_S_max(__data(__lo), __data(__v)))};
3427 template <
size_t... _Sizes,
typename _Tp,
typename _Ap,
3429 inline tuple<simd<_Tp, simd_abi::deduce_t<_Tp, _Sizes>>...>
3430 split(
const simd<_Tp, _Ap>&);
3433 template <
int _Index,
int _Total,
int _Combine = 1,
typename _Tp,
size_t _Np>
3434 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_CONST
3435 _SimdWrapper<_Tp, _Np / _Total * _Combine>
3436 __extract_part(
const _SimdWrapper<_Tp, _Np> __x);
3438 template <
int Index,
int Parts,
int _Combine = 1,
typename _Tp,
typename _A0,
3440 _GLIBCXX_SIMD_INTRINSIC
auto
3441 __extract_part(
const _SimdTuple<_Tp, _A0, _As...>& __x);
3445 template <
size_t _V0,
size_t... _Values>
3448 template <
size_t _I>
3449 static constexpr
size_t _S_at(_SizeConstant<_I> = {})
3451 if constexpr (_I == 0)
3454 return _SizeList<_Values...>::template _S_at<_I - 1>();
3457 template <
size_t _I>
3458 static constexpr auto _S_before(_SizeConstant<_I> = {})
3460 if constexpr (_I == 0)
3461 return _SizeConstant<0>();
3463 return _SizeConstant<
3464 _V0 + _SizeList<_Values...>::template _S_before<_I - 1>()>();
3467 template <
size_t _Np>
3468 static constexpr auto _S_pop_front(_SizeConstant<_Np> = {})
3470 if constexpr (_Np == 0)
3473 return _SizeList<_Values...>::template _S_pop_front<_Np - 1>();
3479 template <typename _Tp,
size_t _Np>
3480 _GLIBCXX_SIMD_INTRINSIC _SimdWrapper<_Tp, _Np / 2>
3481 __extract_center(_SimdWrapper<_Tp, _Np> __x)
3483 static_assert(_Np >= 4);
3484 static_assert(_Np % 4 == 0);
3485 #if _GLIBCXX_SIMD_X86INTRIN
3486 if constexpr (__have_avx512f &&
sizeof(_Tp) * _Np == 64)
3488 const auto __intrin = __to_intrin(__x);
3489 if constexpr (is_integral_v<_Tp>)
3490 return __vector_bitcast<_Tp>(_mm512_castsi512_si256(
3491 _mm512_shuffle_i32x4(__intrin, __intrin,
3492 1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
3493 else if constexpr (
sizeof(_Tp) == 4)
3494 return __vector_bitcast<_Tp>(_mm512_castps512_ps256(
3495 _mm512_shuffle_f32x4(__intrin, __intrin,
3496 1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
3497 else if constexpr (sizeof(_Tp) == 8)
3498 return __vector_bitcast<_Tp>(_mm512_castpd512_pd256(
3499 _mm512_shuffle_f64x2(__intrin, __intrin,
3500 1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
3502 __assert_unreachable<_Tp>();
3504 else if constexpr (sizeof(_Tp) * _Np == 32 && is_floating_point_v<_Tp>)
3505 return __vector_bitcast<_Tp>(
3506 _mm_shuffle_pd(__lo128(__vector_bitcast<
double>(__x)),
3507 __hi128(__vector_bitcast<
double>(__x)), 1));
3508 else if constexpr (sizeof(__x) == 32 && sizeof(_Tp) * _Np <= 32)
3509 return __vector_bitcast<_Tp>(
3510 _mm_alignr_epi8(__hi128(__vector_bitcast<_LLong>(__x)),
3511 __lo128(__vector_bitcast<_LLong>(__x)),
3512 sizeof(_Tp) * _Np / 4));
3516 __vector_type_t<_Tp, _Np / 2> __r;
3517 __builtin_memcpy(&__r,
3518 reinterpret_cast<const char*
>(&__x)
3519 +
sizeof(_Tp) * _Np / 4,
3520 sizeof(_Tp) * _Np / 2);
3525 template <
typename _Tp,
typename _A0,
typename... _As>
3526 _GLIBCXX_SIMD_INTRINSIC
3527 _SimdWrapper<_Tp, _SimdTuple<_Tp, _A0, _As...>::_S_size() / 2>
3528 __extract_center(
const _SimdTuple<_Tp, _A0, _As...>& __x)
3530 if constexpr (
sizeof...(_As) == 0)
3531 return __extract_center(__x.first);
3533 return __extract_part<1, 4, 2>(__x);
3538 template <
size_t... _Sizes, typename _Tp, typename... _As>
3540 __split_wrapper(_SizeList<_Sizes...>, const _SimdTuple<_Tp, _As...>& __x)
3542 return split<_Sizes...>(
3543 fixed_size_simd<_Tp, _SimdTuple<_Tp, _As...>::_S_size()>(__private_init,
3550 template <
typename _V,
typename _Ap,
3551 size_t Parts = simd_size_v<typename _V::value_type, _Ap> /
_V::size()>
3552 enable_if_t<simd_size_v<typename _V::value_type, _Ap> == Parts *
_V::size()
3553 && is_simd_v<_V>, array<_V, Parts>>
3554 split(
const simd<typename _V::value_type, _Ap>& __x)
3556 using _Tp =
typename _V::value_type;
3557 if constexpr (Parts == 1)
3559 return {simd_cast<_V>(__x)};
3561 else if (__x._M_is_constprop())
3563 return __generate_from_n_evaluations<Parts, array<_V, Parts>>([&](
3564 auto __i) constexpr {
3565 return _V([&](
auto __j) constexpr {
3566 return __x[__i *
_V::size() + __j];
3571 __is_fixed_size_abi_v<_Ap>
3572 && (is_same_v<typename _V::abi_type, simd_abi::scalar>
3573 || (__is_fixed_size_abi_v<typename _V::abi_type>
3574 &&
sizeof(_V) ==
sizeof(_Tp) *
_V::size()
3578 #ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
3579 const __may_alias<_Tp>*
const __element_ptr
3580 =
reinterpret_cast<const __may_alias<_Tp>*
>(&__data(__x));
3581 return __generate_from_n_evaluations<Parts, array<_V, Parts>>([&](
3582 auto __i) constexpr {
3583 return _V(__element_ptr + __i *
_V::size(), vector_aligned);
3586 const auto& __xx = __data(__x);
3587 return __generate_from_n_evaluations<Parts, array<_V, Parts>>([&](
3588 auto __i) constexpr {
3589 [[maybe_unused]] constexpr
size_t __offset
3590 = decltype(__i)::value * _V::
size();
3591 return _V([&](
auto __j) constexpr {
3592 constexpr _SizeConstant<__j + __offset> __k;
3598 else if constexpr (is_same_v<typename _V::abi_type, simd_abi::scalar>)
3601 return __generate_from_n_evaluations<Parts, array<_V, Parts>>([&](
3602 auto __i) constexpr {
return __x[__i]; });
3606 return __generate_from_n_evaluations<Parts, array<_V, Parts>>([&](
3607 auto __i) constexpr {
3608 if constexpr (__is_fixed_size_abi_v<typename _V::abi_type>)
3609 return _V([&](
auto __j) constexpr {
3610 return __x[__i *
_V::size() + __j];
3613 return _V(__private_init,
3614 __extract_part<decltype(__i)::value, Parts>(__data(__x)));
3621 template <
typename _V,
typename _Ap,
3623 = simd_size_v<typename _V::simd_type::value_type, _Ap> /
_V::size()>
3624 enable_if_t<is_simd_mask_v<_V> && simd_size_v<
typename
3625 _V::simd_type::value_type, _Ap> == _Parts *
_V::size(), array<_V, _Parts>>
3626 split(
const simd_mask<typename _V::simd_type::value_type, _Ap>& __x)
3628 if constexpr (is_same_v<_Ap, typename _V::abi_type>)
3630 else if constexpr (_Parts == 1)
3631 return {__proposed::static_simd_cast<_V>(__x)};
3632 else if constexpr (_Parts == 2 && __is_sse_abi<typename _V::abi_type>()
3633 && __is_avx_abi<_Ap>())
3634 return {_V(__private_init, __lo128(__data(__x))),
3635 _V(__private_init, __hi128(__data(__x)))};
3636 else if constexpr (
_V::size() <= __CHAR_BIT__ *
sizeof(_ULLong))
3638 const bitset __bits = __x.__to_bitset();
3639 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>([&](
3640 auto __i) constexpr {
3641 constexpr
size_t __offset = __i *
_V::size();
3642 return _V(__bitset_init, (__bits >> __offset).to_ullong());
3647 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>([&](
3648 auto __i) constexpr {
3649 constexpr
size_t __offset = __i *
_V::size();
3651 __private_init, [&](
auto __j) constexpr {
3652 return __x[__j + __offset];
3660 template <
size_t... _Sizes,
typename _Tp,
typename _Ap,
typename>
3661 _GLIBCXX_SIMD_ALWAYS_INLINE
3662 tuple<simd<_Tp, simd_abi::deduce_t<_Tp, _Sizes>>...>
3663 split(
const simd<_Tp, _Ap>& __x)
3665 using _SL = _SizeList<_Sizes...>;
3666 using _Tuple = tuple<__deduced_simd<_Tp, _Sizes>...>;
3667 constexpr
size_t _Np = simd_size_v<_Tp, _Ap>;
3668 constexpr
size_t _N0 = _SL::template _S_at<0>();
3669 using _V = __deduced_simd<_Tp, _N0>;
3671 if (__x._M_is_constprop())
3672 return __generate_from_n_evaluations<
sizeof...(_Sizes), _Tuple>([&](
3673 auto __i) constexpr {
3674 using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
3675 constexpr
size_t __offset = _SL::_S_before(__i);
3676 return _Vi([&](
auto __j) constexpr {
return __x[__offset + __j]; });
3678 else if constexpr (_Np == _N0)
3680 static_assert(
sizeof...(_Sizes) == 1);
3681 return {simd_cast<_V>(__x)};
3684 (__is_fixed_size_abi_v<
3685 _Ap> && __fixed_size_storage_t<_Tp, _Np>::_S_first_size == _N0)
3688 !__is_fixed_size_abi_v<typename _V::abi_type>,
3689 "How can <_Tp, _Np> be __a single _SimdTuple entry but __a "
3694 return tuple_cat(make_tuple(_V(__private_init, __data(__x).first)),
3695 __split_wrapper(_SL::template _S_pop_front<1>(),
3696 __data(__x).second));
3698 else if constexpr ((!is_same_v<simd_abi::scalar,
3699 simd_abi::deduce_t<_Tp, _Sizes>> && ...)
3700 && (!__is_fixed_size_abi_v<
3701 simd_abi::deduce_t<_Tp, _Sizes>> && ...))
3703 if constexpr (((_Sizes * 2 == _Np) && ...))
3704 return {{__private_init, __extract_part<0, 2>(__data(__x))},
3705 {__private_init, __extract_part<1, 2>(__data(__x))}};
3706 else if constexpr (is_same_v<_SizeList<_Sizes...>,
3707 _SizeList<_Np / 3, _Np / 3, _Np / 3>>)
3708 return {{__private_init, __extract_part<0, 3>(__data(__x))},
3709 {__private_init, __extract_part<1, 3>(__data(__x))},
3710 {__private_init, __extract_part<2, 3>(__data(__x))}};
3711 else if constexpr (is_same_v<_SizeList<_Sizes...>,
3712 _SizeList<2 * _Np / 3, _Np / 3>>)
3713 return {{__private_init, __extract_part<0, 3, 2>(__data(__x))},
3714 {__private_init, __extract_part<2, 3>(__data(__x))}};
3715 else if constexpr (is_same_v<_SizeList<_Sizes...>,
3716 _SizeList<_Np / 3, 2 * _Np / 3>>)
3717 return {{__private_init, __extract_part<0, 3>(__data(__x))},
3718 {__private_init, __extract_part<1, 3, 2>(__data(__x))}};
3719 else if constexpr (is_same_v<_SizeList<_Sizes...>,
3720 _SizeList<_Np / 2, _Np / 4, _Np / 4>>)
3721 return {{__private_init, __extract_part<0, 2>(__data(__x))},
3722 {__private_init, __extract_part<2, 4>(__data(__x))},
3723 {__private_init, __extract_part<3, 4>(__data(__x))}};
3724 else if constexpr (is_same_v<_SizeList<_Sizes...>,
3725 _SizeList<_Np / 4, _Np / 4, _Np / 2>>)
3726 return {{__private_init, __extract_part<0, 4>(__data(__x))},
3727 {__private_init, __extract_part<1, 4>(__data(__x))},
3728 {__private_init, __extract_part<1, 2>(__data(__x))}};
3729 else if constexpr (is_same_v<_SizeList<_Sizes...>,
3730 _SizeList<_Np / 4, _Np / 2, _Np / 4>>)
3731 return {{__private_init, __extract_part<0, 4>(__data(__x))},
3732 {__private_init, __extract_center(__data(__x))},
3733 {__private_init, __extract_part<3, 4>(__data(__x))}};
3734 else if constexpr (((_Sizes * 4 == _Np) && ...))
3735 return {{__private_init, __extract_part<0, 4>(__data(__x))},
3736 {__private_init, __extract_part<1, 4>(__data(__x))},
3737 {__private_init, __extract_part<2, 4>(__data(__x))},
3738 {__private_init, __extract_part<3, 4>(__data(__x))}};
3741 #ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
3742 const __may_alias<_Tp>*
const __element_ptr
3743 =
reinterpret_cast<const __may_alias<_Tp>*
>(&__x);
3744 return __generate_from_n_evaluations<
sizeof...(_Sizes), _Tuple>([&](
3745 auto __i) constexpr {
3746 using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
3747 constexpr
size_t __offset = _SL::_S_before(__i);
3748 constexpr
size_t __base_align =
alignof(simd<_Tp, _Ap>);
3749 constexpr
size_t __a
3750 = __base_align - ((__offset *
sizeof(_Tp)) % __base_align);
3751 constexpr
size_t __b = ((__a - 1) & __a) ^ __a;
3752 constexpr
size_t __alignment = __b == 0 ? __a : __b;
3753 return _Vi(__element_ptr + __offset, overaligned<__alignment>);
3756 return __generate_from_n_evaluations<
sizeof...(_Sizes), _Tuple>([&](
3757 auto __i) constexpr {
3758 using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
3759 const auto& __xx = __data(__x);
3760 using _Offset = decltype(_SL::_S_before(__i));
3761 return _Vi([&](
auto __j) constexpr {
3762 constexpr _SizeConstant<_Offset::value + __j> __k;
3772 template <
size_t _I,
typename _Tp,
typename _Ap,
typename... _As>
3773 _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
3774 __subscript_in_pack(
const simd<_Tp, _Ap>& __x,
const simd<_Tp, _As>&... __xs)
3776 if constexpr (_I < simd_size_v<_Tp, _Ap>)
3779 return __subscript_in_pack<_I - simd_size_v<_Tp, _Ap>>(__xs...);
3784 template <
typename _Tp,
typename _A0,
typename... _As>
3785 _GLIBCXX_SIMD_INTRINSIC
void
3786 __store_pack_of_simd(
char* __mem,
const simd<_Tp, _A0>& __x0,
3787 const simd<_Tp, _As>&... __xs)
3789 constexpr
size_t __n_bytes =
sizeof(_Tp) * simd_size_v<_Tp, _A0>;
3790 __builtin_memcpy(__mem, &__data(__x0), __n_bytes);
3791 if constexpr (
sizeof...(__xs) > 0)
3792 __store_pack_of_simd(__mem + __n_bytes, __xs...);
3797 template <typename _Tp, typename... _As>
3798 inline _GLIBCXX_SIMD_CONSTEXPR
3799 simd<_Tp, simd_abi::deduce_t<_Tp, (simd_size_v<_Tp, _As> + ...)>>
3800 concat(const simd<_Tp, _As>&... __xs)
3802 using _Rp = __deduced_simd<_Tp, (simd_size_v<_Tp, _As> + ...)>;
3803 if constexpr (
sizeof...(__xs) == 1)
3804 return simd_cast<_Rp>(__xs...);
3805 else if ((... && __xs._M_is_constprop()))
3807 simd_abi::deduce_t<_Tp, (simd_size_v<_Tp, _As> + ...)>>([&](
3808 auto __i) constexpr {
return __subscript_in_pack<__i>(__xs...); });
3812 __store_pack_of_simd(
reinterpret_cast<char*
>(&__data(__r)), __xs...);
3819 template <
typename _Tp,
typename _Abi,
size_t _Np>
3820 _GLIBCXX_SIMD_ALWAYS_INLINE
3821 _GLIBCXX_SIMD_CONSTEXPR __deduced_simd<_Tp, simd_size_v<_Tp, _Abi> * _Np>
3822 concat(
const array<simd<_Tp, _Abi>, _Np>& __x)
3824 return __call_with_subscripts<_Np>(__x, [](
const auto&... __xs) {
3825 return concat(__xs...);
3832 template <
typename _Up,
typename _Accessor = _Up,
3833 typename _ValueType =
typename _Up::value_type>
3834 class _SmartReference
3840 _GLIBCXX_SIMD_INTRINSIC constexpr _ValueType _M_read() const noexcept
3842 if constexpr (is_arithmetic_v<_Up>)
3845 return _M_obj[_M_index];
3848 template <
typename _Tp>
3849 _GLIBCXX_SIMD_INTRINSIC constexpr
void _M_write(_Tp&& __x)
const
3850 { _Accessor::_S_set(_M_obj, _M_index,
static_cast<_Tp&&
>(__x)); }
3853 _GLIBCXX_SIMD_INTRINSIC constexpr
3854 _SmartReference(_Up& __o,
int __i) noexcept
3855 : _M_index(__i), _M_obj(__o) {}
3857 using value_type = _ValueType;
3859 _GLIBCXX_SIMD_INTRINSIC _SmartReference(
const _SmartReference&) =
delete;
3861 _GLIBCXX_SIMD_INTRINSIC constexpr
operator value_type() const noexcept
3862 {
return _M_read(); }
3864 template <
typename _Tp,
3866 = _ValuePreservingOrInt<__remove_cvref_t<_Tp>, value_type>>
3867 _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference
operator=(_Tp&& __x) &&
3869 _M_write(
static_cast<_Tp&&
>(__x));
3870 return {_M_obj, _M_index};
3873 #define _GLIBCXX_SIMD_OP_(__op) \
3874 template <typename _Tp, \
3876 = decltype(declval<value_type>() __op declval<_Tp>()), \
3877 typename = _ValuePreservingOrInt<__remove_cvref_t<_Tp>, _TT>, \
3878 typename = _ValuePreservingOrInt<_TT, value_type>> \
3879 _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference \
3880 operator __op##=(_Tp&& __x) && \
3882 const value_type& __lhs = _M_read(); \
3883 _M_write(__lhs __op __x); \
3884 return {_M_obj, _M_index}; \
3886 _GLIBCXX_SIMD_ALL_ARITHMETICS(_GLIBCXX_SIMD_OP_);
3887 _GLIBCXX_SIMD_ALL_SHIFTS(_GLIBCXX_SIMD_OP_);
3888 _GLIBCXX_SIMD_ALL_BINARY(_GLIBCXX_SIMD_OP_);
3889 #undef _GLIBCXX_SIMD_OP_
3891 template <
typename _Tp = void,
3893 = decltype(++declval<conditional_t<true, value_type, _Tp>&>())>
3894 _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference operator++() &&
3896 value_type __x = _M_read();
3898 return {_M_obj, _M_index};
3901 template <
typename _Tp = void,
3903 = decltype(declval<conditional_t<true, value_type, _Tp>&>()++)>
3904 _GLIBCXX_SIMD_INTRINSIC constexpr value_type operator++(
int) &&
3906 const value_type __r = _M_read();
3907 value_type __x = __r;
3912 template <
typename _Tp = void,
3914 = decltype(--declval<conditional_t<true, value_type, _Tp>&>())>
3915 _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference operator--() &&
3917 value_type __x = _M_read();
3919 return {_M_obj, _M_index};
3922 template <
typename _Tp = void,
3924 = decltype(declval<conditional_t<true, value_type, _Tp>&>()--)>
3925 _GLIBCXX_SIMD_INTRINSIC constexpr value_type operator--(
int) &&
3927 const value_type __r = _M_read();
3928 value_type __x = __r;
3933 _GLIBCXX_SIMD_INTRINSIC
friend void
3934 swap(_SmartReference&& __a, _SmartReference&& __b) noexcept(
3936 is_nothrow_constructible<value_type, _SmartReference&&>,
3937 is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
3939 value_type __tmp =
static_cast<_SmartReference&&
>(__a);
3940 static_cast<_SmartReference&&
>(__a) =
static_cast<value_type
>(__b);
3941 static_cast<_SmartReference&&
>(__b) =
std::move(__tmp);
3944 _GLIBCXX_SIMD_INTRINSIC
friend void
3945 swap(value_type& __a, _SmartReference&& __b) noexcept(
3947 is_nothrow_constructible<value_type, value_type&&>,
3948 is_nothrow_assignable<value_type&, value_type&&>,
3949 is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
3952 __a =
static_cast<value_type
>(__b);
3953 static_cast<_SmartReference&&
>(__b) =
std::move(__tmp);
3956 _GLIBCXX_SIMD_INTRINSIC
friend void
3957 swap(_SmartReference&& __a, value_type& __b) noexcept(
3959 is_nothrow_constructible<value_type, _SmartReference&&>,
3960 is_nothrow_assignable<value_type&, value_type&&>,
3961 is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
3963 value_type __tmp(__a);
3964 static_cast<_SmartReference&&
>(__a) =
std::move(__b);
3971 template <
int _Bytes>
3972 struct __scalar_abi_wrapper
3974 template <
typename _Tp>
static constexpr
size_t _S_full_size = 1;
3975 template <
typename _Tp>
static constexpr
size_t _S_size = 1;
3976 template <
typename _Tp>
static constexpr
size_t _S_is_partial =
false;
3978 template <
typename _Tp,
typename _Abi = simd_abi::scalar>
3979 static constexpr
bool _S_is_valid_v
3980 = _Abi::template _IsValid<_Tp>::value &&
sizeof(_Tp) == _Bytes;
3985 template <
typename _Tp>
3986 struct __decay_abi {
using type = _Tp; };
3988 template <
int _Bytes>
3989 struct __decay_abi<__scalar_abi_wrapper<_Bytes>>
3990 {
using type = simd_abi::scalar; };
3998 template <
template <
int>
class _Abi,
int _Bytes,
typename _Tp>
3999 struct __find_next_valid_abi
4001 static constexpr
auto _S_choose()
4003 constexpr
int _NextBytes = std::__bit_ceil(_Bytes) / 2;
4004 using _NextAbi = _Abi<_NextBytes>;
4005 if constexpr (_NextBytes <
sizeof(_Tp) * 2)
4006 return _Abi<_Bytes>();
4007 else if constexpr (_NextAbi::template _S_is_partial<_Tp> ==
false
4008 && _NextAbi::template _S_is_valid_v<_Tp>)
4011 return __find_next_valid_abi<_Abi, _NextBytes, _Tp>::_S_choose();
4014 using type = decltype(_S_choose());
4017 template <
int _Bytes,
typename _Tp>
4018 struct __find_next_valid_abi<__scalar_abi_wrapper, _Bytes, _Tp>
4019 {
using type = simd_abi::scalar; };
4022 template <
template <
int>
class...>
4025 template <
typename,
int>
static constexpr
bool _S_has_valid_abi =
false;
4026 template <
typename,
int>
using _FirstValidAbi = void;
4027 template <
typename,
int>
using _BestAbi = void;
4030 template <
template <
int>
class _A0,
template <
int>
class... _Rest>
4031 struct _AbiList<_A0, _Rest...>
4033 template <
typename _Tp,
int _Np>
4034 static constexpr
bool _S_has_valid_abi
4035 = _A0<
sizeof(_Tp) * _Np>::
template _S_is_valid_v<
4036 _Tp> || _AbiList<_Rest...>::template _S_has_valid_abi<_Tp, _Np>;
4038 template <
typename _Tp,
int _Np>
4040 _A0<
sizeof(_Tp) * _Np>::
template _S_is_valid_v<_Tp>,
4041 typename __decay_abi<_A0<
sizeof(_Tp) * _Np>>::type,
4042 typename _AbiList<_Rest...>::template _FirstValidAbi<_Tp, _Np>>;
4044 template <
typename _Tp,
int _Np>
4045 static constexpr
auto _S_determine_best_abi()
4047 static_assert(_Np >= 1);
4048 constexpr
int _Bytes =
sizeof(_Tp) * _Np;
4049 if constexpr (_Np == 1)
4050 return __make_dependent_t<_Tp, simd_abi::scalar>{};
4053 constexpr
int __fullsize = _A0<_Bytes>::template _S_full_size<_Tp>;
4058 if constexpr (_A0<_Bytes>::template _S_is_valid_v<
4059 _Tp> && __fullsize / 2 < _Np)
4060 return typename __decay_abi<_A0<_Bytes>>::type{};
4064 typename __find_next_valid_abi<_A0, _Bytes, _Tp>::type;
4065 if constexpr (_B::template _S_is_valid_v<
4066 _Tp> && _B::template _S_size<_Tp> <= _Np)
4070 typename _AbiList<_Rest...>::template _BestAbi<_Tp, _Np>{};
4075 template <
typename _Tp,
int _Np>
4076 using _BestAbi = decltype(_S_determine_best_abi<_Tp, _Np>());
4084 using _AllNativeAbis = _AbiList<simd_abi::_VecBltnBtmsk, simd_abi::_VecBuiltin,
4085 __scalar_abi_wrapper>;
4088 template <
typename _Tp,
typename _Abi>
4089 struct _SimdTraits<_Tp, _Abi,
void_t<typename _Abi::template _IsValid<_Tp>>>
4090 : _Abi::template __traits<_Tp> {};
4094 template <
typename _Tp,
size_t _Np>
4095 struct __deduce_impl<
4096 _Tp, _Np,
enable_if_t<_AllNativeAbis::template _S_has_valid_abi<_Tp, _Np>>>
4097 {
using type = _AllNativeAbis::_FirstValidAbi<_Tp, _Np>; };
4100 template <
typename _Tp,
size_t _Np,
typename =
void>
4101 struct __deduce_fixed_size_fallback {};
4103 template <
typename _Tp,
size_t _Np>
4104 struct __deduce_fixed_size_fallback<_Tp, _Np,
4105 enable_if_t<simd_abi::fixed_size<_Np>::template _S_is_valid_v<_Tp>>>
4106 {
using type = simd_abi::fixed_size<_Np>; };
4108 template <
typename _Tp,
size_t _Np,
typename>
4109 struct __deduce_impl :
public __deduce_fixed_size_fallback<_Tp, _Np> {};
4114 template <
typename _Tp,
typename _Abi>
4115 class simd_mask :
public _SimdTraits<_Tp, _Abi>::_MaskBase
4118 using _Traits = _SimdTraits<_Tp, _Abi>;
4119 using _MemberType =
typename _Traits::_MaskMember;
4125 using _Ip = __int_for_sizeof_t<_Tp>;
4126 static constexpr _Ip* _S_type_tag =
nullptr;
4128 friend typename _Traits::_MaskBase;
4129 friend class simd<_Tp, _Abi>;
4130 friend typename _Traits::_SimdImpl;
4133 using _Impl =
typename _Traits::_MaskImpl;
4138 using value_type = bool;
4139 using reference = _SmartReference<_MemberType, _Impl, value_type>;
4140 using simd_type = simd<_Tp, _Abi>;
4141 using abi_type = _Abi;
4144 static constexpr
size_t size()
4145 {
return __size_or_zero_v<_Tp, _Abi>; }
4149 simd_mask() =
default;
4150 simd_mask(
const simd_mask&) =
default;
4151 simd_mask(simd_mask&&) =
default;
4152 simd_mask&
operator=(
const simd_mask&) =
default;
4153 simd_mask&
operator=(simd_mask&&) =
default;
4157 _GLIBCXX_SIMD_ALWAYS_INLINE
explicit
4158 simd_mask(
typename _Traits::_MaskCastType __init)
4159 : _M_data{__init} {}
4168 _GLIBCXX_SIMD_ALWAYS_INLINE
static simd_mask
4169 __from_bitset(bitset<
size()> bs)
4170 {
return {__bitset_init, bs}; }
4172 _GLIBCXX_SIMD_ALWAYS_INLINE bitset<
size()>
4174 {
return _Impl::_S_to_bits(_M_data)._M_to_bitset(); }
4178 _GLIBCXX_SIMD_ALWAYS_INLINE
explicit _GLIBCXX_SIMD_CONSTEXPR
4179 simd_mask(value_type __x)
4180 : _M_data(_Impl::template _S_broadcast<_Ip>(__x)) {}
4184 #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4186 template <
typename _Up,
typename _A2,
4187 typename = enable_if_t<simd_size_v<_Up, _A2> ==
size()>>
4188 _GLIBCXX_SIMD_ALWAYS_INLINE
explicit(
sizeof(_MemberType)
4189 !=
sizeof(
typename _SimdTraits<_Up, _A2>::_MaskMember))
4190 simd_mask(
const simd_mask<_Up, _A2>& __x)
4191 : simd_mask(__proposed::static_simd_cast<simd_mask>(__x)) {}
4194 template <
typename _Up,
typename =
enable_if_t<conjunction<
4195 is_same<abi_type, simd_abi::fixed_size<
size()>>,
4196 is_same<_Up, _Up>>::value>>
4197 _GLIBCXX_SIMD_ALWAYS_INLINE
4198 simd_mask(
const simd_mask<_Up, simd_abi::fixed_size<
size()>>& __x)
4199 : _M_data(_Impl::_S_from_bitmask(__data(__x), _S_type_tag)) {}
4204 template <
typename _Flags>
4205 _GLIBCXX_SIMD_ALWAYS_INLINE
4206 simd_mask(
const value_type* __mem, _Flags)
4207 : _M_data(_Impl::template _S_load<_Ip>(
4208 _Flags::template _S_apply<simd_mask>(__mem))) {}
4210 template <
typename _Flags>
4211 _GLIBCXX_SIMD_ALWAYS_INLINE
4212 simd_mask(
const value_type* __mem, simd_mask __k, _Flags)
4216 = _Impl::_S_masked_load(_M_data, __k._M_data,
4217 _Flags::template _S_apply<simd_mask>(__mem));
4222 template <
typename _Flags>
4223 _GLIBCXX_SIMD_ALWAYS_INLINE
void
4224 copy_from(
const value_type* __mem, _Flags)
4226 _M_data = _Impl::template _S_load<_Ip>(
4227 _Flags::template _S_apply<simd_mask>(__mem));
4232 template <
typename _Flags>
4233 _GLIBCXX_SIMD_ALWAYS_INLINE
void
4234 copy_to(value_type* __mem, _Flags)
const
4235 { _Impl::_S_store(_M_data, _Flags::template _S_apply<simd_mask>(__mem)); }
4239 _GLIBCXX_SIMD_ALWAYS_INLINE reference
4240 operator[](
size_t __i)
4243 __invoke_ub(
"Subscript %d is out of range [0, %d]", __i,
size() - 1);
4244 return {_M_data, int(__i)};
4247 _GLIBCXX_SIMD_ALWAYS_INLINE value_type
4248 operator[](
size_t __i)
const
4251 __invoke_ub(
"Subscript %d is out of range [0, %d]", __i,
size() - 1);
4252 if constexpr (__is_scalar_abi<_Abi>())
4255 return static_cast<
bool>(_M_data[__i]);
4260 _GLIBCXX_SIMD_ALWAYS_INLINE simd_mask
4262 {
return {__private_init, _Impl::_S_bit_not(_M_data)}; }
4266 #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4268 template <
typename _Up,
typename _A2,
4270 = enable_if_t<is_convertible_v<simd_mask<_Up, _A2>, simd_mask>>>
4271 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask
4272 operator&&(
const simd_mask& __x,
const simd_mask<_Up, _A2>& __y)
4274 return {__private_init,
4275 _Impl::_S_logical_and(__x._M_data, simd_mask(__y)._M_data)};
4278 template <
typename _Up,
typename _A2,
4280 = enable_if_t<is_convertible_v<simd_mask<_Up, _A2>, simd_mask>>>
4281 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask
4282 operator||(
const simd_mask& __x,
const simd_mask<_Up, _A2>& __y)
4284 return {__private_init,
4285 _Impl::_S_logical_or(__x._M_data, simd_mask(__y)._M_data)};
4289 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask
4290 operator&&(
const simd_mask& __x,
const simd_mask& __y)
4292 return {__private_init, _Impl::_S_logical_and(__x._M_data, __y._M_data)};
4295 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask
4296 operator||(
const simd_mask& __x,
const simd_mask& __y)
4298 return {__private_init, _Impl::_S_logical_or(__x._M_data, __y._M_data)};
4301 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask
4302 operator&(
const simd_mask& __x,
const simd_mask& __y)
4303 {
return {__private_init, _Impl::_S_bit_and(__x._M_data, __y._M_data)}; }
4305 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask
4306 operator|(
const simd_mask& __x,
const simd_mask& __y)
4307 {
return {__private_init, _Impl::_S_bit_or(__x._M_data, __y._M_data)}; }
4309 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask
4310 operator^(
const simd_mask& __x,
const simd_mask& __y)
4311 {
return {__private_init, _Impl::_S_bit_xor(__x._M_data, __y._M_data)}; }
4313 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask&
4314 operator&=(simd_mask& __x,
const simd_mask& __y)
4316 __x._M_data = _Impl::_S_bit_and(__x._M_data, __y._M_data);
4320 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask&
4321 operator|=(simd_mask& __x,
const simd_mask& __y)
4323 __x._M_data = _Impl::_S_bit_or(__x._M_data, __y._M_data);
4327 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask&
4328 operator^=(simd_mask& __x,
const simd_mask& __y)
4330 __x._M_data = _Impl::_S_bit_xor(__x._M_data, __y._M_data);
4336 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask
4337 operator==(
const simd_mask& __x,
const simd_mask& __y)
4338 {
return !operator!=(__x, __y); }
4340 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask
4341 operator!=(
const simd_mask& __x,
const simd_mask& __y)
4342 {
return {__private_init, _Impl::_S_bit_xor(__x._M_data, __y._M_data)}; }
4346 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
4347 simd_mask(_PrivateInit,
typename _Traits::_MaskMember __init)
4348 : _M_data(__init) {}
4352 template <typename _Fp, typename = decltype(bool(declval<_Fp>()(size_t())))>
4353 _GLIBCXX_SIMD_INTRINSIC constexpr
4354 simd_mask(_PrivateInit, _Fp&& __gen)
4357 __execute_n_times<size()>([&](
auto __i) constexpr {
4358 _Impl::_S_set(_M_data, __i, __gen(__i));
4364 _GLIBCXX_SIMD_INTRINSIC simd_mask(_BitsetInit, bitset<
size()> __init)
4366 _Impl::_S_from_bitmask(_SanitizedBitMask<
size()>(__init), _S_type_tag))
4377 template <
typename _Up,
typename _A2,
4379 = enable_if_t<simd_size_v<_Up, _A2> == simd_size_v<_Tp, _Abi>>>
4380 operator simd_mask<_Up, _A2>() &&
4382 using namespace std::experimental::__proposed;
4383 return static_simd_cast<simd_mask<_Up, _A2>>(_M_data);
4386 const simd_mask<_Tp, _Abi>& _M_data;
4389 _GLIBCXX_SIMD_INTRINSIC _CvtProxy
4395 #ifdef __GXX_CONDITIONAL_IS_OVERLOADABLE__
4396 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask
4397 operator?:(
const simd_mask& __k,
const simd_mask& __where_true,
4398 const simd_mask& __where_false)
4400 auto __ret = __where_false;
4401 _Impl::_S_masked_assign(__k._M_data, __ret._M_data, __where_true._M_data);
4405 template <
typename _U1,
typename _U2,
4406 typename _Rp = simd<common_type_t<_U1, _U2>, _Abi>,
4408 is_convertible<_U1, _Rp>, is_convertible<_U2, _Rp>,
4409 is_convertible<simd_mask, typename _Rp::mask_type>>>>
4410 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend _Rp
4411 operator?:(
const simd_mask& __k,
const _U1& __where_true,
4412 const _U2& __where_false)
4414 _Rp __ret = __where_false;
4415 _Rp::_Impl::_S_masked_assign(
4416 __data(
static_cast<typename _Rp::mask_type
>(__k)), __data(__ret),
4417 __data(
static_cast<_Rp
>(__where_true)));
4421 #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4422 template <
typename _Kp,
typename _Ak,
typename _Up,
typename _Au,
4424 conjunction_v<is_convertible<simd_mask<_Kp, _Ak>, simd_mask>,
4425 is_convertible<simd_mask<_Up, _Au>, simd_mask>>>>
4426 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask
4427 operator?:(
const simd_mask<_Kp, _Ak>& __k,
const simd_mask& __where_true,
4428 const simd_mask<_Up, _Au>& __where_false)
4430 simd_mask __ret = __where_false;
4431 _Impl::_S_masked_assign(simd_mask(__k)._M_data, __ret._M_data,
4432 __where_true._M_data);
4440 _GLIBCXX_SIMD_INTRINSIC constexpr
bool
4441 _M_is_constprop()
const
4443 if constexpr (__is_scalar_abi<_Abi>())
4444 return __builtin_constant_p(_M_data);
4446 return _M_data._M_is_constprop();
4452 friend const auto& __data<_Tp, abi_type>(const simd_mask&);
4453 friend auto& __data<_Tp, abi_type>(simd_mask&);
4454 alignas(_Traits::_S_mask_align) _MemberType _M_data;
4460 template <typename _Tp, typename _Ap>
4461 _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
4462 __data(const simd_mask<_Tp, _Ap>& __x)
4463 {
return __x._M_data; }
4465 template <
typename _Tp,
typename _Ap>
4466 _GLIBCXX_SIMD_INTRINSIC constexpr
auto&
4467 __data(simd_mask<_Tp, _Ap>& __x)
4468 {
return __x._M_data; }
4473 template <
typename _Tp,
typename _Abi>
4474 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
4475 all_of(
const simd_mask<_Tp, _Abi>& __k) noexcept
4477 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4479 for (
size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
4485 return _Abi::_MaskImpl::_S_all_of(__k);
4488 template <
typename _Tp,
typename _Abi>
4489 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
4490 any_of(
const simd_mask<_Tp, _Abi>& __k) noexcept
4492 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4494 for (
size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
4500 return _Abi::_MaskImpl::_S_any_of(__k);
4503 template <
typename _Tp,
typename _Abi>
4504 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
4505 none_of(
const simd_mask<_Tp, _Abi>& __k) noexcept
4507 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4509 for (
size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
4515 return _Abi::_MaskImpl::_S_none_of(__k);
4518 template <
typename _Tp,
typename _Abi>
4519 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
4520 some_of(
const simd_mask<_Tp, _Abi>& __k) noexcept
4522 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4524 for (
size_t __i = 1; __i < simd_size_v<_Tp, _Abi>; ++__i)
4525 if (__k[__i] != __k[__i - 1])
4530 return _Abi::_MaskImpl::_S_some_of(__k);
4533 template <
typename _Tp,
typename _Abi>
4534 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
int
4535 popcount(
const simd_mask<_Tp, _Abi>& __k) noexcept
4537 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4539 const int __r = __call_with_subscripts<simd_size_v<_Tp, _Abi>>(
4540 __k, [](
auto... __elements) {
return ((__elements != 0) + ...); });
4541 if (__builtin_is_constant_evaluated() || __builtin_constant_p(__r))
4544 return _Abi::_MaskImpl::_S_popcount(__k);
4547 template <
typename _Tp,
typename _Abi>
4548 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
int
4549 find_first_set(
const simd_mask<_Tp, _Abi>& __k)
4551 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4553 constexpr
size_t _Np = simd_size_v<_Tp, _Abi>;
4554 const size_t _Idx = __call_with_n_evaluations<_Np>(
4555 [](
auto... __indexes) {
return std::min({__indexes...}); },
4556 [&](
auto __i) {
return __k[__i] ? +__i : _Np; });
4558 __invoke_ub(
"find_first_set(empty mask) is UB");
4559 if (__builtin_constant_p(_Idx))
4562 return _Abi::_MaskImpl::_S_find_first_set(__k);
4565 template <
typename _Tp,
typename _Abi>
4566 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
int
4567 find_last_set(
const simd_mask<_Tp, _Abi>& __k)
4569 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4571 constexpr
size_t _Np = simd_size_v<_Tp, _Abi>;
4572 const int _Idx = __call_with_n_evaluations<_Np>(
4573 [](
auto... __indexes) {
return std::max({__indexes...}); },
4574 [&](
auto __i) {
return __k[__i] ? int(__i) : -1; });
4576 __invoke_ub(
"find_first_set(empty mask) is UB");
4577 if (__builtin_constant_p(_Idx))
4580 return _Abi::_MaskImpl::_S_find_last_set(__k);
4583 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
4584 all_of(_ExactBool __x) noexcept
4587 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
4588 any_of(_ExactBool __x) noexcept
4591 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
4592 none_of(_ExactBool __x) noexcept
4595 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
4596 some_of(_ExactBool) noexcept
4599 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
int
4600 popcount(_ExactBool __x) noexcept
4603 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
int
4604 find_first_set(_ExactBool)
4607 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
int
4608 find_last_set(_ExactBool)
4614 template <
typename _V,
typename _Impl,
bool>
4615 class _SimdIntOperators {};
4617 template <
typename _V,
typename _Impl>
4618 class _SimdIntOperators<_V, _Impl, true>
4620 _GLIBCXX_SIMD_INTRINSIC
const _V& __derived()
const
4621 {
return *
static_cast<const _V*
>(
this); }
4623 template <
typename _Tp>
4624 _GLIBCXX_SIMD_INTRINSIC
static _GLIBCXX_SIMD_CONSTEXPR _V
4625 _S_make_derived(_Tp&& __d)
4626 {
return {__private_init,
static_cast<_Tp&&
>(__d)}; }
4629 _GLIBCXX_SIMD_CONSTEXPR
friend _V& operator%=(_V& __lhs,
const _V& __x)
4630 {
return __lhs = __lhs % __x; }
4632 _GLIBCXX_SIMD_CONSTEXPR
friend _V& operator&=(_V& __lhs,
const _V& __x)
4633 {
return __lhs = __lhs & __x; }
4635 _GLIBCXX_SIMD_CONSTEXPR
friend _V& operator|=(_V& __lhs,
const _V& __x)
4636 {
return __lhs = __lhs | __x; }
4638 _GLIBCXX_SIMD_CONSTEXPR
friend _V& operator^=(_V& __lhs,
const _V& __x)
4639 {
return __lhs = __lhs ^ __x; }
4641 _GLIBCXX_SIMD_CONSTEXPR
friend _V& operator<<=(_V& __lhs,
const _V& __x)
4642 {
return __lhs = __lhs << __x; }
4644 _GLIBCXX_SIMD_CONSTEXPR
friend _V& operator>>=(_V& __lhs,
const _V& __x)
4645 {
return __lhs = __lhs >> __x; }
4647 _GLIBCXX_SIMD_CONSTEXPR
friend _V& operator<<=(_V& __lhs,
int __x)
4648 {
return __lhs = __lhs << __x; }
4650 _GLIBCXX_SIMD_CONSTEXPR
friend _V& operator>>=(_V& __lhs,
int __x)
4651 {
return __lhs = __lhs >> __x; }
4653 _GLIBCXX_SIMD_CONSTEXPR
friend _V operator%(
const _V& __x,
const _V& __y)
4655 return _SimdIntOperators::_S_make_derived(
4656 _Impl::_S_modulus(__data(__x), __data(__y)));
4659 _GLIBCXX_SIMD_CONSTEXPR
friend _V
operator&(
const _V& __x,
const _V& __y)
4661 return _SimdIntOperators::_S_make_derived(
4662 _Impl::_S_bit_and(__data(__x), __data(__y)));
4665 _GLIBCXX_SIMD_CONSTEXPR
friend _V operator|(
const _V& __x,
const _V& __y)
4667 return _SimdIntOperators::_S_make_derived(
4668 _Impl::_S_bit_or(__data(__x), __data(__y)));
4671 _GLIBCXX_SIMD_CONSTEXPR
friend _V operator^(
const _V& __x,
const _V& __y)
4673 return _SimdIntOperators::_S_make_derived(
4674 _Impl::_S_bit_xor(__data(__x), __data(__y)));
4677 _GLIBCXX_SIMD_CONSTEXPR
friend _V operator<<(
const _V& __x,
const _V& __y)
4679 return _SimdIntOperators::_S_make_derived(
4680 _Impl::_S_bit_shift_left(__data(__x), __data(__y)));
4683 _GLIBCXX_SIMD_CONSTEXPR
friend _V
operator>>(
const _V& __x,
const _V& __y)
4685 return _SimdIntOperators::_S_make_derived(
4686 _Impl::_S_bit_shift_right(__data(__x), __data(__y)));
4689 template <
typename _VV = _V>
4690 _GLIBCXX_SIMD_CONSTEXPR
friend _V operator<<(
const _V& __x,
int __y)
4692 using _Tp =
typename _VV::value_type;
4694 __invoke_ub(
"The behavior is undefined if the right operand of a "
4695 "shift operation is negative. [expr.shift]\nA shift by "
4698 if (
size_t(__y) >=
sizeof(declval<_Tp>() << __y) * __CHAR_BIT__)
4700 "The behavior is undefined if the right operand of a "
4701 "shift operation is greater than or equal to the width of the "
4702 "promoted left operand. [expr.shift]\nA shift by %d was requested",
4704 return _SimdIntOperators::_S_make_derived(
4705 _Impl::_S_bit_shift_left(__data(__x), __y));
4708 template <
typename _VV = _V>
4709 _GLIBCXX_SIMD_CONSTEXPR
friend _V
operator>>(
const _V& __x,
int __y)
4711 using _Tp =
typename _VV::value_type;
4714 "The behavior is undefined if the right operand of a shift "
4715 "operation is negative. [expr.shift]\nA shift by %d was requested",
4717 if (
size_t(__y) >=
sizeof(declval<_Tp>() << __y) * __CHAR_BIT__)
4719 "The behavior is undefined if the right operand of a shift "
4720 "operation is greater than or equal to the width of the promoted "
4721 "left operand. [expr.shift]\nA shift by %d was requested",
4723 return _SimdIntOperators::_S_make_derived(
4724 _Impl::_S_bit_shift_right(__data(__x), __y));
4728 _GLIBCXX_SIMD_CONSTEXPR _V operator~()
const
4729 {
return {__private_init, _Impl::_S_complement(__derived()._M_data)}; }
4735 template <
typename _Tp,
typename _Abi>
4736 class simd :
public _SimdIntOperators<
4737 simd<_Tp, _Abi>, typename _SimdTraits<_Tp, _Abi>::_SimdImpl,
4738 conjunction<is_integral<_Tp>,
4739 typename _SimdTraits<_Tp, _Abi>::_IsValid>::value>,
4740 public _SimdTraits<_Tp, _Abi>::_SimdBase
4742 using _Traits = _SimdTraits<_Tp, _Abi>;
4743 using _MemberType =
typename _Traits::_SimdMember;
4744 using _CastType =
typename _Traits::_SimdCastType;
4745 static constexpr _Tp* _S_type_tag =
nullptr;
4746 friend typename _Traits::_SimdBase;
4749 using _Impl =
typename _Traits::_SimdImpl;
4751 friend _SimdIntOperators<simd, _Impl, true>;
4753 using value_type = _Tp;
4754 using reference = _SmartReference<_MemberType, _Impl, value_type>;
4755 using mask_type = simd_mask<_Tp, _Abi>;
4756 using abi_type = _Abi;
4758 static constexpr
size_t size()
4759 {
return __size_or_zero_v<_Tp, _Abi>; }
4761 _GLIBCXX_SIMD_CONSTEXPR simd() =
default;
4762 _GLIBCXX_SIMD_CONSTEXPR simd(
const simd&) =
default;
4763 _GLIBCXX_SIMD_CONSTEXPR simd(simd&&) noexcept = default;
4764 _GLIBCXX_SIMD_CONSTEXPR simd& operator=(const simd&) = default;
4765 _GLIBCXX_SIMD_CONSTEXPR simd& operator=(simd&&) noexcept = default;
4768 template <typename _Up,
4769 typename =
enable_if_t<!is_same_v<__remove_cvref_t<_Up>,
bool>>>
4770 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
4771 simd(_ValuePreservingOrInt<_Up, value_type>&& __x)
4773 _Impl::_S_broadcast(static_cast<value_type>(static_cast<_Up&&>(__x))))
4778 template <
typename _Up>
4779 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
4780 simd(
const simd<_Up, simd_abi::fixed_size<
size()>>& __x,
4783 is_same<simd_abi::fixed_size<
size()>, abi_type>,
4784 negation<__is_narrowing_conversion<_Up, value_type>>,
4785 __converts_to_higher_integer_rank<_Up, value_type>>::value,
4787 : simd{static_cast<array<_Up,
size()>>(__x).
data(), vector_aligned} {}
4790 #ifdef _GLIBCXX_SIMD_ENABLE_STATIC_CAST
4791 template <
typename _Up,
typename _A2,
4792 typename = decltype(static_simd_cast<simd>(
4793 declval<
const simd<_Up, _A2>&>()))>
4794 _GLIBCXX_SIMD_ALWAYS_INLINE
explicit _GLIBCXX_SIMD_CONSTEXPR
4795 simd(
const simd<_Up, _A2>& __x)
4796 : simd(static_simd_cast<simd>(__x)) {}
4800 template <
typename _Fp>
4801 _GLIBCXX_SIMD_ALWAYS_INLINE
explicit _GLIBCXX_SIMD_CONSTEXPR
4802 simd(_Fp&& __gen, _ValuePreservingOrInt<decltype(declval<_Fp>()(
4803 declval<_SizeConstant<0>&>())),
4804 value_type>* =
nullptr)
4805 : _M_data(_Impl::_S_generator(static_cast<_Fp&&>(__gen), _S_type_tag)) {}
4808 template <
typename _Up,
typename _Flags>
4809 _GLIBCXX_SIMD_ALWAYS_INLINE
4810 simd(
const _Up* __mem, _Flags)
4812 _Impl::_S_load(_Flags::template _S_apply<simd>(__mem), _S_type_tag))
4816 template <
typename _Up,
typename _Flags>
4817 _GLIBCXX_SIMD_ALWAYS_INLINE
void
4818 copy_from(
const _Vectorizable<_Up>* __mem, _Flags)
4820 _M_data =
static_cast<decltype(_M_data)
>(
4821 _Impl::_S_load(_Flags::template _S_apply<simd>(__mem), _S_type_tag));
4825 template <
typename _Up,
typename _Flags>
4826 _GLIBCXX_SIMD_ALWAYS_INLINE
void
4827 copy_to(_Vectorizable<_Up>* __mem, _Flags)
const
4829 _Impl::_S_store(_M_data, _Flags::template _S_apply<simd>(__mem),
4834 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR reference
4835 operator[](
size_t __i)
4836 {
return {_M_data, int(__i)}; }
4838 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR value_type
4839 operator[]([[maybe_unused]]
size_t __i)
const
4841 if constexpr (__is_scalar_abi<_Abi>())
4843 _GLIBCXX_DEBUG_ASSERT(__i == 0);
4847 return _M_data[__i];
4851 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd&
4854 _Impl::_S_increment(_M_data);
4858 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
4862 _Impl::_S_increment(_M_data);
4866 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd&
4869 _Impl::_S_decrement(_M_data);
4873 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
4877 _Impl::_S_decrement(_M_data);
4882 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR mask_type
4884 {
return {__private_init, _Impl::_S_negate(_M_data)}; }
4886 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
4890 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
4892 {
return {__private_init, _Impl::_S_unary_minus(_M_data)}; }
4895 _GLIBCXX_SIMD_ALWAYS_INLINE
explicit _GLIBCXX_SIMD_CONSTEXPR
4896 simd(_CastType __init) : _M_data(__init) {}
4899 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd&
4900 operator+=(simd& __lhs,
const simd& __x)
4901 {
return __lhs = __lhs + __x; }
4903 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd&
4904 operator-=(simd& __lhs,
const simd& __x)
4905 {
return __lhs = __lhs - __x; }
4907 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd&
4908 operator*=(simd& __lhs,
const simd& __x)
4909 {
return __lhs = __lhs * __x; }
4911 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd&
4912 operator/=(simd& __lhs,
const simd& __x)
4913 {
return __lhs = __lhs / __x; }
4916 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd
4917 operator+(
const simd& __x,
const simd& __y)
4918 {
return {__private_init, _Impl::_S_plus(__x._M_data, __y._M_data)}; }
4920 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd
4921 operator-(
const simd& __x,
const simd& __y)
4922 {
return {__private_init, _Impl::_S_minus(__x._M_data, __y._M_data)}; }
4924 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd
4925 operator*(
const simd& __x,
const simd& __y)
4926 {
return {__private_init, _Impl::_S_multiplies(__x._M_data, __y._M_data)}; }
4928 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd
4929 operator/(
const simd& __x,
const simd& __y)
4930 {
return {__private_init, _Impl::_S_divides(__x._M_data, __y._M_data)}; }
4933 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend mask_type
4934 operator==(
const simd& __x,
const simd& __y)
4935 {
return simd::_S_make_mask(_Impl::_S_equal_to(__x._M_data, __y._M_data)); }
4937 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend mask_type
4938 operator!=(
const simd& __x,
const simd& __y)
4940 return simd::_S_make_mask(
4941 _Impl::_S_not_equal_to(__x._M_data, __y._M_data));
4944 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend mask_type
4945 operator<(
const simd& __x,
const simd& __y)
4946 {
return simd::_S_make_mask(_Impl::_S_less(__x._M_data, __y._M_data)); }
4948 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend mask_type
4949 operator<=(
const simd& __x,
const simd& __y)
4951 return simd::_S_make_mask(_Impl::_S_less_equal(__x._M_data, __y._M_data));
4954 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend mask_type
4955 operator>(
const simd& __x,
const simd& __y)
4956 {
return simd::_S_make_mask(_Impl::_S_less(__y._M_data, __x._M_data)); }
4958 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend mask_type
4959 operator>=(
const simd& __x,
const simd& __y)
4961 return simd::_S_make_mask(_Impl::_S_less_equal(__y._M_data, __x._M_data));
4965 #ifdef __GXX_CONDITIONAL_IS_OVERLOADABLE__
4966 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd
4967 operator?:(
const mask_type& __k,
const simd& __where_true,
4968 const simd& __where_false)
4970 auto __ret = __where_false;
4971 _Impl::_S_masked_assign(__data(__k), __data(__ret), __data(__where_true));
4979 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
4980 simd(_PrivateInit,
const _MemberType& __init)
4981 : _M_data(__init) {}
4984 _GLIBCXX_SIMD_INTRINSIC
4985 simd(_BitsetInit, bitset<
size()> __init) : _M_data()
4986 { where(mask_type(__bitset_init, __init), *
this) = ~*
this; }
4988 _GLIBCXX_SIMD_INTRINSIC constexpr
bool
4989 _M_is_constprop()
const
4991 if constexpr (__is_scalar_abi<_Abi>())
4992 return __builtin_constant_p(_M_data);
4994 return _M_data._M_is_constprop();
4998 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR static mask_type
4999 _S_make_mask(typename mask_type::_MemberType __k)
5000 {
return {__private_init, __k}; }
5002 friend const auto& __data<value_type, abi_type>(
const simd&);
5003 friend auto& __data<value_type, abi_type>(simd&);
5004 alignas(_Traits::_S_simd_align) _MemberType _M_data;
5009 template <
typename _Tp,
typename _Ap>
5010 _GLIBCXX_SIMD_INTRINSIC constexpr
const auto&
5011 __data(
const simd<_Tp, _Ap>& __x)
5012 {
return __x._M_data; }
5014 template <
typename _Tp,
typename _Ap>
5015 _GLIBCXX_SIMD_INTRINSIC constexpr
auto&
5016 __data(simd<_Tp, _Ap>& __x)
5017 {
return __x._M_data; }
5020 namespace __float_bitwise_operators {
5021 template <
typename _Tp,
typename _Ap>
5022 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5023 operator^(
const simd<_Tp, _Ap>& __a,
const simd<_Tp, _Ap>& __b)
5025 return {__private_init,
5026 _Ap::_SimdImpl::_S_bit_xor(__data(__a), __data(__b))};
5029 template <
typename _Tp,
typename _Ap>
5030 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5031 operator|(
const simd<_Tp, _Ap>& __a,
const simd<_Tp, _Ap>& __b)
5033 return {__private_init,
5034 _Ap::_SimdImpl::_S_bit_or(__data(__a), __data(__b))};
5037 template <
typename _Tp,
typename _Ap>
5038 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5039 operator&(
const simd<_Tp, _Ap>& __a,
const simd<_Tp, _Ap>& __b)
5041 return {__private_init,
5042 _Ap::_SimdImpl::_S_bit_and(__data(__a), __data(__b))};
5046 _GLIBCXX_SIMD_END_NAMESPACE
auto_ptr & operator=(auto_ptr &__a)
auto_ptr assignment operator.
element_type * operator->() const
Smart pointer dereferencing.
element_type & operator*() const
Smart pointer dereferencing.
constexpr complex< _Tp > operator/(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x divided by y.
constexpr complex< _Tp > operator-(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x minus y.
constexpr complex< _Tp > operator+(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x plus y.
typename remove_reference< _Tp >::type remove_reference_t
Alias template for remove_reference.
typename make_unsigned< _Tp >::type make_unsigned_t
Alias template for make_unsigned.
void void_t
A metafunction that always yields void, used for detecting valid types.
typename conditional< _Cond, _Iftrue, _Iffalse >::type conditional_t
Alias template for conditional.
typename remove_pointer< _Tp >::type remove_pointer_t
Alias template for remove_pointer.
typename remove_const< _Tp >::type remove_const_t
Alias template for remove_const.
integral_constant< bool, true > true_type
The type used as a compile-time boolean with true value.
typename enable_if< _Cond, _Tp >::type enable_if_t
Alias template for enable_if.
integral_constant< bool, false > false_type
The type used as a compile-time boolean with false value.
constexpr auto tuple_cat(_Tpls &&... __tpls) -> typename __tuple_cat_result< _Tpls... >::__type
tuple_cat
constexpr std::remove_reference< _Tp >::type && move(_Tp &&__t) noexcept
Convert a value to an rvalue.
void swap(any &__x, any &__y) noexcept
Exchange the states of two any objects.
_Tp * begin(valarray< _Tp > &__va)
Return an iterator pointing to the first element of the valarray.
_Tp * end(valarray< _Tp > &__va)
Return an iterator pointing to one past the last element of the valarray.
constexpr const _Tp & clamp(const _Tp &, const _Tp &, const _Tp &)
Returns the value clamped between lo and hi.
constexpr const _Tp & max(const _Tp &, const _Tp &)
This does what you think it does.
constexpr pair< const _Tp &, const _Tp & > minmax(const _Tp &, const _Tp &)
Determines min and max at once as an ordered pair.
constexpr const _Tp & min(const _Tp &, const _Tp &)
This does what you think it does.
constexpr _Tp reduce(_InputIterator __first, _InputIterator __last, _Tp __init, _BinaryOperation __binary_op)
Calculate reduction of values in a range.
std::basic_istream< _CharT, _Traits > & operator>>(std::basic_istream< _CharT, _Traits > &__is, bitset< _Nb > &__x)
Global I/O operators for bitsets.
bitset< _Nb > operator&(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
constexpr auto size(const _Container &__cont) noexcept(noexcept(__cont.size())) -> decltype(__cont.size())
Return the size of a container.
constexpr auto data(_Container &__cont) noexcept(noexcept(__cont.data())) -> decltype(__cont.data())
Return the data pointer of a container.