25 #ifndef _GLIBCXX_EXPERIMENTAL_SIMD_H
26 #define _GLIBCXX_EXPERIMENTAL_SIMD_H
28 #if __cplusplus >= 201703L
30 #include "simd_detail.h"
34 #ifdef _GLIBCXX_DEBUG_UB
42 #if _GLIBCXX_SIMD_X86INTRIN
43 #include <x86intrin.h>
44 #elif _GLIBCXX_SIMD_HAVE_NEON
71 _GLIBCXX_SIMD_BEGIN_NAMESPACE
73 #if !_GLIBCXX_SIMD_X86INTRIN
74 using __m128 [[__gnu__::__vector_size__(16)]] = float;
75 using __m128d [[__gnu__::__vector_size__(16)]] = double;
76 using __m128i [[__gnu__::__vector_size__(16)]] =
long long;
77 using __m256 [[__gnu__::__vector_size__(32)]] = float;
78 using __m256d [[__gnu__::__vector_size__(32)]] = double;
79 using __m256i [[__gnu__::__vector_size__(32)]] =
long long;
80 using __m512 [[__gnu__::__vector_size__(64)]] = float;
81 using __m512d [[__gnu__::__vector_size__(64)]] = double;
82 using __m512i [[__gnu__::__vector_size__(64)]] =
long long;
104 template <
int _UsedBytes>
107 template <
int _UsedBytes>
108 struct _VecBltnBtmsk;
110 template <
typename _Tp,
int _Np>
111 using _VecN = _VecBuiltin<
sizeof(_Tp) * _Np>;
113 template <
int _UsedBytes = 16>
114 using _Sse = _VecBuiltin<_UsedBytes>;
116 template <
int _UsedBytes = 32>
117 using _Avx = _VecBuiltin<_UsedBytes>;
119 template <
int _UsedBytes = 64>
120 using _Avx512 = _VecBltnBtmsk<_UsedBytes>;
122 template <
int _UsedBytes = 16>
123 using _Neon = _VecBuiltin<_UsedBytes>;
126 using __sse = _Sse<>;
127 using __avx = _Avx<>;
128 using __avx512 = _Avx512<>;
129 using __neon = _Neon<>;
130 using __neon128 = _Neon<16>;
131 using __neon64 = _Neon<8>;
134 template <
typename _Tp,
size_t _Np,
typename...>
138 using fixed_size = _Fixed<_Np>;
140 using scalar = _Scalar;
145 template <
typename _Tp>
148 template <
typename _Tp>
151 template <
typename _Tp,
typename _Abi>
154 template <
typename _Tp,
typename _Abi>
157 template <
typename _Tp,
typename _Abi>
162 struct element_aligned_tag
164 template <
typename _Tp,
typename _Up =
typename _Tp::value_type>
165 static constexpr
size_t _S_alignment =
alignof(_Up);
167 template <
typename _Tp,
typename _Up>
168 _GLIBCXX_SIMD_INTRINSIC
static constexpr _Up*
173 struct vector_aligned_tag
175 template <
typename _Tp,
typename _Up =
typename _Tp::value_type>
176 static constexpr
size_t _S_alignment
177 = std::__bit_ceil(
sizeof(_Up) * _Tp::size());
179 template <
typename _Tp,
typename _Up>
180 _GLIBCXX_SIMD_INTRINSIC
static constexpr _Up*
183 return static_cast<_Up*
>(
184 __builtin_assume_aligned(__ptr, _S_alignment<_Tp, _Up>));
188 template <
size_t _Np>
struct overaligned_tag
190 template <
typename _Tp,
typename _Up =
typename _Tp::value_type>
191 static constexpr
size_t _S_alignment = _Np;
193 template <
typename _Tp,
typename _Up>
194 _GLIBCXX_SIMD_INTRINSIC
static constexpr _Up*
196 {
return static_cast<_Up*
>(__builtin_assume_aligned(__ptr, _Np)); }
199 inline constexpr element_aligned_tag element_aligned = {};
201 inline constexpr vector_aligned_tag vector_aligned = {};
203 template <
size_t _Np>
204 inline constexpr overaligned_tag<_Np> overaligned = {};
207 template <
size_t _Xp>
208 using _SizeConstant = integral_constant<size_t, _Xp>;
214 template <
typename _Tp>
215 _GLIBCXX_SIMD_INTRINSIC constexpr
217 operator()(_Tp __a, _Tp __b)
const
220 return min(__a, __b);
226 template <
typename _Tp>
227 _GLIBCXX_SIMD_INTRINSIC constexpr
229 operator()(_Tp __a, _Tp __b)
const
232 return max(__a, __b);
239 template <
typename _Fp,
size_t... _I>
240 [[__gnu__::__flatten__]] _GLIBCXX_SIMD_INTRINSIC constexpr
242 __execute_on_index_sequence(_Fp&& __f, index_sequence<_I...>)
243 { ((void)__f(_SizeConstant<_I>()), ...); }
245 template <
typename _Fp>
246 _GLIBCXX_SIMD_INTRINSIC constexpr
void
247 __execute_on_index_sequence(_Fp&&, index_sequence<>)
250 template <
size_t _Np,
typename _Fp>
251 _GLIBCXX_SIMD_INTRINSIC constexpr
void
252 __execute_n_times(_Fp&& __f)
254 __execute_on_index_sequence(
static_cast<_Fp&&
>(__f),
255 make_index_sequence<_Np>{});
260 template <
typename _R,
typename _Fp,
size_t... _I>
261 [[__gnu__::__flatten__]] _GLIBCXX_SIMD_INTRINSIC constexpr
263 __execute_on_index_sequence_with_return(_Fp&& __f, index_sequence<_I...>)
264 {
return _R{__f(_SizeConstant<_I>())...}; }
266 template <
size_t _Np,
typename _R,
typename _Fp>
267 _GLIBCXX_SIMD_INTRINSIC constexpr _R
268 __generate_from_n_evaluations(_Fp&& __f)
270 return __execute_on_index_sequence_with_return<_R>(
271 static_cast<_Fp&&
>(__f), make_index_sequence<_Np>{});
276 template <
size_t... _I,
typename _F0,
typename _FArgs>
277 [[__gnu__::__flatten__]] _GLIBCXX_SIMD_INTRINSIC constexpr
279 __call_with_n_evaluations(index_sequence<_I...>, _F0&& __f0, _FArgs&& __fargs)
280 {
return __f0(__fargs(_SizeConstant<_I>())...); }
282 template <
size_t _Np,
typename _F0,
typename _FArgs>
283 _GLIBCXX_SIMD_INTRINSIC constexpr
auto
284 __call_with_n_evaluations(_F0&& __f0, _FArgs&& __fargs)
286 return __call_with_n_evaluations(make_index_sequence<_Np>{},
287 static_cast<_F0&&
>(__f0),
288 static_cast<_FArgs&&
>(__fargs));
293 template <
size_t _First = 0,
size_t... _It,
typename _Tp,
typename _Fp>
294 [[__gnu__::__flatten__]] _GLIBCXX_SIMD_INTRINSIC constexpr
296 __call_with_subscripts(_Tp&& __x, index_sequence<_It...>, _Fp&& __fun)
297 {
return __fun(__x[_First + _It]...); }
299 template <
size_t _Np,
size_t _First = 0,
typename _Tp,
typename _Fp>
300 _GLIBCXX_SIMD_INTRINSIC constexpr
auto
301 __call_with_subscripts(_Tp&& __x, _Fp&& __fun)
303 return __call_with_subscripts<_First>(
static_cast<_Tp&&
>(__x),
304 make_index_sequence<_Np>(),
305 static_cast<_Fp&&
>(__fun));
312 using _UChar =
unsigned char;
313 using _SChar =
signed char;
314 using _UShort =
unsigned short;
315 using _UInt =
unsigned int;
316 using _ULong =
unsigned long;
317 using _ULLong =
unsigned long long;
318 using _LLong =
long long;
322 template <
typename _T0,
typename...>
323 struct __first_of_pack
324 {
using type = _T0; };
326 template <
typename... _Ts>
327 using __first_of_pack_t =
typename __first_of_pack<_Ts...>::type;
331 template <
typename _Tp>
332 typename _Tp::value_type
333 __value_type_or_identity_impl(
int);
335 template <
typename _Tp>
337 __value_type_or_identity_impl(
float);
339 template <
typename _Tp>
340 using __value_type_or_identity_t
341 = decltype(__value_type_or_identity_impl<_Tp>(
int()));
345 template <
typename _Tp>
346 struct __is_vectorizable :
public is_arithmetic<_Tp> {};
349 struct __is_vectorizable<bool> :
public false_type {};
351 template <
typename _Tp>
352 inline constexpr
bool __is_vectorizable_v = __is_vectorizable<_Tp>::value;
355 template <
typename _Tp,
typename = enable_if_t<__is_vectorizable_v<_Tp>>>
356 using _Vectorizable = _Tp;
360 template <
typename _Ptr,
typename _ValueType>
361 struct __is_possible_loadstore_conversion
362 : conjunction<__is_vectorizable<_Ptr>, __is_vectorizable<_ValueType>> {};
365 struct __is_possible_loadstore_conversion<bool, bool> :
true_type {};
368 template <
typename _Ptr,
typename _ValueType,
370 __is_possible_loadstore_conversion<_Ptr, _ValueType>::value>>
371 using _LoadStorePtr = _Ptr;
375 template <
typename _Tp,
typename =
void_t<>>
378 template <
typename _Tp>
379 inline constexpr
bool __is_bitmask_v = __is_bitmask<_Tp>::value;
382 template <
typename _Tp>
383 struct __is_bitmask<_Tp,
384 void_t<decltype(
declval<unsigned&>() = declval<_Tp>() & 1u)>>
389 #pragma GCC diagnostic push
390 #pragma GCC diagnostic ignored "-Wpedantic"
391 template <
size_t _Bytes>
395 if constexpr (_Bytes ==
sizeof(
int))
398 else if constexpr (_Bytes ==
sizeof(
char))
401 else if constexpr (_Bytes ==
sizeof(_SChar))
404 else if constexpr (_Bytes ==
sizeof(
short))
407 else if constexpr (_Bytes ==
sizeof(
long))
410 else if constexpr (_Bytes ==
sizeof(_LLong))
412 #ifdef __SIZEOF_INT128__
413 else if constexpr (_Bytes ==
sizeof(__int128))
416 else if constexpr (_Bytes %
sizeof(
int) == 0)
418 constexpr
size_t _Np = _Bytes /
sizeof(int);
423 _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
426 return __generate_from_n_evaluations<_Np, _Ip>(
427 [&](
auto __i) {
return __rhs._M_data[__i] & _M_data[__i]; });
430 _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
433 return __generate_from_n_evaluations<_Np, _Ip>(
434 [&](
auto __i) {
return __rhs._M_data[__i] | _M_data[__i]; });
437 _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
440 return __generate_from_n_evaluations<_Np, _Ip>(
441 [&](
auto __i) {
return __rhs._M_data[__i] ^ _M_data[__i]; });
444 _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
447 return __generate_from_n_evaluations<_Np, _Ip>(
448 [&](
auto __i) {
return ~_M_data[__i]; });
454 static_assert(_Bytes != _Bytes,
"this should be unreachable");
456 #pragma GCC diagnostic pop
458 template <
typename _Tp>
459 using __int_for_sizeof_t = decltype(__int_for_sizeof<
sizeof(_Tp)>());
461 template <
size_t _Np>
462 using __int_with_sizeof_t = decltype(__int_for_sizeof<_Np>());
466 template <
typename _Tp>
470 struct __is_fixed_size_abi<simd_abi::fixed_size<_Np>> :
true_type {};
472 template <
typename _Tp>
473 inline constexpr
bool __is_fixed_size_abi_v = __is_fixed_size_abi<_Tp>::value;
477 constexpr
inline bool __have_mmx = _GLIBCXX_SIMD_HAVE_MMX;
478 constexpr
inline bool __have_sse = _GLIBCXX_SIMD_HAVE_SSE;
479 constexpr
inline bool __have_sse2 = _GLIBCXX_SIMD_HAVE_SSE2;
480 constexpr
inline bool __have_sse3 = _GLIBCXX_SIMD_HAVE_SSE3;
481 constexpr
inline bool __have_ssse3 = _GLIBCXX_SIMD_HAVE_SSSE3;
482 constexpr
inline bool __have_sse4_1 = _GLIBCXX_SIMD_HAVE_SSE4_1;
483 constexpr
inline bool __have_sse4_2 = _GLIBCXX_SIMD_HAVE_SSE4_2;
484 constexpr
inline bool __have_xop = _GLIBCXX_SIMD_HAVE_XOP;
485 constexpr
inline bool __have_avx = _GLIBCXX_SIMD_HAVE_AVX;
486 constexpr
inline bool __have_avx2 = _GLIBCXX_SIMD_HAVE_AVX2;
487 constexpr
inline bool __have_bmi = _GLIBCXX_SIMD_HAVE_BMI1;
488 constexpr
inline bool __have_bmi2 = _GLIBCXX_SIMD_HAVE_BMI2;
489 constexpr
inline bool __have_lzcnt = _GLIBCXX_SIMD_HAVE_LZCNT;
490 constexpr
inline bool __have_sse4a = _GLIBCXX_SIMD_HAVE_SSE4A;
491 constexpr
inline bool __have_fma = _GLIBCXX_SIMD_HAVE_FMA;
492 constexpr
inline bool __have_fma4 = _GLIBCXX_SIMD_HAVE_FMA4;
493 constexpr
inline bool __have_f16c = _GLIBCXX_SIMD_HAVE_F16C;
494 constexpr
inline bool __have_popcnt = _GLIBCXX_SIMD_HAVE_POPCNT;
495 constexpr
inline bool __have_avx512f = _GLIBCXX_SIMD_HAVE_AVX512F;
496 constexpr
inline bool __have_avx512dq = _GLIBCXX_SIMD_HAVE_AVX512DQ;
497 constexpr
inline bool __have_avx512vl = _GLIBCXX_SIMD_HAVE_AVX512VL;
498 constexpr
inline bool __have_avx512bw = _GLIBCXX_SIMD_HAVE_AVX512BW;
499 constexpr
inline bool __have_avx512dq_vl = __have_avx512dq && __have_avx512vl;
500 constexpr
inline bool __have_avx512bw_vl = __have_avx512bw && __have_avx512vl;
502 constexpr
inline bool __have_neon = _GLIBCXX_SIMD_HAVE_NEON;
503 constexpr
inline bool __have_neon_a32 = _GLIBCXX_SIMD_HAVE_NEON_A32;
504 constexpr
inline bool __have_neon_a64 = _GLIBCXX_SIMD_HAVE_NEON_A64;
505 constexpr
inline bool __support_neon_float =
506 #if defined __GCC_IEC_559
508 #elif defined __FAST_MATH__
515 constexpr
inline bool __have_power10vec =
true;
517 constexpr
inline bool __have_power10vec =
false;
519 #ifdef __POWER9_VECTOR__
520 constexpr
inline bool __have_power9vec =
true;
522 constexpr
inline bool __have_power9vec =
false;
524 #if defined __POWER8_VECTOR__
525 constexpr
inline bool __have_power8vec =
true;
527 constexpr
inline bool __have_power8vec = __have_power9vec;
530 constexpr
inline bool __have_power_vsx =
true;
532 constexpr
inline bool __have_power_vsx = __have_power8vec;
534 #if defined __ALTIVEC__
535 constexpr
inline bool __have_power_vmx =
true;
537 constexpr
inline bool __have_power_vmx = __have_power_vsx;
542 template <
typename _Abi>
545 {
return is_same_v<simd_abi::scalar, _Abi>; }
549 template <
template <
int>
class _Abi,
int _Bytes>
551 __abi_bytes_impl(_Abi<_Bytes>*)
554 template <
typename _Tp>
556 __abi_bytes_impl(_Tp*)
559 template <
typename _Abi>
560 inline constexpr
int __abi_bytes_v
561 = __abi_bytes_impl(
static_cast<_Abi*
>(
nullptr));
565 template <
typename _Abi>
567 __is_builtin_bitmask_abi()
568 {
return is_same_v<simd_abi::_VecBltnBtmsk<__abi_bytes_v<_Abi>>, _Abi>; }
572 template <
typename _Abi>
576 constexpr
auto _Bytes = __abi_bytes_v<_Abi>;
577 return _Bytes <= 16 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
582 template <
typename _Abi>
586 constexpr
auto _Bytes = __abi_bytes_v<_Abi>;
587 return _Bytes > 16 && _Bytes <= 32
588 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
593 template <
typename _Abi>
597 constexpr
auto _Bytes = __abi_bytes_v<_Abi>;
598 return _Bytes <= 64 && is_same_v<simd_abi::_Avx512<_Bytes>, _Abi>;
603 template <
typename _Abi>
607 constexpr
auto _Bytes = __abi_bytes_v<_Abi>;
608 return _Bytes <= 16 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
613 template <
typename,
typename _Up>
614 struct __make_dependent
615 {
using type = _Up; };
617 template <
typename _Tp,
typename _Up>
618 using __make_dependent_t =
typename __make_dependent<_Tp, _Up>::type;
624 template <
typename... _Args>
625 [[noreturn]] _GLIBCXX_SIMD_ALWAYS_INLINE
void
626 __invoke_ub([[maybe_unused]]
const char* __msg,
627 [[maybe_unused]]
const _Args&... __args)
629 #ifdef _GLIBCXX_DEBUG_UB
630 __builtin_fprintf(stderr, __msg, __args...);
633 __builtin_unreachable();
639 template <
typename _Tp>
640 struct __assert_unreachable
641 { static_assert(!is_same_v<_Tp, _Tp>,
"this should be unreachable"); };
645 template <typename _Tp, typename _Ap, size_t _Np = simd_size<_Tp, _Ap>::value>
647 __size_or_zero_dispatch(
int)
650 template <
typename _Tp,
typename _Ap>
652 __size_or_zero_dispatch(
float)
655 template <
typename _Tp,
typename _Ap>
656 inline constexpr
size_t __size_or_zero_v
657 = __size_or_zero_dispatch<_Tp, _Ap>(0);
661 inline constexpr
size_t
662 __div_roundup(
size_t __a,
size_t __b)
663 {
return (__a + __b - 1) / __b; }
672 _GLIBCXX_SIMD_INTRINSIC constexpr _ExactBool(
bool __b) : _M_data(__b) {}
674 _ExactBool(
int) =
delete;
676 _GLIBCXX_SIMD_INTRINSIC constexpr
operator bool()
const {
return _M_data; }
686 template <
typename _Tp>
687 using __may_alias [[__gnu__::__may_alias__]] = _Tp;
692 struct _UnsupportedBase
694 _UnsupportedBase() =
delete;
695 _UnsupportedBase(
const _UnsupportedBase&) =
delete;
696 _UnsupportedBase& operator=(
const _UnsupportedBase&) =
delete;
697 ~_UnsupportedBase() =
delete;
710 struct _InvalidTraits
713 using _SimdBase = _UnsupportedBase;
714 using _MaskBase = _UnsupportedBase;
716 static constexpr
size_t _S_full_size = 0;
717 static constexpr
bool _S_is_partial =
false;
719 static constexpr
size_t _S_simd_align = 1;
721 struct _SimdMember {};
722 struct _SimdCastType;
724 static constexpr
size_t _S_mask_align = 1;
726 struct _MaskMember {};
727 struct _MaskCastType;
732 template <
typename _Tp,
typename _Abi,
typename =
void_t<>>
733 struct _SimdTraits : _InvalidTraits {};
741 inline constexpr
struct _PrivateInit {} __private_init = {};
743 inline constexpr
struct _BitsetInit {} __bitset_init = {};
747 template <
typename _From,
typename _To,
bool = is_arithmetic_v<_From>,
748 bool = is_arithmetic_v<_To>>
749 struct __is_narrowing_conversion;
753 template <
typename _From,
typename _To>
754 struct __is_narrowing_conversion<_From, _To, true, true>
755 :
public __bool_constant<(
756 __digits_v<_From> > __digits_v<_To>
757 || __finite_max_v<_From> > __finite_max_v<_To>
758 || __finite_min_v<_From> < __finite_min_v<_To>
759 || (is_signed_v<_From> && is_unsigned_v<_To>))> {};
761 template <
typename _Tp>
762 struct __is_narrowing_conversion<_Tp, bool, true, true>
766 struct __is_narrowing_conversion<bool, bool, true, true>
769 template <
typename _Tp>
770 struct __is_narrowing_conversion<_Tp, _Tp, true, true>
773 template <
typename _From,
typename _To>
774 struct __is_narrowing_conversion<_From, _To, false, true>
775 :
public negation<is_convertible<_From, _To>> {};
779 template <
typename _From,
typename _To,
bool = (sizeof(_From) < sizeof(_To))>
780 struct __converts_to_higher_
integer_rank : public true_type {};
783 template <
typename _From,
typename _To>
784 struct __converts_to_higher_integer_rank<_From, _To, false>
785 :
public is_same<decltype(declval<_From>() + declval<_To>()), _To> {};
789 template <
typename _Tp,
typename _Ap>
790 _GLIBCXX_SIMD_INTRINSIC constexpr
const auto&
791 __data(
const simd<_Tp, _Ap>& __x);
793 template <
typename _Tp,
typename _Ap>
794 _GLIBCXX_SIMD_INTRINSIC constexpr
auto&
795 __data(simd<_Tp, _Ap>& __x);
797 template <
typename _Tp,
typename _Ap>
798 _GLIBCXX_SIMD_INTRINSIC constexpr
const auto&
799 __data(
const simd_mask<_Tp, _Ap>& __x);
801 template <
typename _Tp,
typename _Ap>
802 _GLIBCXX_SIMD_INTRINSIC constexpr
auto&
803 __data(simd_mask<_Tp, _Ap>& __x);
807 template <
typename _FromT,
typename _FromA,
typename _ToT,
typename _ToA,
809 struct _SimdConverter;
811 template <
typename _Tp,
typename _Ap>
812 struct _SimdConverter<_Tp, _Ap, _Tp, _Ap, void>
814 template <
typename _Up>
815 _GLIBCXX_SIMD_INTRINSIC
const _Up&
816 operator()(
const _Up& __x)
822 template <
typename _V>
823 _GLIBCXX_SIMD_INTRINSIC constexpr
auto
824 __to_value_type_or_member_type(
const _V& __x) -> decltype(__data(__x))
825 {
return __data(__x); }
827 template <
typename _V>
828 _GLIBCXX_SIMD_INTRINSIC constexpr
const typename _V::value_type&
829 __to_value_type_or_member_type(
const typename _V::value_type& __x)
834 template <
size_t _Size>
835 struct __bool_storage_member_type;
837 template <
size_t _Size>
838 using __bool_storage_member_type_t =
839 typename __bool_storage_member_type<_Size>::type;
849 template <
typename _Tp,
typename... _Abis>
854 template <
typename _Tp,
int _Np>
855 struct __fixed_size_storage;
857 template <
typename _Tp,
int _Np>
858 using __fixed_size_storage_t =
typename __fixed_size_storage<_Tp, _Np>::type;
862 template <
typename _Tp,
size_t _Size,
typename =
void_t<>>
865 template <
typename _Tp>
866 using _SimdWrapper8 = _SimdWrapper<_Tp, 8 /
sizeof(_Tp)>;
867 template <
typename _Tp>
868 using _SimdWrapper16 = _SimdWrapper<_Tp, 16 /
sizeof(_Tp)>;
869 template <
typename _Tp>
870 using _SimdWrapper32 = _SimdWrapper<_Tp, 32 /
sizeof(_Tp)>;
871 template <
typename _Tp>
872 using _SimdWrapper64 = _SimdWrapper<_Tp, 64 /
sizeof(_Tp)>;
876 template <
typename _Tp>
879 template <
typename _Tp,
size_t _Np>
880 struct __is_simd_wrapper<_SimdWrapper<_Tp, _Np>> :
true_type {};
882 template <
typename _Tp>
883 inline constexpr
bool __is_simd_wrapper_v = __is_simd_wrapper<_Tp>::value;
890 template <
typename _Tp,
typename _Fp>
892 _S_bit_iteration(_Tp __mask, _Fp&& __f)
894 static_assert(
sizeof(_ULLong) >=
sizeof(_Tp));
895 conditional_t<
sizeof(_Tp) <=
sizeof(_UInt), _UInt, _ULLong> __k;
896 if constexpr (is_convertible_v<_Tp, decltype(__k)>)
899 __k = __mask.to_ullong();
902 __f(std::__countr_zero(__k));
912 template <
typename _Tp =
void>
914 { constexpr _Tp operator()(_Tp __a)
const {
return ++__a; } };
917 struct __increment<void>
919 template <
typename _Tp>
921 operator()(_Tp __a)
const
925 template <
typename _Tp =
void>
927 { constexpr _Tp operator()(_Tp __a)
const {
return --__a; } };
930 struct __decrement<void>
932 template <
typename _Tp>
934 operator()(_Tp __a)
const
940 template <
typename _From,
typename _To,
942 __is_narrowing_conversion<__remove_cvref_t<_From>, _To>>::value>>
943 using _ValuePreserving = _From;
945 template <
typename _From,
typename _To,
946 typename _DecayedFrom = __remove_cvref_t<_From>,
948 is_convertible<_From, _To>,
950 is_same<_DecayedFrom, _To>, is_same<_DecayedFrom, int>,
951 conjunction<is_same<_DecayedFrom, _UInt>, is_unsigned<_To>>,
952 negation<__is_narrowing_conversion<_DecayedFrom, _To>>>>::value>>
953 using _ValuePreservingOrInt = _From;
957 template <
typename _Tp,
size_t _Bytes,
typename =
void_t<>>
958 struct __intrinsic_type;
960 template <
typename _Tp,
size_t _Size>
961 using __intrinsic_type_t =
962 typename __intrinsic_type<_Tp, _Size *
sizeof(_Tp)>::type;
964 template <
typename _Tp>
965 using __intrinsic_type2_t =
typename __intrinsic_type<_Tp, 2>::type;
966 template <
typename _Tp>
967 using __intrinsic_type4_t =
typename __intrinsic_type<_Tp, 4>::type;
968 template <
typename _Tp>
969 using __intrinsic_type8_t =
typename __intrinsic_type<_Tp, 8>::type;
970 template <
typename _Tp>
971 using __intrinsic_type16_t =
typename __intrinsic_type<_Tp, 16>::type;
972 template <
typename _Tp>
973 using __intrinsic_type32_t =
typename __intrinsic_type<_Tp, 32>::type;
974 template <
typename _Tp>
975 using __intrinsic_type64_t =
typename __intrinsic_type<_Tp, 64>::type;
979 template <
size_t _Np,
bool _Sanitized = false>
982 template <
size_t _Np,
bool _Sanitized>
983 struct __is_bitmask<_BitMask<_Np, _Sanitized>, void> :
true_type {};
985 template <
size_t _Np>
986 using _SanitizedBitMask = _BitMask<_Np, true>;
988 template <
size_t _Np,
bool _Sanitized>
991 static_assert(_Np > 0);
993 static constexpr
size_t _NBytes = __div_roundup(_Np, __CHAR_BIT__);
997 sizeof(_ULLong), std::__bit_ceil(_NBytes))>>>;
999 static constexpr
int _S_array_size = __div_roundup(_NBytes,
sizeof(_Tp));
1001 _Tp _M_bits[_S_array_size];
1003 static constexpr
int _S_unused_bits
1004 = _Np == 1 ? 0 : _S_array_size *
sizeof(_Tp) * __CHAR_BIT__ - _Np;
1006 static constexpr _Tp _S_bitmask = +_Tp(~_Tp()) >> _S_unused_bits;
1008 constexpr _BitMask() noexcept = default;
1010 constexpr _BitMask(
unsigned long long __x) noexcept
1011 : _M_bits{
static_cast<_Tp
>(__x)} {}
1013 _BitMask(bitset<_Np> __x) noexcept : _BitMask(__x.to_ullong()) {}
1015 constexpr _BitMask(
const _BitMask&) noexcept =
default;
1017 template <
bool _RhsSanitized,
typename =
enable_if_t<_RhsSanitized ==
false
1018 && _Sanitized ==
true>>
1019 constexpr _BitMask(
const _BitMask<_Np, _RhsSanitized>& __rhs) noexcept
1020 : _BitMask(__rhs._M_sanitized()) {}
1022 constexpr
operator _SimdWrapper<bool, _Np>() const noexcept
1024 static_assert(_S_array_size == 1);
1030 _M_to_bits() const noexcept
1032 static_assert(_S_array_size == 1);
1037 constexpr
unsigned long long
1038 to_ullong() const noexcept
1040 static_assert(_S_array_size == 1);
1045 constexpr
unsigned long
1046 to_ulong() const noexcept
1048 static_assert(_S_array_size == 1);
1052 constexpr bitset<_Np>
1053 _M_to_bitset() const noexcept
1055 static_assert(_S_array_size == 1);
1059 constexpr decltype(
auto)
1060 _M_sanitized() const noexcept
1062 if constexpr (_Sanitized)
1064 else if constexpr (_Np == 1)
1065 return _SanitizedBitMask<_Np>(_M_bits[0]);
1068 _SanitizedBitMask<_Np> __r = {};
1069 for (
int __i = 0; __i < _S_array_size; ++__i)
1070 __r._M_bits[__i] = _M_bits[__i];
1071 if constexpr (_S_unused_bits > 0)
1072 __r._M_bits[_S_array_size - 1] &= _S_bitmask;
1077 template <
size_t _Mp,
bool _LSanitized>
1078 constexpr _BitMask<_Np + _Mp, _Sanitized>
1079 _M_prepend(_BitMask<_Mp, _LSanitized> __lsb)
const noexcept
1081 constexpr
size_t _RN = _Np + _Mp;
1082 using _Rp = _BitMask<_RN, _Sanitized>;
1083 if constexpr (_Rp::_S_array_size == 1)
1085 _Rp __r{{_M_bits[0]}};
1086 __r._M_bits[0] <<= _Mp;
1087 __r._M_bits[0] |= __lsb._M_sanitized()._M_bits[0];
1091 __assert_unreachable<_Rp>();
1097 template <
size_t _DropLsb,
size_t _NewSize = _Np - _DropLsb>
1099 _M_extract() const noexcept
1101 static_assert(_Np > _DropLsb);
1102 static_assert(_DropLsb + _NewSize <=
sizeof(_ULLong) * __CHAR_BIT__,
1103 "not implemented for bitmasks larger than one ullong");
1104 if constexpr (_NewSize == 1)
1106 return _SanitizedBitMask<1>(_M_bits[0] & (_Tp(1) << _DropLsb));
1108 return _BitMask<_NewSize,
1109 ((_NewSize + _DropLsb == sizeof(_Tp) * __CHAR_BIT__
1110 && _NewSize + _DropLsb <= _Np)
1111 || ((_Sanitized || _Np == sizeof(_Tp) * __CHAR_BIT__)
1112 && _NewSize + _DropLsb >= _Np))>(_M_bits[0]
1118 all() const noexcept
1120 if constexpr (_Np == 1)
1122 else if constexpr (!_Sanitized)
1123 return _M_sanitized().all();
1126 constexpr _Tp __allbits = ~_Tp();
1127 for (
int __i = 0; __i < _S_array_size - 1; ++__i)
1128 if (_M_bits[__i] != __allbits)
1130 return _M_bits[_S_array_size - 1] == _S_bitmask;
1137 any() const noexcept
1139 if constexpr (_Np == 1)
1141 else if constexpr (!_Sanitized)
1142 return _M_sanitized().any();
1145 for (
int __i = 0; __i < _S_array_size - 1; ++__i)
1146 if (_M_bits[__i] != 0)
1148 return _M_bits[_S_array_size - 1] != 0;
1154 none() const noexcept
1156 if constexpr (_Np == 1)
1158 else if constexpr (!_Sanitized)
1159 return _M_sanitized().none();
1162 for (
int __i = 0; __i < _S_array_size - 1; ++__i)
1163 if (_M_bits[__i] != 0)
1165 return _M_bits[_S_array_size - 1] == 0;
1172 count() const noexcept
1174 if constexpr (_Np == 1)
1176 else if constexpr (!_Sanitized)
1177 return _M_sanitized().none();
1180 int __result = __builtin_popcountll(_M_bits[0]);
1181 for (
int __i = 1; __i < _S_array_size; ++__i)
1182 __result += __builtin_popcountll(_M_bits[__i]);
1189 operator[](
size_t __i)
const noexcept
1191 if constexpr (_Np == 1)
1193 else if constexpr (_S_array_size == 1)
1194 return (_M_bits[0] >> __i) & 1;
1197 const size_t __j = __i / (
sizeof(_Tp) * __CHAR_BIT__);
1198 const size_t __shift = __i % (
sizeof(_Tp) * __CHAR_BIT__);
1199 return (_M_bits[__j] >> __shift) & 1;
1203 template <
size_t __i>
1205 operator[](_SizeConstant<__i>)
const noexcept
1207 static_assert(__i < _Np);
1208 constexpr
size_t __j = __i / (
sizeof(_Tp) * __CHAR_BIT__);
1209 constexpr
size_t __shift = __i % (
sizeof(_Tp) * __CHAR_BIT__);
1210 return static_cast<bool>(_M_bits[__j] & (_Tp(1) << __shift));
1215 set(
size_t __i,
bool __x) noexcept
1217 if constexpr (_Np == 1)
1219 else if constexpr (_S_array_size == 1)
1221 _M_bits[0] &= ~_Tp(_Tp(1) << __i);
1222 _M_bits[0] |= _Tp(_Tp(__x) << __i);
1226 const size_t __j = __i / (
sizeof(_Tp) * __CHAR_BIT__);
1227 const size_t __shift = __i % (
sizeof(_Tp) * __CHAR_BIT__);
1228 _M_bits[__j] &= ~_Tp(_Tp(1) << __shift);
1229 _M_bits[__j] |= _Tp(_Tp(__x) << __shift);
1233 template <
size_t __i>
1235 set(_SizeConstant<__i>,
bool __x) noexcept
1237 static_assert(__i < _Np);
1238 if constexpr (_Np == 1)
1242 constexpr
size_t __j = __i / (
sizeof(_Tp) * __CHAR_BIT__);
1243 constexpr
size_t __shift = __i % (
sizeof(_Tp) * __CHAR_BIT__);
1244 constexpr _Tp __mask = ~_Tp(_Tp(1) << __shift);
1245 _M_bits[__j] &= __mask;
1246 _M_bits[__j] |= _Tp(_Tp(__x) << __shift);
1252 operator~() const noexcept
1254 if constexpr (_Np == 1)
1258 _BitMask __result{};
1259 for (
int __i = 0; __i < _S_array_size - 1; ++__i)
1260 __result._M_bits[__i] = ~_M_bits[__i];
1261 if constexpr (_Sanitized)
1262 __result._M_bits[_S_array_size - 1]
1263 = _M_bits[_S_array_size - 1] ^ _S_bitmask;
1265 __result._M_bits[_S_array_size - 1] = ~_M_bits[_S_array_size - 1];
1271 operator^=(
const _BitMask& __b) & noexcept
1273 __execute_n_times<_S_array_size>(
1274 [&](
auto __i) { _M_bits[__i] ^= __b._M_bits[__i]; });
1279 operator|=(
const _BitMask& __b) & noexcept
1281 __execute_n_times<_S_array_size>(
1282 [&](
auto __i) { _M_bits[__i] |= __b._M_bits[__i]; });
1287 operator&=(
const _BitMask& __b) & noexcept
1289 __execute_n_times<_S_array_size>(
1290 [&](
auto __i) { _M_bits[__i] &= __b._M_bits[__i]; });
1294 friend constexpr _BitMask
1295 operator^(
const _BitMask& __a,
const _BitMask& __b) noexcept
1302 friend constexpr _BitMask
1303 operator|(
const _BitMask& __a,
const _BitMask& __b) noexcept
1310 friend constexpr _BitMask
1311 operator&(
const _BitMask& __a,
const _BitMask& __b) noexcept
1318 _GLIBCXX_SIMD_INTRINSIC
1320 _M_is_constprop()
const
1322 if constexpr (_S_array_size == 0)
1323 return __builtin_constant_p(_M_bits[0]);
1326 for (
int __i = 0; __i < _S_array_size; ++__i)
1327 if (!__builtin_constant_p(_M_bits[__i]))
1338 template <
typename _Tp =
void>
1339 static inline constexpr
int __min_vector_size = 2 *
sizeof(_Tp);
1341 #if _GLIBCXX_SIMD_HAVE_NEON
1343 inline constexpr
int __min_vector_size<void> = 8;
1346 inline constexpr
int __min_vector_size<void> = 16;
1351 template <
typename _Tp,
size_t _Np,
typename =
void>
1352 struct __vector_type_n {};
1355 template <
typename _Tp>
1356 struct __vector_type_n<_Tp, 0, void> {};
1359 template <
typename _Tp>
1360 struct __vector_type_n<_Tp, 1,
enable_if_t<__is_vectorizable_v<_Tp>>>
1361 {
using type = _Tp; };
1364 template <
typename _Tp,
size_t _Np>
1365 struct __vector_type_n<_Tp, _Np,
1366 enable_if_t<__is_vectorizable_v<_Tp> && _Np >= 2>>
1368 static constexpr
size_t _S_Np2 = std::__bit_ceil(_Np *
sizeof(_Tp));
1370 static constexpr
size_t _S_Bytes =
1376 _S_Np2 < __min_vector_size<_Tp> ? __min_vector_size<_Tp>
1379 using type [[__gnu__::__vector_size__(_S_Bytes)]] = _Tp;
1382 template <
typename _Tp,
size_t _Bytes,
size_t = _Bytes % sizeof(_Tp)>
1383 struct __vector_type;
1385 template <
typename _Tp,
size_t _Bytes>
1386 struct __vector_type<_Tp, _Bytes, 0>
1387 : __vector_type_n<_Tp, _Bytes / sizeof(_Tp)> {};
1389 template <
typename _Tp,
size_t _Size>
1390 using __vector_type_t =
typename __vector_type_n<_Tp, _Size>::type;
1392 template <
typename _Tp>
1393 using __vector_type2_t =
typename __vector_type<_Tp, 2>::type;
1394 template <
typename _Tp>
1395 using __vector_type4_t =
typename __vector_type<_Tp, 4>::type;
1396 template <
typename _Tp>
1397 using __vector_type8_t =
typename __vector_type<_Tp, 8>::type;
1398 template <
typename _Tp>
1399 using __vector_type16_t =
typename __vector_type<_Tp, 16>::type;
1400 template <
typename _Tp>
1401 using __vector_type32_t =
typename __vector_type<_Tp, 32>::type;
1402 template <
typename _Tp>
1403 using __vector_type64_t =
typename __vector_type<_Tp, 64>::type;
1407 template <
typename _Tp,
typename =
void_t<>>
1410 template <
typename _Tp>
1411 struct __is_vector_type<
1412 _Tp,
void_t<typename __vector_type<
1414 : is_same<_Tp, typename __vector_type<
1415 remove_reference_t<decltype(declval<_Tp>()[0])>,
1416 sizeof(_Tp)>::type> {};
1418 template <
typename _Tp>
1419 inline constexpr
bool __is_vector_type_v = __is_vector_type<_Tp>::value;
1423 #if _GLIBCXX_SIMD_HAVE_SSE_ABI
1424 template <
typename _Tp>
1425 using __is_intrinsic_type = __is_vector_type<_Tp>;
1427 template <
typename _Tp,
typename =
void_t<>>
1430 template <
typename _Tp>
1431 struct __is_intrinsic_type<
1432 _Tp,
void_t<typename __intrinsic_type<
1434 : is_same<_Tp, typename __intrinsic_type<
1435 remove_reference_t<decltype(declval<_Tp>()[0])>,
1436 sizeof(_Tp)>::type> {};
1439 template <
typename _Tp>
1440 inline constexpr
bool __is_intrinsic_type_v = __is_intrinsic_type<_Tp>::value;
1444 template <
typename _Tp,
typename =
void_t<>>
1445 struct _VectorTraitsImpl;
1447 template <
typename _Tp>
1448 struct _VectorTraitsImpl<_Tp,
enable_if_t<__is_vector_type_v<_Tp>
1449 || __is_intrinsic_type_v<_Tp>>>
1452 using value_type = remove_reference_t<decltype(declval<_Tp>()[0])>;
1453 static constexpr
int _S_full_size =
sizeof(_Tp) /
sizeof(value_type);
1454 using _Wrapper = _SimdWrapper<value_type, _S_full_size>;
1455 template <
typename _Up,
int _W = _S_full_size>
1456 static constexpr
bool _S_is
1457 = is_same_v<value_type, _Up> && _W == _S_full_size;
1460 template <
typename _Tp,
size_t _Np>
1461 struct _VectorTraitsImpl<_SimdWrapper<_Tp, _Np>,
1462 void_t<__vector_type_t<_Tp, _Np>>>
1464 using type = __vector_type_t<_Tp, _Np>;
1465 using value_type = _Tp;
1466 static constexpr
int _S_full_size =
sizeof(type) /
sizeof(value_type);
1467 using _Wrapper = _SimdWrapper<_Tp, _Np>;
1468 static constexpr
bool _S_is_partial = (_Np == _S_full_size);
1469 static constexpr
int _S_partial_width = _Np;
1470 template <
typename _Up,
int _W = _S_full_size>
1471 static constexpr
bool _S_is
1472 = is_same_v<value_type, _Up>&& _W == _S_full_size;
1475 template <typename _Tp, typename = typename _VectorTraitsImpl<_Tp>::type>
1476 using _VectorTraits = _VectorTraitsImpl<_Tp>;
1480 template <
typename _V>
1481 _GLIBCXX_SIMD_INTRINSIC constexpr
auto
1484 if constexpr (__is_vector_type_v<_V>)
1486 else if constexpr (is_simd<_V>::value || is_simd_mask<_V>::value)
1487 return __data(__x)._M_data;
1488 else if constexpr (__is_vectorizable_v<_V>)
1489 return __vector_type_t<_V, 2>{__x};
1496 template <
size_t _Np = 0,
typename _V>
1497 _GLIBCXX_SIMD_INTRINSIC constexpr
auto
1498 __as_wrapper(_V __x)
1500 if constexpr (__is_vector_type_v<_V>)
1501 return _SimdWrapper<typename _VectorTraits<_V>::value_type,
1502 (_Np > 0 ? _Np : _VectorTraits<_V>::_S_full_size)>(__x);
1503 else if constexpr (is_simd<_V>::value || is_simd_mask<_V>::value)
1505 static_assert(_V::size() == _Np);
1510 static_assert(_V::_S_size == _Np);
1517 template <
typename _To,
typename _From>
1518 _GLIBCXX_SIMD_INTRINSIC constexpr _To
1519 __intrin_bitcast(_From __v)
1521 static_assert((__is_vector_type_v<_From> || __is_intrinsic_type_v<_From>)
1522 && (__is_vector_type_v<_To> || __is_intrinsic_type_v<_To>));
1523 if constexpr (
sizeof(_To) ==
sizeof(_From))
1524 return reinterpret_cast<_To
>(__v);
1525 else if constexpr (
sizeof(_From) >
sizeof(_To))
1526 if constexpr (
sizeof(_To) >= 16)
1527 return reinterpret_cast<const __may_alias<_To>&>(__v);
1531 __builtin_memcpy(&__r, &__v,
sizeof(_To));
1534 #if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1535 else if constexpr (__have_avx &&
sizeof(_From) == 16 &&
sizeof(_To) == 32)
1536 return reinterpret_cast<_To>(__builtin_ia32_ps256_ps(
1537 reinterpret_cast<__vector_type_t<
float, 4>>(__v)));
1538 else if constexpr (__have_avx512f && sizeof(_From) == 16
1539 && sizeof(_To) == 64)
1540 return reinterpret_cast<_To>(__builtin_ia32_ps512_ps(
1541 reinterpret_cast<__vector_type_t<
float, 4>>(__v)));
1542 else if constexpr (__have_avx512f && sizeof(_From) == 32
1543 && sizeof(_To) == 64)
1544 return reinterpret_cast<_To>(__builtin_ia32_ps512_256ps(
1545 reinterpret_cast<__vector_type_t<
float, 8>>(__v)));
1547 else if constexpr (
sizeof(__v) <= 8)
1548 return reinterpret_cast<_To>(
1549 __vector_type_t<__int_for_sizeof_t<_From>, sizeof(_To) / sizeof(_From)>{
1550 reinterpret_cast<__int_for_sizeof_t<_From>
>(__v)});
1553 static_assert(
sizeof(_To) >
sizeof(_From));
1555 __builtin_memcpy(&__r, &__v,
sizeof(_From));
1562 template <
typename _To,
size_t _NN = 0,
typename _From,
1563 typename _FromVT = _VectorTraits<_From>,
1564 size_t _Np = _NN == 0 ?
sizeof(_From) /
sizeof(_To) : _NN>
1565 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_To, _Np>
1566 __vector_bitcast(_From __x)
1568 using _R = __vector_type_t<_To, _Np>;
1569 return __intrin_bitcast<_R>(__x);
1572 template <
typename _To,
size_t _NN = 0,
typename _Tp,
size_t _Nx,
1574 = _NN == 0 ?
sizeof(_SimdWrapper<_Tp, _Nx>) /
sizeof(_To) : _NN>
1575 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_To, _Np>
1576 __vector_bitcast(
const _SimdWrapper<_Tp, _Nx>& __x)
1578 static_assert(_Np > 1);
1579 return __intrin_bitcast<__vector_type_t<_To, _Np>>(__x._M_data);
1584 #ifdef _GLIBCXX_SIMD_WORKAROUND_PR85048
1585 template <
typename _To,
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1586 _To __convert_x86(_Tp);
1588 template <
typename _To,
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1589 _To __convert_x86(_Tp, _Tp);
1591 template <
typename _To,
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1592 _To __convert_x86(_Tp, _Tp, _Tp, _Tp);
1594 template <
typename _To,
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1595 _To __convert_x86(_Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp);
1597 template <
typename _To,
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1598 _To __convert_x86(_Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp,
1599 _Tp, _Tp, _Tp, _Tp);
1604 template <
typename _To,
typename _From>
1605 _GLIBCXX_SIMD_INTRINSIC constexpr _To
1606 __bit_cast(
const _From __x)
1608 #if __has_builtin(__builtin_bit_cast)
1609 return __builtin_bit_cast(_To, __x);
1611 static_assert(
sizeof(_To) ==
sizeof(_From));
1612 constexpr
bool __to_is_vectorizable
1613 = is_arithmetic_v<_To> || is_enum_v<_To>;
1614 constexpr
bool __from_is_vectorizable
1615 = is_arithmetic_v<_From> || is_enum_v<_From>;
1616 if constexpr (__is_vector_type_v<_To> && __is_vector_type_v<_From>)
1617 return reinterpret_cast<_To
>(__x);
1618 else if constexpr (__is_vector_type_v<_To> && __from_is_vectorizable)
1620 using _FV [[gnu::vector_size(
sizeof(_From))]] = _From;
1621 return reinterpret_cast<_To
>(_FV{__x});
1623 else if constexpr (__to_is_vectorizable && __from_is_vectorizable)
1625 using _TV [[gnu::vector_size(
sizeof(_To))]] = _To;
1626 using _FV [[gnu::vector_size(
sizeof(_From))]] = _From;
1627 return reinterpret_cast<_TV
>(_FV{__x})[0];
1629 else if constexpr (__to_is_vectorizable && __is_vector_type_v<_From>)
1631 using _TV [[gnu::vector_size(
sizeof(_To))]] = _To;
1632 return reinterpret_cast<_TV
>(__x)[0];
1637 __builtin_memcpy(
reinterpret_cast<char*
>(&__r),
1638 reinterpret_cast<const char*
>(&__x),
sizeof(_To));
1646 template <
typename _Tp,
typename _TVT = _VectorTraits<_Tp>,
1648 = __
intrinsic_type_t<
typename _TVT::value_type, _TVT::_S_full_size>>
1649 _GLIBCXX_SIMD_INTRINSIC constexpr _R
1650 __to_intrin(_Tp __x)
1652 static_assert(
sizeof(__x) <=
sizeof(_R),
1653 "__to_intrin may never drop values off the end");
1654 if constexpr (
sizeof(__x) ==
sizeof(_R))
1655 return reinterpret_cast<_R
>(__as_vector(__x));
1658 using _Up = __int_for_sizeof_t<_Tp>;
1659 return reinterpret_cast<_R
>(
1660 __vector_type_t<_Up,
sizeof(_R) /
sizeof(_Up)>{__bit_cast<_Up>(__x)});
1666 template <
typename _Tp,
typename... _Args>
1667 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp,
sizeof...(_Args)>
1668 __make_vector(
const _Args&... __args)
1670 return __vector_type_t<_Tp,
sizeof...(_Args)>{
static_cast<_Tp
>(__args)...};
1675 template <
size_t _Np,
typename _Tp>
1676 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1677 __vector_broadcast(_Tp __x)
1679 return __call_with_n_evaluations<_Np>(
1680 [](
auto... __xx) {
return __vector_type_t<_Tp, _Np>{__xx...}; },
1681 [&__x](int) {
return __x; });
1686 template <
typename _Tp,
size_t _Np,
typename _Gp,
size_t... _I>
1687 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1688 __generate_vector_impl(_Gp&& __gen, index_sequence<_I...>)
1690 return __vector_type_t<_Tp, _Np>{
1691 static_cast<_Tp
>(__gen(_SizeConstant<_I>()))...};
1694 template <
typename _V,
typename _VVT = _VectorTraits<_V>,
typename _Gp>
1695 _GLIBCXX_SIMD_INTRINSIC constexpr _V
1696 __generate_vector(_Gp&& __gen)
1698 if constexpr (__is_vector_type_v<_V>)
1699 return __generate_vector_impl<
typename _VVT::value_type,
1700 _VVT::_S_full_size>(
1701 static_cast<_Gp&&
>(__gen), make_index_sequence<_VVT::_S_full_size>());
1703 return __generate_vector_impl<
typename _VVT::value_type,
1704 _VVT::_S_partial_width>(
1705 static_cast<_Gp&&
>(__gen),
1706 make_index_sequence<_VVT::_S_partial_width>());
1709 template <
typename _Tp,
size_t _Np,
typename _Gp>
1710 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1711 __generate_vector(_Gp&& __gen)
1713 return __generate_vector_impl<_Tp, _Np>(
static_cast<_Gp&&
>(__gen),
1714 make_index_sequence<_Np>());
1719 template <
typename _TW>
1720 _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1721 __xor(_TW __a, _TW __b) noexcept
1723 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1725 using _Tp =
typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1726 _VectorTraitsImpl<_TW>>::value_type;
1727 if constexpr (is_floating_point_v<_Tp>)
1729 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1730 return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1731 ^ __vector_bitcast<_Ip>(__b));
1733 else if constexpr (__is_vector_type_v<_TW>)
1736 return __a._M_data ^ __b._M_data;
1744 template <
typename _TW>
1745 _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1746 __or(_TW __a, _TW __b) noexcept
1748 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1750 using _Tp =
typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1751 _VectorTraitsImpl<_TW>>::value_type;
1752 if constexpr (is_floating_point_v<_Tp>)
1754 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1755 return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1756 | __vector_bitcast<_Ip>(__b));
1758 else if constexpr (__is_vector_type_v<_TW>)
1761 return __a._M_data | __b._M_data;
1769 template <
typename _TW>
1770 _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1771 __and(_TW __a, _TW __b) noexcept
1773 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1775 using _Tp =
typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1776 _VectorTraitsImpl<_TW>>::value_type;
1777 if constexpr (is_floating_point_v<_Tp>)
1779 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1780 return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1781 & __vector_bitcast<_Ip>(__b));
1783 else if constexpr (__is_vector_type_v<_TW>)
1786 return __a._M_data & __b._M_data;
1794 #if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1795 static constexpr
struct
1797 _GLIBCXX_SIMD_INTRINSIC __v4sf
1798 operator()(__v4sf __a, __v4sf __b)
const noexcept
1799 {
return __builtin_ia32_andnps(__a, __b); }
1801 _GLIBCXX_SIMD_INTRINSIC __v2df
1802 operator()(__v2df __a, __v2df __b)
const noexcept
1803 {
return __builtin_ia32_andnpd(__a, __b); }
1805 _GLIBCXX_SIMD_INTRINSIC __v2di
1806 operator()(__v2di __a, __v2di __b)
const noexcept
1807 {
return __builtin_ia32_pandn128(__a, __b); }
1809 _GLIBCXX_SIMD_INTRINSIC __v8sf
1810 operator()(__v8sf __a, __v8sf __b)
const noexcept
1811 {
return __builtin_ia32_andnps256(__a, __b); }
1813 _GLIBCXX_SIMD_INTRINSIC __v4df
1814 operator()(__v4df __a, __v4df __b)
const noexcept
1815 {
return __builtin_ia32_andnpd256(__a, __b); }
1817 _GLIBCXX_SIMD_INTRINSIC __v4di
1818 operator()(__v4di __a, __v4di __b)
const noexcept
1820 if constexpr (__have_avx2)
1821 return __builtin_ia32_andnotsi256(__a, __b);
1823 return reinterpret_cast<__v4di
>(
1824 __builtin_ia32_andnpd256(
reinterpret_cast<__v4df
>(__a),
1825 reinterpret_cast<__v4df
>(__b)));
1828 _GLIBCXX_SIMD_INTRINSIC __v16sf
1829 operator()(__v16sf __a, __v16sf __b)
const noexcept
1831 if constexpr (__have_avx512dq)
1832 return _mm512_andnot_ps(__a, __b);
1834 return reinterpret_cast<__v16sf
>(
1835 _mm512_andnot_si512(
reinterpret_cast<__v8di
>(__a),
1836 reinterpret_cast<__v8di
>(__b)));
1839 _GLIBCXX_SIMD_INTRINSIC __v8df
1840 operator()(__v8df __a, __v8df __b)
const noexcept
1842 if constexpr (__have_avx512dq)
1843 return _mm512_andnot_pd(__a, __b);
1845 return reinterpret_cast<__v8df
>(
1846 _mm512_andnot_si512(
reinterpret_cast<__v8di
>(__a),
1847 reinterpret_cast<__v8di
>(__b)));
1850 _GLIBCXX_SIMD_INTRINSIC __v8di
1851 operator()(__v8di __a, __v8di __b)
const noexcept
1852 {
return _mm512_andnot_si512(__a, __b); }
1856 template <
typename _TW>
1857 _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1858 __andnot(_TW __a, _TW __b) noexcept
1860 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1862 using _TVT = conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1863 _VectorTraitsImpl<_TW>>;
1864 using _Tp =
typename _TVT::value_type;
1865 #if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1866 if constexpr (
sizeof(_TW) >= 16)
1868 const auto __ai = __to_intrin(__a);
1869 const auto __bi = __to_intrin(__b);
1870 if (!__builtin_is_constant_evaluated()
1871 && !(__builtin_constant_p(__ai) && __builtin_constant_p(__bi)))
1873 const auto __r = _S_x86_andnot(__ai, __bi);
1874 if constexpr (is_convertible_v<decltype(__r), _TW>)
1877 return reinterpret_cast<typename _TVT::type>(__r);
1881 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1882 return __vector_bitcast<_Tp>(~__vector_bitcast<_Ip>(__a)
1883 & __vector_bitcast<_Ip>(__b));
1891 template <
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1892 _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
1893 __not(_Tp __a) noexcept
1895 if constexpr (is_floating_point_v<typename _TVT::value_type>)
1896 return reinterpret_cast<typename _TVT::type
>(
1897 ~__vector_bitcast<unsigned>(__a));
1904 template <
typename _Tp,
typename _TVT = _VectorTraits<_Tp>,
1905 typename _R = __vector_type_t<
typename _TVT::value_type,
1906 _TVT::_S_full_size * 2>>
1908 __concat(_Tp a_, _Tp b_)
1910 #ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_1
1912 = conditional_t<is_floating_point_v<typename _TVT::value_type>, double,
1914 long long,
typename _TVT::value_type>>;
1915 constexpr
int input_width =
sizeof(_Tp) /
sizeof(_W);
1916 const auto __a = __vector_bitcast<_W>(a_);
1917 const auto __b = __vector_bitcast<_W>(b_);
1918 using _Up = __vector_type_t<_W,
sizeof(_R) /
sizeof(_W)>;
1920 constexpr
int input_width = _TVT::_S_full_size;
1921 const _Tp& __a = a_;
1922 const _Tp& __b = b_;
1925 if constexpr (input_width == 2)
1926 return reinterpret_cast<_R>(_Up{__a[0], __a[1], __b[0], __b[1]});
1927 else if constexpr (input_width == 4)
1928 return reinterpret_cast<_R>(
1929 _Up{__a[0], __a[1], __a[2], __a[3], __b[0], __b[1], __b[2], __b[3]});
1930 else if constexpr (input_width == 8)
1931 return reinterpret_cast<_R>(
1932 _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6], __a[7],
1933 __b[0], __b[1], __b[2], __b[3], __b[4], __b[5], __b[6], __b[7]});
1934 else if constexpr (input_width == 16)
1935 return reinterpret_cast<_R>(
1936 _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6],
1937 __a[7], __a[8], __a[9], __a[10], __a[11], __a[12], __a[13],
1938 __a[14], __a[15], __b[0], __b[1], __b[2], __b[3], __b[4],
1939 __b[5], __b[6], __b[7], __b[8], __b[9], __b[10], __b[11],
1940 __b[12], __b[13], __b[14], __b[15]});
1941 else if constexpr (input_width == 32)
1942 return reinterpret_cast<_R>(
1943 _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6],
1944 __a[7], __a[8], __a[9], __a[10], __a[11], __a[12], __a[13],
1945 __a[14], __a[15], __a[16], __a[17], __a[18], __a[19], __a[20],
1946 __a[21], __a[22], __a[23], __a[24], __a[25], __a[26], __a[27],
1947 __a[28], __a[29], __a[30], __a[31], __b[0], __b[1], __b[2],
1948 __b[3], __b[4], __b[5], __b[6], __b[7], __b[8], __b[9],
1949 __b[10], __b[11], __b[12], __b[13], __b[14], __b[15], __b[16],
1950 __b[17], __b[18], __b[19], __b[20], __b[21], __b[22], __b[23],
1951 __b[24], __b[25], __b[26], __b[27], __b[28], __b[29], __b[30],
1957 template <
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1958 struct _ZeroExtendProxy
1960 using value_type =
typename _TVT::value_type;
1961 static constexpr
size_t _Np = _TVT::_S_full_size;
1964 template <
typename _To,
typename _ToVT = _VectorTraits<_To>,
1966 = enable_if_t<is_same_v<
typename _ToVT::value_type, value_type>>>
1967 _GLIBCXX_SIMD_INTRINSIC
operator _To()
const
1969 constexpr
size_t _ToN = _ToVT::_S_full_size;
1970 if constexpr (_ToN == _Np)
1972 else if constexpr (_ToN == 2 * _Np)
1974 #ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_3
1975 if constexpr (__have_avx && _TVT::template _S_is<float, 4>)
1976 return __vector_bitcast<value_type>(
1977 _mm256_insertf128_ps(__m256(), __x, 0));
1978 else if constexpr (__have_avx && _TVT::template _S_is<double, 2>)
1979 return __vector_bitcast<value_type>(
1980 _mm256_insertf128_pd(__m256d(), __x, 0));
1981 else if constexpr (__have_avx2 && _Np *
sizeof(value_type) == 16)
1982 return __vector_bitcast<value_type>(
1983 _mm256_insertf128_si256(__m256i(), __to_intrin(__x), 0));
1984 else if constexpr (__have_avx512f && _TVT::template _S_is<
float, 8>)
1986 if constexpr (__have_avx512dq)
1987 return __vector_bitcast<value_type>(
1988 _mm512_insertf32x8(__m512(), __x, 0));
1990 return reinterpret_cast<__m512
>(
1991 _mm512_insertf64x4(__m512d(),
1992 reinterpret_cast<__m256d
>(__x), 0));
1994 else if constexpr (__have_avx512f
1995 && _TVT::template _S_is<double, 4>)
1996 return __vector_bitcast<value_type>(
1997 _mm512_insertf64x4(__m512d(), __x, 0));
1998 else if constexpr (__have_avx512f && _Np *
sizeof(value_type) == 32)
1999 return __vector_bitcast<value_type>(
2000 _mm512_inserti64x4(__m512i(), __to_intrin(__x), 0));
2002 return __concat(__x, _Tp());
2004 else if constexpr (_ToN == 4 * _Np)
2006 #ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_3
2007 if constexpr (__have_avx512dq && _TVT::template _S_is<double, 2>)
2009 return __vector_bitcast<value_type>(
2010 _mm512_insertf64x2(__m512d(), __x, 0));
2012 else if constexpr (__have_avx512f
2013 && is_floating_point_v<value_type>)
2015 return __vector_bitcast<value_type>(
2016 _mm512_insertf32x4(__m512(),
reinterpret_cast<__m128
>(__x),
2019 else if constexpr (__have_avx512f && _Np *
sizeof(value_type) == 16)
2021 return __vector_bitcast<value_type>(
2022 _mm512_inserti32x4(__m512i(), __to_intrin(__x), 0));
2025 return __concat(__concat(__x, _Tp()),
2026 __vector_type_t<value_type, _Np * 2>());
2028 else if constexpr (_ToN == 8 * _Np)
2029 return __concat(
operator __vector_type_t<value_type, _Np * 4>(),
2030 __vector_type_t<value_type, _Np * 4>());
2031 else if constexpr (_ToN == 16 * _Np)
2032 return __concat(
operator __vector_type_t<value_type, _Np * 8>(),
2033 __vector_type_t<value_type, _Np * 8>());
2035 __assert_unreachable<_Tp>();
2039 template <
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
2040 _GLIBCXX_SIMD_INTRINSIC _ZeroExtendProxy<_Tp, _TVT>
2041 __zero_extend(_Tp __x)
2046 template <
int _Offset,
2049 typename _TVT = _VectorTraits<_Tp>,
2050 typename _R = __vector_type_t<
typename _TVT::value_type,
2051 _TVT::_S_full_size / _SplitBy>>
2052 _GLIBCXX_SIMD_INTRINSIC constexpr _R
2055 using value_type =
typename _TVT::value_type;
2056 #if _GLIBCXX_SIMD_X86INTRIN
2057 if constexpr (
sizeof(_Tp) == 64 && _SplitBy == 4 && _Offset > 0)
2059 if constexpr (__have_avx512dq && is_same_v<double, value_type>)
2060 return _mm512_extractf64x2_pd(__to_intrin(__in), _Offset);
2061 else if constexpr (is_floating_point_v<value_type>)
2062 return __vector_bitcast<value_type>(
2063 _mm512_extractf32x4_ps(__intrin_bitcast<__m512>(__in), _Offset));
2065 return reinterpret_cast<_R
>(
2066 _mm512_extracti32x4_epi32(__intrin_bitcast<__m512i>(__in),
2072 #ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_1
2074 is_floating_point_v<value_type>, double,
2076 static_assert(
sizeof(_R) %
sizeof(_W) == 0);
2077 constexpr
int __return_width =
sizeof(_R) /
sizeof(_W);
2078 using _Up = __vector_type_t<_W, __return_width>;
2079 const auto __x = __vector_bitcast<_W>(__in);
2081 constexpr
int __return_width = _TVT::_S_full_size / _SplitBy;
2083 const __vector_type_t<value_type, _TVT::_S_full_size>& __x
2086 constexpr
int _O = _Offset * __return_width;
2087 return __call_with_subscripts<__return_width, _O>(
2088 __x, [](
auto... __entries) {
2089 return reinterpret_cast<_R
>(_Up{__entries...});
2096 template <
typename _Tp,
2098 = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2099 _GLIBCXX_SIMD_INTRINSIC constexpr _R
2103 __builtin_memcpy(&__r, &__x, 8);
2107 template <
typename _Tp,
2109 = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2110 _GLIBCXX_SIMD_INTRINSIC constexpr _R
2113 static_assert(
sizeof(_Tp) == 16,
"use __hi64z if you meant it");
2115 __builtin_memcpy(&__r,
reinterpret_cast<const char*
>(&__x) + 8, 8);
2119 template <
typename _Tp,
2121 = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2122 _GLIBCXX_SIMD_INTRINSIC constexpr _R
2123 __hi64z([[maybe_unused]] _Tp __x)
2126 if constexpr (
sizeof(_Tp) == 16)
2127 __builtin_memcpy(&__r, reinterpret_cast<const
char*>(&__x) + 8, 8);
2133 template <typename _Tp>
2134 _GLIBCXX_SIMD_INTRINSIC constexpr auto
2136 {
return __extract<0,
sizeof(_Tp) / 16>(__x); }
2138 template <
typename _Tp>
2139 _GLIBCXX_SIMD_INTRINSIC constexpr
auto
2142 static_assert(
sizeof(__x) == 32);
2143 return __extract<1, 2>(__x);
2148 template <
typename _Tp>
2149 _GLIBCXX_SIMD_INTRINSIC constexpr
auto
2152 static_assert(
sizeof(__x) == 64);
2153 return __extract<0, 2>(__x);
2156 template <
typename _Tp>
2157 _GLIBCXX_SIMD_INTRINSIC constexpr
auto
2160 static_assert(
sizeof(__x) == 64);
2161 return __extract<1, 2>(__x);
2166 template <
typename _Tp>
2169 static_assert(__is_vector_type_v<_Tp>);
2173 template <
typename _Up,
typename _UVT = _VectorTraits<_Up>>
2174 _GLIBCXX_SIMD_INTRINSIC constexpr
operator _Up()
const
2175 {
return __intrin_bitcast<typename _UVT::type>(__x); }
2178 template <
typename _Tp>
2179 _GLIBCXX_SIMD_INTRINSIC constexpr _AutoCast<_Tp>
2180 __auto_bitcast(
const _Tp& __x)
2183 template <
typename _Tp,
size_t _Np>
2184 _GLIBCXX_SIMD_INTRINSIC constexpr
2185 _AutoCast<typename _SimdWrapper<_Tp, _Np>::_BuiltinType>
2186 __auto_bitcast(
const _SimdWrapper<_Tp, _Np>& __x)
2187 {
return {__x._M_data}; }
2192 #if _GLIBCXX_SIMD_HAVE_SSE_ABI
2194 #if _GLIBCXX_SIMD_HAVE_AVX512F && _GLIBCXX_SIMD_X86INTRIN
2195 template <
size_t _Size>
2196 struct __bool_storage_member_type
2198 static_assert((_Size & (_Size - 1)) != 0,
2199 "This trait may only be used for non-power-of-2 sizes. "
2200 "Power-of-2 sizes must be specialized.");
2202 typename __bool_storage_member_type<std::__bit_ceil(_Size)>::type;
2206 struct __bool_storage_member_type<1> {
using type = bool; };
2209 struct __bool_storage_member_type<2> {
using type = __mmask8; };
2212 struct __bool_storage_member_type<4> {
using type = __mmask8; };
2215 struct __bool_storage_member_type<8> {
using type = __mmask8; };
2218 struct __bool_storage_member_type<16> {
using type = __mmask16; };
2221 struct __bool_storage_member_type<32> {
using type = __mmask32; };
2224 struct __bool_storage_member_type<64> {
using type = __mmask64; };
2230 #if _GLIBCXX_SIMD_HAVE_SSE
2231 template <
typename _Tp,
size_t _Bytes>
2232 struct __intrinsic_type<_Tp, _Bytes,
2233 enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 64>>
2235 static_assert(!is_same_v<_Tp, long double>,
2236 "no __intrinsic_type support for long double on x86");
2238 static constexpr
size_t _S_VBytes = _Bytes <= 16 ? 16
2242 using type [[__gnu__::__vector_size__(_S_VBytes)]]
2243 = conditional_t<is_integral_v<_Tp>,
long long int, _Tp>;
2250 #if _GLIBCXX_SIMD_HAVE_NEON
2252 struct __intrinsic_type<float, 8, void>
2253 {
using type = float32x2_t; };
2256 struct __intrinsic_type<float, 16, void>
2257 {
using type = float32x4_t; };
2259 #if _GLIBCXX_SIMD_HAVE_NEON_A64
2261 struct __intrinsic_type<double, 8, void>
2262 {
using type = float64x1_t; };
2265 struct __intrinsic_type<double, 16, void>
2266 {
using type = float64x2_t; };
2269 #define _GLIBCXX_SIMD_ARM_INTRIN(_Bits, _Np) \
2271 struct __intrinsic_type<__int_with_sizeof_t<_Bits / 8>, \
2272 _Np * _Bits / 8, void> \
2273 { using type = int##_Bits##x##_Np##_t; }; \
2275 struct __intrinsic_type<make_unsigned_t<__int_with_sizeof_t<_Bits / 8>>, \
2276 _Np * _Bits / 8, void> \
2277 { using type = uint##_Bits##x##_Np##_t; }
2278 _GLIBCXX_SIMD_ARM_INTRIN(8, 8);
2279 _GLIBCXX_SIMD_ARM_INTRIN(8, 16);
2280 _GLIBCXX_SIMD_ARM_INTRIN(16, 4);
2281 _GLIBCXX_SIMD_ARM_INTRIN(16, 8);
2282 _GLIBCXX_SIMD_ARM_INTRIN(32, 2);
2283 _GLIBCXX_SIMD_ARM_INTRIN(32, 4);
2284 _GLIBCXX_SIMD_ARM_INTRIN(64, 1);
2285 _GLIBCXX_SIMD_ARM_INTRIN(64, 2);
2286 #undef _GLIBCXX_SIMD_ARM_INTRIN
2288 template <
typename _Tp,
size_t _Bytes>
2289 struct __intrinsic_type<_Tp, _Bytes,
2290 enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 16>>
2292 static constexpr
int _SVecBytes = _Bytes <= 8 ? 8 : 16;
2293 using _Ip = __int_for_sizeof_t<_Tp>;
2295 is_floating_point_v<_Tp>, _Tp,
2296 conditional_t<is_unsigned_v<_Tp>, make_unsigned_t<_Ip>, _Ip>>;
2297 static_assert(!is_same_v<_Tp, _Up> || _SVecBytes != _Bytes,
2298 "should use explicit specialization above");
2299 using type =
typename __intrinsic_type<_Up, _SVecBytes>::type;
2306 template <
typename _Tp>
2307 struct __intrinsic_type_impl;
2309 #define _GLIBCXX_SIMD_PPC_INTRIN(_Tp) \
2311 struct __intrinsic_type_impl<_Tp> { using type = __vector _Tp; }
2312 _GLIBCXX_SIMD_PPC_INTRIN(
float);
2313 _GLIBCXX_SIMD_PPC_INTRIN(
double);
2314 _GLIBCXX_SIMD_PPC_INTRIN(
signed char);
2315 _GLIBCXX_SIMD_PPC_INTRIN(
unsigned char);
2316 _GLIBCXX_SIMD_PPC_INTRIN(
signed short);
2317 _GLIBCXX_SIMD_PPC_INTRIN(
unsigned short);
2318 _GLIBCXX_SIMD_PPC_INTRIN(
signed int);
2319 _GLIBCXX_SIMD_PPC_INTRIN(
unsigned int);
2320 _GLIBCXX_SIMD_PPC_INTRIN(
signed long);
2321 _GLIBCXX_SIMD_PPC_INTRIN(
unsigned long);
2322 _GLIBCXX_SIMD_PPC_INTRIN(
signed long long);
2323 _GLIBCXX_SIMD_PPC_INTRIN(
unsigned long long);
2324 #undef _GLIBCXX_SIMD_PPC_INTRIN
2326 template <
typename _Tp,
size_t _Bytes>
2327 struct __intrinsic_type<_Tp, _Bytes,
2328 enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 16>>
2330 static constexpr
bool _S_is_ldouble = is_same_v<_Tp, long double>;
2332 static_assert(!(_S_is_ldouble &&
sizeof(
long double) >
sizeof(
double)),
2333 "no __intrinsic_type support for long double on PPC");
2335 static_assert(!is_same_v<_Tp, double>,
2336 "no __intrinsic_type support for double on PPC w/o VSX");
2339 typename __intrinsic_type_impl<
2340 conditional_t<is_floating_point_v<_Tp>,
2341 conditional_t<_S_is_ldouble, double, _Tp>,
2342 __int_for_sizeof_t<_Tp>>>::type;
2348 template <
size_t _W
idth>
2349 struct _SimdWrapper<bool, _Width,
2350 void_t<typename __bool_storage_member_type<_Width>::type>>
2352 using _BuiltinType =
typename __bool_storage_member_type<_Width>::type;
2353 using value_type = bool;
2355 static constexpr
size_t _S_full_size =
sizeof(_BuiltinType) * __CHAR_BIT__;
2357 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<bool, _S_full_size>
2358 __as_full_vector()
const {
return _M_data; }
2360 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper() =
default;
2361 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper(_BuiltinType __k)
2364 _GLIBCXX_SIMD_INTRINSIC
operator const _BuiltinType&()
const
2367 _GLIBCXX_SIMD_INTRINSIC
operator _BuiltinType&()
2370 _GLIBCXX_SIMD_INTRINSIC _BuiltinType __intrin()
const
2373 _GLIBCXX_SIMD_INTRINSIC constexpr value_type operator[](
size_t __i)
const
2374 {
return _M_data & (_BuiltinType(1) << __i); }
2376 template <
size_t __i>
2377 _GLIBCXX_SIMD_INTRINSIC constexpr value_type
2378 operator[](_SizeConstant<__i>)
const
2379 {
return _M_data & (_BuiltinType(1) << __i); }
2381 _GLIBCXX_SIMD_INTRINSIC constexpr
void _M_set(
size_t __i, value_type __x)
2384 _M_data |= (_BuiltinType(1) << __i);
2386 _M_data &= ~(_BuiltinType(1) << __i);
2389 _GLIBCXX_SIMD_INTRINSIC
2390 constexpr
bool _M_is_constprop()
const
2391 {
return __builtin_constant_p(_M_data); }
2393 _GLIBCXX_SIMD_INTRINSIC constexpr
bool _M_is_constprop_none_of()
const
2395 if (__builtin_constant_p(_M_data))
2397 constexpr
int __nbits =
sizeof(_BuiltinType) * __CHAR_BIT__;
2398 constexpr _BuiltinType __active_mask
2399 = ~_BuiltinType() >> (__nbits - _Width);
2400 return (_M_data & __active_mask) == 0;
2405 _GLIBCXX_SIMD_INTRINSIC constexpr
bool _M_is_constprop_all_of()
const
2407 if (__builtin_constant_p(_M_data))
2409 constexpr
int __nbits =
sizeof(_BuiltinType) * __CHAR_BIT__;
2410 constexpr _BuiltinType __active_mask
2411 = ~_BuiltinType() >> (__nbits - _Width);
2412 return (_M_data & __active_mask) == __active_mask;
2417 _BuiltinType _M_data;
2421 template <
bool _MustZeroInitPadding,
typename _BuiltinType>
2422 struct _SimdWrapperBase;
2424 template <
typename _BuiltinType>
2425 struct _SimdWrapperBase<false, _BuiltinType>
2427 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapperBase() =
default;
2428 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapperBase(_BuiltinType __init)
2432 _BuiltinType _M_data;
2435 template <
typename _BuiltinType>
2436 struct _SimdWrapperBase<true, _BuiltinType>
2439 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapperBase() : _M_data() {}
2440 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapperBase(_BuiltinType __init)
2444 _BuiltinType _M_data;
2449 template <
typename _Tp,
size_t _W
idth>
2450 struct _SimdWrapper<
2452 void_t<__vector_type_t<_Tp, _Width>, __intrinsic_type_t<_Tp, _Width>>>
2453 : _SimdWrapperBase<__has_iec559_behavior<__signaling_NaN, _Tp>::value
2454 && sizeof(_Tp) * _Width
2455 == sizeof(__vector_type_t<_Tp, _Width>),
2456 __vector_type_t<_Tp, _Width>>
2459 = _SimdWrapperBase<__has_iec559_behavior<__signaling_NaN, _Tp>::value
2460 &&
sizeof(_Tp) * _Width
2461 ==
sizeof(__vector_type_t<_Tp, _Width>),
2462 __vector_type_t<_Tp, _Width>>;
2464 static_assert(__is_vectorizable_v<_Tp>);
2465 static_assert(_Width >= 2);
2467 using _BuiltinType = __vector_type_t<_Tp, _Width>;
2468 using value_type = _Tp;
2470 static inline constexpr
size_t _S_full_size
2471 =
sizeof(_BuiltinType) /
sizeof(value_type);
2472 static inline constexpr
int _S_size = _Width;
2473 static inline constexpr
bool _S_is_partial = _S_full_size != _S_size;
2475 using _Base::_M_data;
2477 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<_Tp, _S_full_size>
2478 __as_full_vector()
const
2481 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper(initializer_list<_Tp> __init)
2482 : _Base(__generate_from_n_evaluations<_Width, _BuiltinType>(
2483 [&](auto __i) {
return __init.begin()[__i.value]; })) {}
2485 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper() =
default;
2486 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper(
const _SimdWrapper&)
2488 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper(_SimdWrapper&&) =
default;
2490 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper&
2491 operator=(
const _SimdWrapper&) =
default;
2492 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper&
2493 operator=(_SimdWrapper&&) =
default;
2495 template <
typename _V,
typename =
enable_if_t<disjunction_v<
2496 is_same<_V, __vector_type_t<_Tp, _Width>>,
2497 is_same<_V, __intrinsic_type_t<_Tp, _Width>>>>>
2498 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper(_V __x)
2500 : _Base(__vector_bitcast<_Tp, _Width>(__x)) {}
2502 template <
typename... _As,
2503 typename =
enable_if_t<((is_same_v<simd_abi::scalar, _As> && ...)
2504 &&
sizeof...(_As) <= _Width)>>
2505 _GLIBCXX_SIMD_INTRINSIC constexpr
2506 operator _SimdTuple<_Tp, _As...>()
const
2508 const auto& dd = _M_data;
2509 return __generate_from_n_evaluations<
sizeof...(_As),
2510 _SimdTuple<_Tp, _As...>>([&](
2511 auto __i) constexpr {
return dd[int(__i)]; });
2514 _GLIBCXX_SIMD_INTRINSIC constexpr
operator const _BuiltinType&()
const
2517 _GLIBCXX_SIMD_INTRINSIC constexpr
operator _BuiltinType&()
2520 _GLIBCXX_SIMD_INTRINSIC constexpr _Tp operator[](
size_t __i)
const
2521 {
return _M_data[__i]; }
2523 template <
size_t __i>
2524 _GLIBCXX_SIMD_INTRINSIC constexpr _Tp operator[](_SizeConstant<__i>)
const
2525 {
return _M_data[__i]; }
2527 _GLIBCXX_SIMD_INTRINSIC constexpr
void _M_set(
size_t __i, _Tp __x)
2528 { _M_data[__i] = __x; }
2530 _GLIBCXX_SIMD_INTRINSIC
2531 constexpr
bool _M_is_constprop()
const
2532 {
return __builtin_constant_p(_M_data); }
2534 _GLIBCXX_SIMD_INTRINSIC constexpr
bool _M_is_constprop_none_of()
const
2536 if (__builtin_constant_p(_M_data))
2539 if constexpr (is_floating_point_v<_Tp>)
2541 using _Ip = __int_for_sizeof_t<_Tp>;
2542 const auto __intdata = __vector_bitcast<_Ip>(_M_data);
2543 __execute_n_times<_Width>(
2544 [&](
auto __i) { __r &= __intdata[__i.value] == _Ip(); });
2547 __execute_n_times<_Width>(
2548 [&](
auto __i) { __r &= _M_data[__i.value] == _Tp(); });
2554 _GLIBCXX_SIMD_INTRINSIC constexpr
bool _M_is_constprop_all_of()
const
2556 if (__builtin_constant_p(_M_data))
2559 if constexpr (is_floating_point_v<_Tp>)
2561 using _Ip = __int_for_sizeof_t<_Tp>;
2562 const auto __intdata = __vector_bitcast<_Ip>(_M_data);
2563 __execute_n_times<_Width>(
2564 [&](
auto __i) { __r &= __intdata[__i.value] == ~_Ip(); });
2567 __execute_n_times<_Width>(
2568 [&](
auto __i) { __r &= _M_data[__i.value] == ~_Tp(); });
2578 template <
typename _Tp>
2580 __vectorized_sizeof()
2582 if constexpr (!__is_vectorizable_v<_Tp>)
2585 if constexpr (
sizeof(_Tp) <= 8)
2588 if constexpr (__have_avx512bw)
2590 if constexpr (__have_avx512f &&
sizeof(_Tp) >= 4)
2592 if constexpr (__have_avx2)
2594 if constexpr (__have_avx && is_floating_point_v<_Tp>)
2596 if constexpr (__have_sse2)
2598 if constexpr (__have_sse && is_same_v<_Tp,
float>)
2608 if constexpr (__have_power8vec
2609 || (__have_power_vmx && (sizeof(_Tp) < 8))
2610 || (__have_power_vsx && is_floating_point_v<_Tp>) )
2614 if constexpr (__have_neon_a64
2615 || (__have_neon_a32 && !is_same_v<_Tp,
double>) )
2617 if constexpr (__have_neon
2622 && (__support_neon_float || !is_floating_point_v<_Tp>))
2630 namespace simd_abi {
2632 template <
typename _Tp>
2633 inline constexpr
int max_fixed_size
2634 = (__have_avx512bw &&
sizeof(_Tp) == 1) ? 64 : 32;
2637 #if defined __x86_64__ || defined __aarch64__
2638 template <
typename _Tp>
2639 using compatible =
conditional_t<(
sizeof(_Tp) <= 8), _VecBuiltin<16>, scalar>;
2640 #elif defined __ARM_NEON
2643 template <
typename _Tp>
2646 && (__support_neon_float || !is_floating_point_v<_Tp>)),
2647 _VecBuiltin<16>, scalar>;
2650 using compatible = scalar;
2655 template <
typename _Tp>
2657 __determine_native_abi()
2659 constexpr
size_t __bytes = __vectorized_sizeof<_Tp>();
2660 if constexpr (__bytes ==
sizeof(_Tp))
2661 return static_cast<scalar*
>(
nullptr);
2662 else if constexpr (__have_avx512vl || (__have_avx512f && __bytes == 64))
2663 return static_cast<_VecBltnBtmsk<__bytes>*>(
nullptr);
2665 return static_cast<_VecBuiltin<__bytes>*>(
nullptr);
2668 template <typename _Tp, typename =
enable_if_t<__is_vectorizable_v<_Tp>>>
2673 #if defined _GLIBCXX_SIMD_DEFAULT_ABI
2674 template <
typename _Tp>
2675 using __default_abi = _GLIBCXX_SIMD_DEFAULT_ABI<_Tp>;
2677 template <
typename _Tp>
2678 using __default_abi = compatible<_Tp>;
2686 template <
typename _Tp,
typename =
void_t<>>
2689 template <
typename _Tp>
2690 struct is_abi_tag<_Tp,
void_t<typename _Tp::_IsValidAbiTag>>
2691 :
public _Tp::_IsValidAbiTag {};
2693 template <
typename _Tp>
2694 inline constexpr
bool is_abi_tag_v = is_abi_tag<_Tp>::value;
2697 template <
typename _Tp>
2700 template <
typename _Tp>
2701 inline constexpr
bool is_simd_v = is_simd<_Tp>::value;
2703 template <
typename _Tp>
2706 template <
typename _Tp>
2707 inline constexpr
bool is_simd_mask_v = is_simd_mask<_Tp>::value;
2710 template <
typename _Tp,
typename _Abi,
typename =
void>
2711 struct __simd_size_impl {};
2713 template <
typename _Tp,
typename _Abi>
2714 struct __simd_size_impl<
2716 enable_if_t<conjunction_v<__is_vectorizable<_Tp>, is_abi_tag<_Abi>>>>
2717 : _SizeConstant<_Abi::template _S_size<_Tp>> {};
2719 template <
typename _Tp,
typename _Abi = simd_abi::__default_abi<_Tp>>
2720 struct simd_size : __simd_size_impl<_Tp, _Abi> {};
2722 template <
typename _Tp,
typename _Abi = simd_abi::__default_abi<_Tp>>
2723 inline constexpr
size_t simd_size_v = simd_size<_Tp, _Abi>::value;
2726 template <
typename _Tp,
size_t _Np,
typename =
void>
2727 struct __deduce_impl;
2729 namespace simd_abi {
2738 template <
typename _Tp,
size_t _Np,
typename...>
2741 template <
typename _Tp,
size_t _Np,
typename... _Abis>
2742 using deduce_t =
typename deduce<_Tp, _Np, _Abis...>::type;
2747 template <
typename _Tp,
typename _V,
typename =
void>
2750 template <
typename _Tp,
typename _Up,
typename _Abi>
2752 _Tp, simd<_Up, _Abi>,
2753 void_t<simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>>
2756 = simd<_Tp, simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>;
2759 template <
typename _Tp,
typename _Up,
typename _Abi>
2761 _Tp, simd_mask<_Up, _Abi>,
2762 void_t<simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>>
2765 = simd_mask<_Tp, simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>;
2768 template <
typename _Tp,
typename _V>
2769 using rebind_simd_t =
typename rebind_simd<_Tp, _V>::type;
2772 template <
int _Np,
typename _V,
typename =
void>
2775 template <
int _Np,
typename _Tp,
typename _Abi>
2776 struct resize_simd<_Np, simd<_Tp, _Abi>,
2777 void_t<simd_abi::deduce_t<_Tp, _Np, _Abi>>>
2778 {
using type = simd<_Tp, simd_abi::deduce_t<_Tp, _Np, _Abi>>; };
2780 template <
int _Np,
typename _Tp,
typename _Abi>
2781 struct resize_simd<_Np, simd_mask<_Tp, _Abi>,
2782 void_t<simd_abi::deduce_t<_Tp, _Np, _Abi>>>
2783 {
using type = simd_mask<_Tp, simd_abi::deduce_t<_Tp, _Np, _Abi>>; };
2785 template <
int _Np,
typename _V>
2786 using resize_simd_t =
typename resize_simd<_Np, _V>::type;
2790 template <
typename _Tp,
typename _Up =
typename _Tp::value_type>
2791 struct memory_alignment
2792 :
public _SizeConstant<vector_aligned_tag::_S_alignment<_Tp, _Up>> {};
2794 template <
typename _Tp,
typename _Up =
typename _Tp::value_type>
2795 inline constexpr
size_t memory_alignment_v = memory_alignment<_Tp, _Up>::value;
2798 template <
typename _Tp,
typename _Abi = simd_abi::__default_abi<_Tp>>
2801 template <
typename _Tp,
typename _Abi>
2802 struct is_simd<simd<_Tp, _Abi>> :
public true_type {};
2804 template <
typename _Tp>
2805 using native_simd = simd<_Tp, simd_abi::native<_Tp>>;
2807 template <
typename _Tp,
int _Np>
2808 using fixed_size_simd = simd<_Tp, simd_abi::fixed_size<_Np>>;
2810 template <
typename _Tp,
size_t _Np>
2811 using __deduced_simd = simd<_Tp, simd_abi::deduce_t<_Tp, _Np>>;
2814 template <
typename _Tp,
typename _Abi = simd_abi::__default_abi<_Tp>>
2817 template <
typename _Tp,
typename _Abi>
2818 struct is_simd_mask<simd_mask<_Tp, _Abi>> :
public true_type {};
2820 template <
typename _Tp>
2821 using native_simd_mask = simd_mask<_Tp, simd_abi::native<_Tp>>;
2823 template <
typename _Tp,
int _Np>
2824 using fixed_size_simd_mask = simd_mask<_Tp, simd_abi::fixed_size<_Np>>;
2826 template <
typename _Tp,
size_t _Np>
2827 using __deduced_simd_mask = simd_mask<_Tp, simd_abi::deduce_t<_Tp, _Np>>;
2831 template <
typename _Tp,
typename _Up,
typename _Ap,
bool = is_simd_v<_Tp>,
2833 struct __static_simd_cast_return_type;
2835 template <
typename _Tp,
typename _A0,
typename _Up,
typename _Ap>
2836 struct __static_simd_cast_return_type<simd_mask<_Tp, _A0>, _Up, _Ap, false,
2838 : __static_simd_cast_return_type<simd<_Tp, _A0>, _Up, _Ap> {};
2840 template <
typename _Tp,
typename _Up,
typename _Ap>
2841 struct __static_simd_cast_return_type<
2842 _Tp, _Up, _Ap, true,
enable_if_t<_Tp::size() == simd_size_v<_Up, _Ap>>>
2843 {
using type = _Tp; };
2845 template <
typename _Tp,
typename _Ap>
2846 struct __static_simd_cast_return_type<_Tp, _Tp, _Ap, false,
2847 #ifdef _GLIBCXX_SIMD_FIX_P2TS_ISSUE66
2853 {
using type = simd<_Tp, _Ap>; };
2855 template <
typename _Tp,
typename =
void>
2856 struct __safe_make_signed {
using type = _Tp;};
2858 template <
typename _Tp>
2859 struct __safe_make_signed<_Tp,
enable_if_t<is_integral_v<_Tp>>>
2862 using type = make_signed_t<make_unsigned_t<_Tp>>;
2865 template <
typename _Tp>
2866 using safe_make_signed_t =
typename __safe_make_signed<_Tp>::type;
2868 template <
typename _Tp,
typename _Up,
typename _Ap>
2869 struct __static_simd_cast_return_type<_Tp, _Up, _Ap, false,
2870 #ifdef _GLIBCXX_SIMD_FIX_P2TS_ISSUE66
2878 (is_integral_v<_Up> && is_integral_v<_Tp> &&
2879 #ifndef _GLIBCXX_SIMD_FIX_P2TS_ISSUE65
2880 is_signed_v<_Up> != is_signed_v<_Tp> &&
2882 is_same_v<safe_make_signed_t<_Up>, safe_make_signed_t<_Tp>>),
2883 simd<_Tp, _Ap>, fixed_size_simd<_Tp, simd_size_v<_Up, _Ap>>>;
2886 template <
typename _Tp,
typename _Up,
typename _Ap,
2888 =
typename __static_simd_cast_return_type<_Tp, _Up, _Ap>::type>
2889 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _R
2890 static_simd_cast(
const simd<_Up, _Ap>& __x)
2892 if constexpr (is_same<_R, simd<_Up, _Ap>>::value)
2896 _SimdConverter<_Up, _Ap, typename _R::value_type, typename _R::abi_type>
2898 return _R(__private_init, __c(__data(__x)));
2902 namespace __proposed {
2903 template <
typename _Tp,
typename _Up,
typename _Ap,
2905 =
typename __static_simd_cast_return_type<_Tp, _Up, _Ap>::type>
2906 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
typename _R::mask_type
2907 static_simd_cast(
const simd_mask<_Up, _Ap>& __x)
2909 using _RM =
typename _R::mask_type;
2910 return {__private_init, _RM::abi_type::_MaskImpl::template _S_convert<
2911 typename _RM::simd_type::value_type>(__x)};
2914 template <
typename _To,
typename _Up,
typename _Abi>
2915 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
2917 simd_bit_cast(
const simd<_Up, _Abi>& __x)
2919 using _Tp =
typename _To::value_type;
2920 using _ToMember =
typename _SimdTraits<_Tp, typename _To::abi_type>::_SimdMember;
2921 using _From = simd<_Up, _Abi>;
2922 using _FromMember =
typename _SimdTraits<_Up, _Abi>::_SimdMember;
2924 static_assert(
sizeof(_To) ==
sizeof(_From));
2925 static_assert(is_trivially_copyable_v<_Tp> && is_trivially_copyable_v<_Up>);
2926 static_assert(is_trivially_copyable_v<_ToMember> && is_trivially_copyable_v<_FromMember>);
2927 #if __has_builtin(__builtin_bit_cast)
2928 return {__private_init, __builtin_bit_cast(_ToMember, __data(__x))};
2930 return {__private_init, __bit_cast<_ToMember>(__data(__x))};
2934 template <
typename _To,
typename _Up,
typename _Abi>
2935 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
2937 simd_bit_cast(
const simd_mask<_Up, _Abi>& __x)
2939 using _From = simd_mask<_Up, _Abi>;
2940 static_assert(
sizeof(_To) ==
sizeof(_From));
2941 static_assert(is_trivially_copyable_v<_From>);
2944 if constexpr (is_simd_v<_To>)
2946 using _Tp =
typename _To::value_type;
2947 using _ToMember =
typename _SimdTraits<_Tp, typename _To::abi_type>::_SimdMember;
2948 static_assert(is_trivially_copyable_v<_ToMember>);
2949 #if __has_builtin(__builtin_bit_cast)
2950 return {__private_init, __builtin_bit_cast(_ToMember, __x)};
2952 return {__private_init, __bit_cast<_ToMember>(__x)};
2957 static_assert(is_trivially_copyable_v<_To>);
2958 #if __has_builtin(__builtin_bit_cast)
2959 return __builtin_bit_cast(_To, __x);
2961 return __bit_cast<_To>(__x);
2968 template <
typename _Tp,
typename _Up,
typename _Ap,
2969 typename _To = __value_type_or_identity_t<_Tp>>
2970 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
auto
2971 simd_cast(
const simd<_ValuePreserving<_Up, _To>, _Ap>& __x)
2972 -> decltype(static_simd_cast<_Tp>(__x))
2973 {
return static_simd_cast<_Tp>(__x); }
2975 namespace __proposed {
2976 template <
typename _Tp,
typename _Up,
typename _Ap,
2977 typename _To = __value_type_or_identity_t<_Tp>>
2978 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
auto
2979 simd_cast(
const simd_mask<_ValuePreserving<_Up, _To>, _Ap>& __x)
2980 -> decltype(static_simd_cast<_Tp>(__x))
2981 {
return static_simd_cast<_Tp>(__x); }
2986 namespace __proposed {
3013 template <
typename _Tp,
typename _Up,
typename _Ap>
3014 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
enable_if_t<
3015 conjunction_v<is_simd<_Tp>, is_same<typename _Tp::value_type, _Up>>, _Tp>
3016 resizing_simd_cast(
const simd<_Up, _Ap>& __x)
3018 if constexpr (is_same_v<typename _Tp::abi_type, _Ap>)
3020 else if constexpr (simd_size_v<_Up, _Ap> == 1)
3026 else if constexpr (_Tp::size() == 1)
3028 else if constexpr (sizeof(_Tp) == sizeof(__x)
3029 && !__is_fixed_size_abi_v<_Ap>)
3030 return {__private_init,
3031 __vector_bitcast<typename _Tp::value_type, _Tp::size()>(
3032 _Ap::_S_masked(__data(__x))._M_data)};
3036 __builtin_memcpy(&__data(__r), &__data(__x),
3038 *
std::min(_Tp::size(), simd_size_v<_Up, _Ap>));
3043 template <
typename _Tp,
typename _Up,
typename _Ap>
3044 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3045 enable_if_t<is_simd_mask_v<_Tp>, _Tp>
3046 resizing_simd_cast(
const simd_mask<_Up, _Ap>& __x)
3048 return {__private_init, _Tp::abi_type::_MaskImpl::template _S_convert<
3049 typename _Tp::simd_type::value_type>(__x)};
3055 template <
typename _Tp,
int _Np>
3056 _GLIBCXX_SIMD_INTRINSIC fixed_size_simd<_Tp, _Np>
3057 to_fixed_size(
const fixed_size_simd<_Tp, _Np>& __x)
3060 template <
typename _Tp,
int _Np>
3061 _GLIBCXX_SIMD_INTRINSIC fixed_size_simd_mask<_Tp, _Np>
3062 to_fixed_size(
const fixed_size_simd_mask<_Tp, _Np>& __x)
3065 template <
typename _Tp,
typename _Ap>
3066 _GLIBCXX_SIMD_INTRINSIC
auto
3067 to_fixed_size(
const simd<_Tp, _Ap>& __x)
3069 return simd<_Tp, simd_abi::fixed_size<simd_size_v<_Tp, _Ap>>>([&__x](
3070 auto __i) constexpr {
return __x[__i]; });
3073 template <
typename _Tp,
typename _Ap>
3074 _GLIBCXX_SIMD_INTRINSIC
auto
3075 to_fixed_size(
const simd_mask<_Tp, _Ap>& __x)
3077 constexpr
int _Np = simd_mask<_Tp, _Ap>::size();
3078 fixed_size_simd_mask<_Tp, _Np> __r;
3079 __execute_n_times<_Np>([&](
auto __i) constexpr { __r[__i] = __x[__i]; });
3084 template <
typename _Tp,
int _Np>
3085 _GLIBCXX_SIMD_INTRINSIC
3086 enable_if_t<(_Np == native_simd<_Tp>::size()), native_simd<_Tp>>
3087 to_native(
const fixed_size_simd<_Tp, _Np>& __x)
3089 alignas(memory_alignment_v<native_simd<_Tp>>) _Tp __mem[_Np];
3090 __x.copy_to(__mem, vector_aligned);
3091 return {__mem, vector_aligned};
3094 template <
typename _Tp,
size_t _Np>
3095 _GLIBCXX_SIMD_INTRINSIC
3096 enable_if_t<(_Np == native_simd_mask<_Tp>::size()), native_simd_mask<_Tp>>
3097 to_native(
const fixed_size_simd_mask<_Tp, _Np>& __x)
3099 return native_simd_mask<_Tp>([&](
auto __i) constexpr {
return __x[__i]; });
3103 template <
typename _Tp,
size_t _Np>
3104 _GLIBCXX_SIMD_INTRINSIC
enable_if_t<(_Np == simd<_Tp>::size()), simd<_Tp>>
3105 to_compatible(
const simd<_Tp, simd_abi::fixed_size<_Np>>& __x)
3107 alignas(memory_alignment_v<simd<_Tp>>) _Tp __mem[_Np];
3108 __x.copy_to(__mem, vector_aligned);
3109 return {__mem, vector_aligned};
3112 template <
typename _Tp,
size_t _Np>
3113 _GLIBCXX_SIMD_INTRINSIC
3114 enable_if_t<(_Np == simd_mask<_Tp>::size()), simd_mask<_Tp>>
3115 to_compatible(
const simd_mask<_Tp, simd_abi::fixed_size<_Np>>& __x)
3116 {
return simd_mask<_Tp>([&](
auto __i) constexpr {
return __x[__i]; }); }
3122 template <
typename _M,
typename _Tp>
3123 class const_where_expression
3126 static_assert(is_same_v<_V, __remove_cvref_t<_Tp>>);
3128 struct _Wrapper {
using value_type = _V; };
3131 using _Impl =
typename _V::_Impl;
3134 typename conditional_t<is_arithmetic_v<_V>, _Wrapper, _V>::value_type;
3136 _GLIBCXX_SIMD_INTRINSIC
friend const _M&
3137 __get_mask(
const const_where_expression& __x)
3138 {
return __x._M_k; }
3140 _GLIBCXX_SIMD_INTRINSIC
friend const _Tp&
3141 __get_lvalue(
const const_where_expression& __x)
3142 {
return __x._M_value; }
3148 const_where_expression(
const const_where_expression&) =
delete;
3149 const_where_expression& operator=(
const const_where_expression&) =
delete;
3151 _GLIBCXX_SIMD_INTRINSIC const_where_expression(
const _M& __kk,
const _Tp& dd)
3152 : _M_k(__kk), _M_value(const_cast<_Tp&>(dd)) {}
3154 _GLIBCXX_SIMD_INTRINSIC _V
3157 return {__private_init,
3158 _Impl::template _S_masked_unary<negate>(__data(_M_k),
3162 template <
typename _Up,
typename _Flags>
3163 [[nodiscard]] _GLIBCXX_SIMD_INTRINSIC _V
3164 copy_from(
const _LoadStorePtr<_Up, value_type>* __mem, _Flags)
const&&
3166 return {__private_init,
3167 _Impl::_S_masked_load(__data(_M_value), __data(_M_k),
3168 _Flags::template _S_apply<_V>(__mem))};
3171 template <
typename _Up,
typename _Flags>
3172 _GLIBCXX_SIMD_INTRINSIC
void
3173 copy_to(_LoadStorePtr<_Up, value_type>* __mem, _Flags)
const&&
3175 _Impl::_S_masked_store(__data(_M_value),
3176 _Flags::template _S_apply<_V>(__mem),
3182 template <
typename _Tp>
3183 class const_where_expression<bool, _Tp>
3188 static_assert(is_same_v<_V, __remove_cvref_t<_Tp>>);
3190 struct _Wrapper {
using value_type = _V; };
3194 typename conditional_t<is_arithmetic_v<_V>, _Wrapper, _V>::value_type;
3196 _GLIBCXX_SIMD_INTRINSIC
friend const _M&
3197 __get_mask(
const const_where_expression& __x)
3198 {
return __x._M_k; }
3200 _GLIBCXX_SIMD_INTRINSIC
friend const _Tp&
3201 __get_lvalue(
const const_where_expression& __x)
3202 {
return __x._M_value; }
3208 const_where_expression(
const const_where_expression&) =
delete;
3209 const_where_expression& operator=(
const const_where_expression&) =
delete;
3211 _GLIBCXX_SIMD_INTRINSIC const_where_expression(
const bool __kk,
const _Tp& dd)
3212 : _M_k(__kk), _M_value(const_cast<_Tp&>(dd)) {}
3214 _GLIBCXX_SIMD_INTRINSIC _V
operator-() const&&
3215 {
return _M_k ? -_M_value : _M_value; }
3217 template <
typename _Up,
typename _Flags>
3218 [[nodiscard]] _GLIBCXX_SIMD_INTRINSIC _V
3219 copy_from(
const _LoadStorePtr<_Up, value_type>* __mem, _Flags)
const&&
3220 {
return _M_k ?
static_cast<_V
>(__mem[0]) : _M_value; }
3222 template <
typename _Up,
typename _Flags>
3223 _GLIBCXX_SIMD_INTRINSIC
void
3224 copy_to(_LoadStorePtr<_Up, value_type>* __mem, _Flags)
const&&
3227 __mem[0] = _M_value;
3232 template <
typename _M,
typename _Tp>
3233 class where_expression :
public const_where_expression<_M, _Tp>
3235 using _Impl =
typename const_where_expression<_M, _Tp>::_Impl;
3237 static_assert(!is_const<_Tp>::value,
3238 "where_expression may only be instantiated with __a non-const "
3241 using typename const_where_expression<_M, _Tp>::value_type;
3242 using const_where_expression<_M, _Tp>::_M_k;
3243 using const_where_expression<_M, _Tp>::_M_value;
3246 is_same<typename _M::abi_type, typename _Tp::abi_type>::value,
"");
3247 static_assert(_M::size() == _Tp::size(),
"");
3249 _GLIBCXX_SIMD_INTRINSIC
friend _Tp& __get_lvalue(where_expression& __x)
3250 {
return __x._M_value; }
3253 where_expression(
const where_expression&) =
delete;
3254 where_expression& operator=(
const where_expression&) =
delete;
3256 _GLIBCXX_SIMD_INTRINSIC where_expression(
const _M& __kk, _Tp& dd)
3257 : const_where_expression<_M, _Tp>(__kk, dd) {}
3259 template <
typename _Up>
3260 _GLIBCXX_SIMD_INTRINSIC
void operator=(_Up&& __x) &&
3262 _Impl::_S_masked_assign(__data(_M_k), __data(_M_value),
3263 __to_value_type_or_member_type<_Tp>(
3264 static_cast<_Up&&
>(__x)));
3267 #define _GLIBCXX_SIMD_OP_(__op, __name) \
3268 template <typename _Up> \
3269 _GLIBCXX_SIMD_INTRINSIC void operator __op##=(_Up&& __x)&& \
3271 _Impl::template _S_masked_cassign( \
3272 __data(_M_k), __data(_M_value), \
3273 __to_value_type_or_member_type<_Tp>(static_cast<_Up&&>(__x)), \
3274 [](auto __impl, auto __lhs, auto __rhs) constexpr { \
3275 return __impl.__name(__lhs, __rhs); \
3279 _GLIBCXX_SIMD_OP_(+, _S_plus);
3280 _GLIBCXX_SIMD_OP_(-, _S_minus);
3281 _GLIBCXX_SIMD_OP_(*, _S_multiplies);
3282 _GLIBCXX_SIMD_OP_(/, _S_divides);
3283 _GLIBCXX_SIMD_OP_(%, _S_modulus);
3284 _GLIBCXX_SIMD_OP_(&, _S_bit_and);
3285 _GLIBCXX_SIMD_OP_(|, _S_bit_or);
3286 _GLIBCXX_SIMD_OP_(^, _S_bit_xor);
3287 _GLIBCXX_SIMD_OP_(<<, _S_shift_left);
3288 _GLIBCXX_SIMD_OP_(>>, _S_shift_right);
3289 #undef _GLIBCXX_SIMD_OP_
3291 _GLIBCXX_SIMD_INTRINSIC
void operator++() &&
3294 = _Impl::template _S_masked_unary<__increment>(__data(_M_k),
3298 _GLIBCXX_SIMD_INTRINSIC
void operator++(
int) &&
3301 = _Impl::template _S_masked_unary<__increment>(__data(_M_k),
3305 _GLIBCXX_SIMD_INTRINSIC
void operator--() &&
3308 = _Impl::template _S_masked_unary<__decrement>(__data(_M_k),
3312 _GLIBCXX_SIMD_INTRINSIC
void operator--(
int) &&
3315 = _Impl::template _S_masked_unary<__decrement>(__data(_M_k),
3320 template <
typename _Up,
typename _Flags>
3321 _GLIBCXX_SIMD_INTRINSIC
void
3322 copy_from(
const _LoadStorePtr<_Up, value_type>* __mem, _Flags) &&
3325 = _Impl::_S_masked_load(__data(_M_value), __data(_M_k),
3326 _Flags::template _S_apply<_Tp>(__mem));
3331 template <
typename _Tp>
3332 class where_expression<bool, _Tp> :
public const_where_expression<bool, _Tp>
3335 using typename const_where_expression<_M, _Tp>::value_type;
3336 using const_where_expression<_M, _Tp>::_M_k;
3337 using const_where_expression<_M, _Tp>::_M_value;
3340 where_expression(
const where_expression&) =
delete;
3341 where_expression& operator=(
const where_expression&) =
delete;
3343 _GLIBCXX_SIMD_INTRINSIC where_expression(
const _M& __kk, _Tp& dd)
3344 : const_where_expression<_M, _Tp>(__kk, dd) {}
3346 #define _GLIBCXX_SIMD_OP_(__op) \
3347 template <typename _Up> \
3348 _GLIBCXX_SIMD_INTRINSIC void operator __op(_Up&& __x)&& \
3349 { if (_M_k) _M_value __op static_cast<_Up&&>(__x); }
3351 _GLIBCXX_SIMD_OP_(=)
3352 _GLIBCXX_SIMD_OP_(+=)
3353 _GLIBCXX_SIMD_OP_(-=)
3354 _GLIBCXX_SIMD_OP_(*=)
3355 _GLIBCXX_SIMD_OP_(/=)
3356 _GLIBCXX_SIMD_OP_(%=)
3357 _GLIBCXX_SIMD_OP_(&=)
3358 _GLIBCXX_SIMD_OP_(|=)
3359 _GLIBCXX_SIMD_OP_(^=)
3360 _GLIBCXX_SIMD_OP_(<<=)
3361 _GLIBCXX_SIMD_OP_(>>=)
3362 #undef _GLIBCXX_SIMD_OP_
3364 _GLIBCXX_SIMD_INTRINSIC
void operator++() &&
3365 {
if (_M_k) ++_M_value; }
3367 _GLIBCXX_SIMD_INTRINSIC
void operator++(
int) &&
3368 {
if (_M_k) ++_M_value; }
3370 _GLIBCXX_SIMD_INTRINSIC
void operator--() &&
3371 {
if (_M_k) --_M_value; }
3373 _GLIBCXX_SIMD_INTRINSIC
void operator--(
int) &&
3374 {
if (_M_k) --_M_value; }
3377 template <
typename _Up,
typename _Flags>
3378 _GLIBCXX_SIMD_INTRINSIC
void
3379 copy_from(
const _LoadStorePtr<_Up, value_type>* __mem, _Flags) &&
3380 {
if (_M_k) _M_value = __mem[0]; }
3384 template <
typename _Tp,
typename _Ap>
3385 _GLIBCXX_SIMD_INTRINSIC where_expression<simd_mask<_Tp, _Ap>, simd<_Tp, _Ap>>
3386 where(
const typename simd<_Tp, _Ap>::mask_type& __k, simd<_Tp, _Ap>& __value)
3387 {
return {__k, __value}; }
3389 template <
typename _Tp,
typename _Ap>
3390 _GLIBCXX_SIMD_INTRINSIC
3391 const_where_expression<simd_mask<_Tp, _Ap>, simd<_Tp, _Ap>>
3392 where(
const typename simd<_Tp, _Ap>::mask_type& __k,
3393 const simd<_Tp, _Ap>& __value)
3394 {
return {__k, __value}; }
3396 template <
typename _Tp,
typename _Ap>
3397 _GLIBCXX_SIMD_INTRINSIC
3398 where_expression<simd_mask<_Tp, _Ap>, simd_mask<_Tp, _Ap>>
3400 simd_mask<_Tp, _Ap>& __value)
3401 {
return {__k, __value}; }
3403 template <
typename _Tp,
typename _Ap>
3404 _GLIBCXX_SIMD_INTRINSIC
3405 const_where_expression<simd_mask<_Tp, _Ap>, simd_mask<_Tp, _Ap>>
3407 const simd_mask<_Tp, _Ap>& __value)
3408 {
return {__k, __value}; }
3410 template <
typename _Tp>
3411 _GLIBCXX_SIMD_INTRINSIC where_expression<bool, _Tp>
3412 where(_ExactBool __k, _Tp& __value)
3413 {
return {__k, __value}; }
3415 template <
typename _Tp>
3416 _GLIBCXX_SIMD_INTRINSIC const_where_expression<bool, _Tp>
3417 where(_ExactBool __k,
const _Tp& __value)
3418 {
return {__k, __value}; }
3420 template <
typename _Tp,
typename _Ap>
3421 void where(
bool __k, simd<_Tp, _Ap>& __value) =
delete;
3423 template <
typename _Tp,
typename _Ap>
3424 void where(
bool __k,
const simd<_Tp, _Ap>& __value) =
delete;
3427 namespace __proposed {
3428 template <
size_t _Np>
3431 const bitset<_Np> __bits;
3434 where_range(bitset<_Np> __b) : __bits(__b) {}
3441 _GLIBCXX_SIMD_INTRINSIC
void __next_bit()
3442 { __bit = __builtin_ctzl(__mask); }
3444 _GLIBCXX_SIMD_INTRINSIC
void __reset_lsb()
3447 __mask &= (__mask - 1);
3452 iterator(decltype(__mask) __m) : __mask(__m) { __next_bit(); }
3453 iterator(
const iterator&) =
default;
3454 iterator(iterator&&) =
default;
3456 _GLIBCXX_SIMD_ALWAYS_INLINE
size_t operator->()
const
3459 _GLIBCXX_SIMD_ALWAYS_INLINE
size_t operator*()
const
3462 _GLIBCXX_SIMD_ALWAYS_INLINE iterator& operator++()
3469 _GLIBCXX_SIMD_ALWAYS_INLINE iterator operator++(
int)
3471 iterator __tmp = *
this;
3477 _GLIBCXX_SIMD_ALWAYS_INLINE
bool operator==(
const iterator& __rhs)
const
3478 {
return __mask == __rhs.__mask; }
3480 _GLIBCXX_SIMD_ALWAYS_INLINE
bool operator!=(
const iterator& __rhs)
const
3481 {
return __mask != __rhs.__mask; }
3484 iterator begin()
const
3485 {
return __bits.to_ullong(); }
3487 iterator end()
const
3491 template <
typename _Tp,
typename _Ap>
3492 where_range<simd_size_v<_Tp, _Ap>>
3493 where(
const simd_mask<_Tp, _Ap>& __k)
3494 {
return __k.__to_bitset(); }
3500 template <
typename _Tp,
typename _Abi,
typename _BinaryOperation = plus<>>
3501 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3502 reduce(
const simd<_Tp, _Abi>& __v,
3503 _BinaryOperation __binary_op = _BinaryOperation())
3504 {
return _Abi::_SimdImpl::_S_reduce(__v, __binary_op); }
3506 template <
typename _M,
typename _V,
typename _BinaryOperation = plus<>>
3507 _GLIBCXX_SIMD_INTRINSIC
typename _V::value_type
3508 reduce(
const const_where_expression<_M, _V>& __x,
3509 typename _V::value_type __identity_element,
3510 _BinaryOperation __binary_op)
3512 if (__builtin_expect(none_of(__get_mask(__x)),
false))
3513 return __identity_element;
3515 _V __tmp = __identity_element;
3516 _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3517 __data(__get_lvalue(__x)));
3518 return reduce(__tmp, __binary_op);
3521 template <
typename _M,
typename _V>
3522 _GLIBCXX_SIMD_INTRINSIC
typename _V::value_type
3523 reduce(
const const_where_expression<_M, _V>& __x, plus<> __binary_op = {})
3524 {
return reduce(__x, 0, __binary_op); }
3526 template <
typename _M,
typename _V>
3527 _GLIBCXX_SIMD_INTRINSIC
typename _V::value_type
3528 reduce(
const const_where_expression<_M, _V>& __x, multiplies<> __binary_op)
3529 {
return reduce(__x, 1, __binary_op); }
3531 template <
typename _M,
typename _V>
3532 _GLIBCXX_SIMD_INTRINSIC
typename _V::value_type
3533 reduce(
const const_where_expression<_M, _V>& __x, bit_and<> __binary_op)
3534 {
return reduce(__x, ~
typename _V::value_type(), __binary_op); }
3536 template <
typename _M,
typename _V>
3537 _GLIBCXX_SIMD_INTRINSIC
typename _V::value_type
3538 reduce(
const const_where_expression<_M, _V>& __x, bit_or<> __binary_op)
3539 {
return reduce(__x, 0, __binary_op); }
3541 template <
typename _M,
typename _V>
3542 _GLIBCXX_SIMD_INTRINSIC
typename _V::value_type
3543 reduce(
const const_where_expression<_M, _V>& __x, bit_xor<> __binary_op)
3544 {
return reduce(__x, 0, __binary_op); }
3546 template <
typename _Tp,
typename _Abi>
3547 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3548 hmin(
const simd<_Tp, _Abi>& __v) noexcept
3550 return _Abi::_SimdImpl::_S_reduce(__v, __detail::_Minimum());
3553 template <
typename _Tp,
typename _Abi>
3554 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3555 hmax(
const simd<_Tp, _Abi>& __v) noexcept
3557 return _Abi::_SimdImpl::_S_reduce(__v, __detail::_Maximum());
3560 template <
typename _M,
typename _V>
3561 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3562 typename _V::value_type
3563 hmin(
const const_where_expression<_M, _V>& __x) noexcept
3565 using _Tp =
typename _V::value_type;
3566 constexpr _Tp __id_elem =
3567 #ifdef __FINITE_MATH_ONLY__
3568 __finite_max_v<_Tp>;
3570 __value_or<__infinity, _Tp>(__finite_max_v<_Tp>);
3572 _V __tmp = __id_elem;
3573 _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3574 __data(__get_lvalue(__x)));
3575 return _V::abi_type::_SimdImpl::_S_reduce(__tmp, __detail::_Minimum());
3578 template <
typename _M,
typename _V>
3579 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3580 typename _V::value_type
3581 hmax(
const const_where_expression<_M, _V>& __x) noexcept
3583 using _Tp =
typename _V::value_type;
3584 constexpr _Tp __id_elem =
3585 #ifdef __FINITE_MATH_ONLY__
3586 __finite_min_v<_Tp>;
3589 if constexpr (__value_exists_v<__infinity, _Tp>)
3590 return -__infinity_v<_Tp>;
3592 return __finite_min_v<_Tp>;
3595 _V __tmp = __id_elem;
3596 _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3597 __data(__get_lvalue(__x)));
3598 return _V::abi_type::_SimdImpl::_S_reduce(__tmp, __detail::_Maximum());
3603 template <
typename _Tp,
typename _Ap>
3604 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3605 min(
const simd<_Tp, _Ap>& __a,
const simd<_Tp, _Ap>& __b)
3606 {
return {__private_init, _Ap::_SimdImpl::_S_min(__data(__a), __data(__b))}; }
3608 template <
typename _Tp,
typename _Ap>
3609 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3610 max(
const simd<_Tp, _Ap>& __a,
const simd<_Tp, _Ap>& __b)
3611 {
return {__private_init, _Ap::_SimdImpl::_S_max(__data(__a), __data(__b))}; }
3613 template <
typename _Tp,
typename _Ap>
3614 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3615 pair<simd<_Tp, _Ap>, simd<_Tp, _Ap>>
3616 minmax(
const simd<_Tp, _Ap>& __a,
const simd<_Tp, _Ap>& __b)
3618 const auto pair_of_members
3619 = _Ap::_SimdImpl::_S_minmax(__data(__a), __data(__b));
3620 return {simd<_Tp, _Ap>(__private_init, pair_of_members.first),
3621 simd<_Tp, _Ap>(__private_init, pair_of_members.second)};
3624 template <
typename _Tp,
typename _Ap>
3625 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3626 clamp(
const simd<_Tp, _Ap>& __v,
const simd<_Tp, _Ap>& __lo,
3627 const simd<_Tp, _Ap>& __hi)
3629 using _Impl =
typename _Ap::_SimdImpl;
3630 return {__private_init,
3631 _Impl::_S_min(__data(__hi),
3632 _Impl::_S_max(__data(__lo), __data(__v)))};
3637 template <
size_t... _Sizes,
typename _Tp,
typename _Ap,
3638 typename =
enable_if_t<((_Sizes + ...) == simd<_Tp, _Ap>::size())>>
3639 inline tuple<simd<_Tp, simd_abi::deduce_t<_Tp, _Sizes>>...>
3640 split(
const simd<_Tp, _Ap>&);
3643 template <
int _Index,
int _Total,
int _Combine = 1,
typename _Tp,
size_t _Np>
3644 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_CONST
3645 _SimdWrapper<_Tp, _Np / _Total * _Combine>
3646 __extract_part(
const _SimdWrapper<_Tp, _Np> __x);
3648 template <
int Index,
int Parts,
int _Combine = 1,
typename _Tp,
typename _A0,
3650 _GLIBCXX_SIMD_INTRINSIC
auto
3651 __extract_part(
const _SimdTuple<_Tp, _A0, _As...>& __x);
3655 template <
size_t _V0,
size_t... _Values>
3658 template <
size_t _I>
3659 static constexpr
size_t _S_at(_SizeConstant<_I> = {})
3661 if constexpr (_I == 0)
3664 return _SizeList<_Values...>::template _S_at<_I - 1>();
3667 template <
size_t _I>
3668 static constexpr auto _S_before(_SizeConstant<_I> = {})
3670 if constexpr (_I == 0)
3671 return _SizeConstant<0>();
3673 return _SizeConstant<
3674 _V0 + _SizeList<_Values...>::template _S_before<_I - 1>()>();
3677 template <
size_t _Np>
3678 static constexpr auto _S_pop_front(_SizeConstant<_Np> = {})
3680 if constexpr (_Np == 0)
3683 return _SizeList<_Values...>::template _S_pop_front<_Np - 1>();
3689 template <typename _Tp,
size_t _Np>
3690 _GLIBCXX_SIMD_INTRINSIC _SimdWrapper<_Tp, _Np / 2>
3691 __extract_center(_SimdWrapper<_Tp, _Np> __x)
3693 static_assert(_Np >= 4);
3694 static_assert(_Np % 4 == 0);
3695 #if _GLIBCXX_SIMD_X86INTRIN
3696 if constexpr (__have_avx512f &&
sizeof(_Tp) * _Np == 64)
3698 const auto __intrin = __to_intrin(__x);
3699 if constexpr (is_integral_v<_Tp>)
3700 return __vector_bitcast<_Tp>(_mm512_castsi512_si256(
3701 _mm512_shuffle_i32x4(__intrin, __intrin,
3702 1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
3703 else if constexpr (
sizeof(_Tp) == 4)
3704 return __vector_bitcast<_Tp>(_mm512_castps512_ps256(
3705 _mm512_shuffle_f32x4(__intrin, __intrin,
3706 1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
3707 else if constexpr (sizeof(_Tp) == 8)
3708 return __vector_bitcast<_Tp>(_mm512_castpd512_pd256(
3709 _mm512_shuffle_f64x2(__intrin, __intrin,
3710 1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
3712 __assert_unreachable<_Tp>();
3714 else if constexpr (sizeof(_Tp) * _Np == 32 && is_floating_point_v<_Tp>)
3715 return __vector_bitcast<_Tp>(
3716 _mm_shuffle_pd(__lo128(__vector_bitcast<
double>(__x)),
3717 __hi128(__vector_bitcast<
double>(__x)), 1));
3718 else if constexpr (sizeof(__x) == 32 && sizeof(_Tp) * _Np <= 32)
3719 return __vector_bitcast<_Tp>(
3720 _mm_alignr_epi8(__hi128(__vector_bitcast<_LLong>(__x)),
3721 __lo128(__vector_bitcast<_LLong>(__x)),
3722 sizeof(_Tp) * _Np / 4));
3726 __vector_type_t<_Tp, _Np / 2> __r;
3727 __builtin_memcpy(&__r,
3728 reinterpret_cast<const char*
>(&__x)
3729 +
sizeof(_Tp) * _Np / 4,
3730 sizeof(_Tp) * _Np / 2);
3735 template <
typename _Tp,
typename _A0,
typename... _As>
3736 _GLIBCXX_SIMD_INTRINSIC
3737 _SimdWrapper<_Tp, _SimdTuple<_Tp, _A0, _As...>::_S_size() / 2>
3738 __extract_center(
const _SimdTuple<_Tp, _A0, _As...>& __x)
3740 if constexpr (
sizeof...(_As) == 0)
3741 return __extract_center(__x.first);
3743 return __extract_part<1, 4, 2>(__x);
3748 template <
size_t... _Sizes, typename _Tp, typename... _As>
3750 __split_wrapper(_SizeList<_Sizes...>, const _SimdTuple<_Tp, _As...>& __x)
3752 return split<_Sizes...>(
3753 fixed_size_simd<_Tp, _SimdTuple<_Tp, _As...>::_S_size()>(__private_init,
3760 template <
typename _V,
typename _Ap,
3761 size_t Parts = simd_size_v<typename _V::value_type, _Ap> / _V::size()>
3762 enable_if_t<simd_size_v<typename _V::value_type, _Ap> == Parts * _V::size()
3763 && is_simd_v<_V>, array<_V, Parts>>
3764 split(
const simd<typename _V::value_type, _Ap>& __x)
3766 using _Tp =
typename _V::value_type;
3767 if constexpr (Parts == 1)
3769 return {simd_cast<_V>(__x)};
3771 else if (__x._M_is_constprop())
3773 return __generate_from_n_evaluations<Parts, array<_V, Parts>>([&](
3774 auto __i) constexpr {
3775 return _V([&](
auto __j) constexpr {
3776 return __x[__i * _V::size() + __j];
3781 __is_fixed_size_abi_v<_Ap>
3782 && (is_same_v<typename _V::abi_type, simd_abi::scalar>
3783 || (__is_fixed_size_abi_v<typename _V::abi_type>
3784 &&
sizeof(_V) ==
sizeof(_Tp) * _V::size()
3788 #ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
3789 const __may_alias<_Tp>*
const __element_ptr
3790 =
reinterpret_cast<const __may_alias<_Tp>*
>(&__data(__x));
3791 return __generate_from_n_evaluations<Parts, array<_V, Parts>>([&](
3792 auto __i) constexpr {
3793 return _V(__element_ptr + __i * _V::size(), vector_aligned);
3796 const auto& __xx = __data(__x);
3797 return __generate_from_n_evaluations<Parts, array<_V, Parts>>([&](
3798 auto __i) constexpr {
3799 [[maybe_unused]] constexpr
size_t __offset
3800 = decltype(__i)::value * _V::size();
3801 return _V([&](
auto __j) constexpr {
3802 constexpr _SizeConstant<__j + __offset> __k;
3808 else if constexpr (is_same_v<typename _V::abi_type, simd_abi::scalar>)
3811 return __generate_from_n_evaluations<Parts, array<_V, Parts>>([&](
3812 auto __i) constexpr {
return __x[__i]; });
3816 return __generate_from_n_evaluations<Parts, array<_V, Parts>>([&](
3817 auto __i) constexpr {
3818 if constexpr (__is_fixed_size_abi_v<typename _V::abi_type>)
3819 return _V([&](
auto __j) constexpr {
3820 return __x[__i * _V::size() + __j];
3823 return _V(__private_init,
3824 __extract_part<decltype(__i)::value, Parts>(__data(__x)));
3831 template <
typename _V,
typename _Ap,
3833 = simd_size_v<typename _V::simd_type::value_type, _Ap> / _V::size()>
3834 enable_if_t<is_simd_mask_v<_V> && simd_size_v<
typename
3835 _V::simd_type::value_type, _Ap> == _Parts * _V::size(), array<_V, _Parts>>
3836 split(
const simd_mask<typename _V::simd_type::value_type, _Ap>& __x)
3838 if constexpr (is_same_v<_Ap, typename _V::abi_type>)
3840 else if constexpr (_Parts == 1)
3841 return {__proposed::static_simd_cast<_V>(__x)};
3842 else if constexpr (_Parts == 2 && __is_sse_abi<typename _V::abi_type>()
3843 && __is_avx_abi<_Ap>())
3844 return {_V(__private_init, __lo128(__data(__x))),
3845 _V(__private_init, __hi128(__data(__x)))};
3846 else if constexpr (_V::size() <= __CHAR_BIT__ *
sizeof(_ULLong))
3848 const bitset __bits = __x.__to_bitset();
3849 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>([&](
3850 auto __i) constexpr {
3851 constexpr
size_t __offset = __i * _V::size();
3852 return _V(__bitset_init, (__bits >> __offset).to_ullong());
3857 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>([&](
3858 auto __i) constexpr {
3859 constexpr
size_t __offset = __i * _V::size();
3861 __private_init, [&](
auto __j) constexpr {
3862 return __x[__j + __offset];
3870 template <
size_t... _Sizes,
typename _Tp,
typename _Ap,
typename>
3871 _GLIBCXX_SIMD_ALWAYS_INLINE
3872 tuple<simd<_Tp, simd_abi::deduce_t<_Tp, _Sizes>>...>
3873 split(
const simd<_Tp, _Ap>& __x)
3875 using _SL = _SizeList<_Sizes...>;
3876 using _Tuple = tuple<__deduced_simd<_Tp, _Sizes>...>;
3877 constexpr
size_t _Np = simd_size_v<_Tp, _Ap>;
3878 constexpr
size_t _N0 = _SL::template _S_at<0>();
3879 using _V = __deduced_simd<_Tp, _N0>;
3881 if (__x._M_is_constprop())
3882 return __generate_from_n_evaluations<
sizeof...(_Sizes), _Tuple>([&](
3883 auto __i) constexpr {
3884 using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
3885 constexpr
size_t __offset = _SL::_S_before(__i);
3886 return _Vi([&](
auto __j) constexpr {
return __x[__offset + __j]; });
3888 else if constexpr (_Np == _N0)
3890 static_assert(
sizeof...(_Sizes) == 1);
3891 return {simd_cast<_V>(__x)};
3894 (__is_fixed_size_abi_v<
3895 _Ap> && __fixed_size_storage_t<_Tp, _Np>::_S_first_size == _N0)
3898 !__is_fixed_size_abi_v<typename _V::abi_type>,
3899 "How can <_Tp, _Np> be __a single _SimdTuple entry but __a "
3904 return tuple_cat(make_tuple(_V(__private_init, __data(__x).first)),
3905 __split_wrapper(_SL::template _S_pop_front<1>(),
3906 __data(__x).second));
3908 else if constexpr ((!is_same_v<simd_abi::scalar,
3909 simd_abi::deduce_t<_Tp, _Sizes>> && ...)
3910 && (!__is_fixed_size_abi_v<
3911 simd_abi::deduce_t<_Tp, _Sizes>> && ...))
3913 if constexpr (((_Sizes * 2 == _Np) && ...))
3914 return {{__private_init, __extract_part<0, 2>(__data(__x))},
3915 {__private_init, __extract_part<1, 2>(__data(__x))}};
3916 else if constexpr (is_same_v<_SizeList<_Sizes...>,
3917 _SizeList<_Np / 3, _Np / 3, _Np / 3>>)
3918 return {{__private_init, __extract_part<0, 3>(__data(__x))},
3919 {__private_init, __extract_part<1, 3>(__data(__x))},
3920 {__private_init, __extract_part<2, 3>(__data(__x))}};
3921 else if constexpr (is_same_v<_SizeList<_Sizes...>,
3922 _SizeList<2 * _Np / 3, _Np / 3>>)
3923 return {{__private_init, __extract_part<0, 3, 2>(__data(__x))},
3924 {__private_init, __extract_part<2, 3>(__data(__x))}};
3925 else if constexpr (is_same_v<_SizeList<_Sizes...>,
3926 _SizeList<_Np / 3, 2 * _Np / 3>>)
3927 return {{__private_init, __extract_part<0, 3>(__data(__x))},
3928 {__private_init, __extract_part<1, 3, 2>(__data(__x))}};
3929 else if constexpr (is_same_v<_SizeList<_Sizes...>,
3930 _SizeList<_Np / 2, _Np / 4, _Np / 4>>)
3931 return {{__private_init, __extract_part<0, 2>(__data(__x))},
3932 {__private_init, __extract_part<2, 4>(__data(__x))},
3933 {__private_init, __extract_part<3, 4>(__data(__x))}};
3934 else if constexpr (is_same_v<_SizeList<_Sizes...>,
3935 _SizeList<_Np / 4, _Np / 4, _Np / 2>>)
3936 return {{__private_init, __extract_part<0, 4>(__data(__x))},
3937 {__private_init, __extract_part<1, 4>(__data(__x))},
3938 {__private_init, __extract_part<1, 2>(__data(__x))}};
3939 else if constexpr (is_same_v<_SizeList<_Sizes...>,
3940 _SizeList<_Np / 4, _Np / 2, _Np / 4>>)
3941 return {{__private_init, __extract_part<0, 4>(__data(__x))},
3942 {__private_init, __extract_center(__data(__x))},
3943 {__private_init, __extract_part<3, 4>(__data(__x))}};
3944 else if constexpr (((_Sizes * 4 == _Np) && ...))
3945 return {{__private_init, __extract_part<0, 4>(__data(__x))},
3946 {__private_init, __extract_part<1, 4>(__data(__x))},
3947 {__private_init, __extract_part<2, 4>(__data(__x))},
3948 {__private_init, __extract_part<3, 4>(__data(__x))}};
3951 #ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
3952 const __may_alias<_Tp>*
const __element_ptr
3953 =
reinterpret_cast<const __may_alias<_Tp>*
>(&__x);
3954 return __generate_from_n_evaluations<
sizeof...(_Sizes), _Tuple>([&](
3955 auto __i) constexpr {
3956 using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
3957 constexpr
size_t __offset = _SL::_S_before(__i);
3958 constexpr
size_t __base_align =
alignof(simd<_Tp, _Ap>);
3959 constexpr
size_t __a
3960 = __base_align - ((__offset *
sizeof(_Tp)) % __base_align);
3961 constexpr
size_t __b = ((__a - 1) & __a) ^ __a;
3962 constexpr
size_t __alignment = __b == 0 ? __a : __b;
3963 return _Vi(__element_ptr + __offset, overaligned<__alignment>);
3966 return __generate_from_n_evaluations<
sizeof...(_Sizes), _Tuple>([&](
3967 auto __i) constexpr {
3968 using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
3969 const auto& __xx = __data(__x);
3970 using _Offset = decltype(_SL::_S_before(__i));
3971 return _Vi([&](
auto __j) constexpr {
3972 constexpr _SizeConstant<_Offset::value + __j> __k;
3982 template <
size_t _I,
typename _Tp,
typename _Ap,
typename... _As>
3983 _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
3984 __subscript_in_pack(
const simd<_Tp, _Ap>& __x,
const simd<_Tp, _As>&... __xs)
3986 if constexpr (_I < simd_size_v<_Tp, _Ap>)
3989 return __subscript_in_pack<_I - simd_size_v<_Tp, _Ap>>(__xs...);
3994 template <
typename _Tp,
typename _A0,
typename... _As>
3995 _GLIBCXX_SIMD_INTRINSIC
void
3996 __store_pack_of_simd(
char* __mem,
const simd<_Tp, _A0>& __x0,
3997 const simd<_Tp, _As>&... __xs)
3999 constexpr
size_t __n_bytes =
sizeof(_Tp) * simd_size_v<_Tp, _A0>;
4000 __builtin_memcpy(__mem, &__data(__x0), __n_bytes);
4001 if constexpr (
sizeof...(__xs) > 0)
4002 __store_pack_of_simd(__mem + __n_bytes, __xs...);
4007 template <typename _Tp, typename... _As>
4008 inline _GLIBCXX_SIMD_CONSTEXPR
4009 simd<_Tp, simd_abi::deduce_t<_Tp, (simd_size_v<_Tp, _As> + ...)>>
4010 concat(const simd<_Tp, _As>&... __xs)
4012 using _Rp = __deduced_simd<_Tp, (simd_size_v<_Tp, _As> + ...)>;
4013 if constexpr (
sizeof...(__xs) == 1)
4014 return simd_cast<_Rp>(__xs...);
4015 else if ((... && __xs._M_is_constprop()))
4017 simd_abi::deduce_t<_Tp, (simd_size_v<_Tp, _As> + ...)>>([&](
4018 auto __i) constexpr {
return __subscript_in_pack<__i>(__xs...); });
4022 __store_pack_of_simd(
reinterpret_cast<char*
>(&__data(__r)), __xs...);
4029 template <
typename _Tp,
typename _Abi,
size_t _Np>
4030 _GLIBCXX_SIMD_ALWAYS_INLINE
4031 _GLIBCXX_SIMD_CONSTEXPR __deduced_simd<_Tp, simd_size_v<_Tp, _Abi> * _Np>
4032 concat(
const array<simd<_Tp, _Abi>, _Np>& __x)
4034 return __call_with_subscripts<_Np>(__x, [](
const auto&... __xs) {
4035 return concat(__xs...);
4043 template <
typename _Up,
typename _Accessor = _Up,
4044 typename _ValueType =
typename _Up::value_type>
4045 class _SmartReference
4051 _GLIBCXX_SIMD_INTRINSIC constexpr _ValueType _M_read() const noexcept
4053 if constexpr (is_arithmetic_v<_Up>)
4056 return _M_obj[_M_index];
4059 template <
typename _Tp>
4060 _GLIBCXX_SIMD_INTRINSIC constexpr
void _M_write(_Tp&& __x)
const
4061 { _Accessor::_S_set(_M_obj, _M_index,
static_cast<_Tp&&
>(__x)); }
4064 _GLIBCXX_SIMD_INTRINSIC constexpr
4065 _SmartReference(_Up& __o,
int __i) noexcept
4066 : _M_index(__i), _M_obj(__o) {}
4068 using value_type = _ValueType;
4070 _GLIBCXX_SIMD_INTRINSIC _SmartReference(
const _SmartReference&) =
delete;
4072 _GLIBCXX_SIMD_INTRINSIC constexpr
operator value_type() const noexcept
4073 {
return _M_read(); }
4075 template <
typename _Tp,
4077 = _ValuePreservingOrInt<__remove_cvref_t<_Tp>, value_type>>
4078 _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference operator=(_Tp&& __x) &&
4080 _M_write(
static_cast<_Tp&&
>(__x));
4081 return {_M_obj, _M_index};
4084 #define _GLIBCXX_SIMD_OP_(__op) \
4085 template <typename _Tp, \
4087 = decltype(declval<value_type>() __op declval<_Tp>()), \
4088 typename = _ValuePreservingOrInt<__remove_cvref_t<_Tp>, _TT>, \
4089 typename = _ValuePreservingOrInt<_TT, value_type>> \
4090 _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference \
4091 operator __op##=(_Tp&& __x) && \
4093 const value_type& __lhs = _M_read(); \
4094 _M_write(__lhs __op __x); \
4095 return {_M_obj, _M_index}; \
4097 _GLIBCXX_SIMD_ALL_ARITHMETICS(_GLIBCXX_SIMD_OP_);
4098 _GLIBCXX_SIMD_ALL_SHIFTS(_GLIBCXX_SIMD_OP_);
4099 _GLIBCXX_SIMD_ALL_BINARY(_GLIBCXX_SIMD_OP_);
4100 #undef _GLIBCXX_SIMD_OP_
4102 template <
typename _Tp = void,
4104 = decltype(++
declval<conditional_t<true, value_type, _Tp>&>())>
4105 _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference operator++() &&
4107 value_type __x = _M_read();
4109 return {_M_obj, _M_index};
4112 template <
typename _Tp = void,
4114 = decltype(
declval<conditional_t<true, value_type, _Tp>&>()++)>
4115 _GLIBCXX_SIMD_INTRINSIC constexpr value_type operator++(
int) &&
4117 const value_type __r = _M_read();
4118 value_type __x = __r;
4123 template <
typename _Tp = void,
4125 = decltype(--
declval<conditional_t<true, value_type, _Tp>&>())>
4126 _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference operator--() &&
4128 value_type __x = _M_read();
4130 return {_M_obj, _M_index};
4133 template <
typename _Tp = void,
4135 = decltype(
declval<conditional_t<true, value_type, _Tp>&>()--)>
4136 _GLIBCXX_SIMD_INTRINSIC constexpr value_type operator--(
int) &&
4138 const value_type __r = _M_read();
4139 value_type __x = __r;
4144 _GLIBCXX_SIMD_INTRINSIC
friend void
4145 swap(_SmartReference&& __a, _SmartReference&& __b) noexcept(
4147 is_nothrow_constructible<value_type, _SmartReference&&>,
4148 is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4150 value_type __tmp =
static_cast<_SmartReference&&
>(__a);
4151 static_cast<_SmartReference&&
>(__a) =
static_cast<value_type
>(__b);
4152 static_cast<_SmartReference&&
>(__b) =
std::move(__tmp);
4155 _GLIBCXX_SIMD_INTRINSIC
friend void
4156 swap(value_type& __a, _SmartReference&& __b) noexcept(
4158 is_nothrow_constructible<value_type, value_type&&>,
4159 is_nothrow_assignable<value_type&, value_type&&>,
4160 is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4163 __a =
static_cast<value_type
>(__b);
4164 static_cast<_SmartReference&&
>(__b) =
std::move(__tmp);
4167 _GLIBCXX_SIMD_INTRINSIC
friend void
4168 swap(_SmartReference&& __a, value_type& __b) noexcept(
4170 is_nothrow_constructible<value_type, _SmartReference&&>,
4171 is_nothrow_assignable<value_type&, value_type&&>,
4172 is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4174 value_type __tmp(__a);
4175 static_cast<_SmartReference&&
>(__a) =
std::move(__b);
4182 template <
int _Bytes>
4183 struct __scalar_abi_wrapper
4185 template <
typename _Tp>
static constexpr
size_t _S_full_size = 1;
4186 template <
typename _Tp>
static constexpr
size_t _S_size = 1;
4187 template <
typename _Tp>
static constexpr
size_t _S_is_partial =
false;
4189 template <
typename _Tp,
typename _Abi = simd_abi::scalar>
4190 static constexpr
bool _S_is_valid_v
4191 = _Abi::template _IsValid<_Tp>::value &&
sizeof(_Tp) == _Bytes;
4196 template <
typename _Tp>
4197 struct __decay_abi {
using type = _Tp; };
4199 template <
int _Bytes>
4200 struct __decay_abi<__scalar_abi_wrapper<_Bytes>>
4201 {
using type = simd_abi::scalar; };
4209 template <
template <
int>
class _Abi,
int _Bytes,
typename _Tp>
4210 struct __find_next_valid_abi
4212 static constexpr
auto _S_choose()
4214 constexpr
int _NextBytes = std::__bit_ceil(_Bytes) / 2;
4215 using _NextAbi = _Abi<_NextBytes>;
4216 if constexpr (_NextBytes <
sizeof(_Tp) * 2)
4217 return _Abi<_Bytes>();
4218 else if constexpr (_NextAbi::template _S_is_partial<_Tp> ==
false
4219 && _NextAbi::template _S_is_valid_v<_Tp>)
4222 return __find_next_valid_abi<_Abi, _NextBytes, _Tp>::_S_choose();
4225 using type = decltype(_S_choose());
4228 template <
int _Bytes,
typename _Tp>
4229 struct __find_next_valid_abi<__scalar_abi_wrapper, _Bytes, _Tp>
4230 {
using type = simd_abi::scalar; };
4233 template <
template <
int>
class...>
4236 template <
typename,
int>
static constexpr
bool _S_has_valid_abi =
false;
4237 template <
typename,
int>
using _FirstValidAbi = void;
4238 template <
typename,
int>
using _BestAbi = void;
4241 template <
template <
int>
class _A0,
template <
int>
class... _Rest>
4242 struct _AbiList<_A0, _Rest...>
4244 template <
typename _Tp,
int _Np>
4245 static constexpr
bool _S_has_valid_abi
4246 = _A0<
sizeof(_Tp) * _Np>::
template _S_is_valid_v<
4247 _Tp> || _AbiList<_Rest...>::template _S_has_valid_abi<_Tp, _Np>;
4249 template <
typename _Tp,
int _Np>
4251 _A0<
sizeof(_Tp) * _Np>::
template _S_is_valid_v<_Tp>,
4252 typename __decay_abi<_A0<
sizeof(_Tp) * _Np>>::type,
4253 typename _AbiList<_Rest...>::template _FirstValidAbi<_Tp, _Np>>;
4255 template <
typename _Tp,
int _Np>
4256 static constexpr
auto _S_determine_best_abi()
4258 static_assert(_Np >= 1);
4259 constexpr
int _Bytes =
sizeof(_Tp) * _Np;
4260 if constexpr (_Np == 1)
4261 return __make_dependent_t<_Tp, simd_abi::scalar>{};
4264 constexpr
int __fullsize = _A0<_Bytes>::template _S_full_size<_Tp>;
4269 if constexpr (_A0<_Bytes>::template _S_is_valid_v<
4270 _Tp> && __fullsize / 2 < _Np)
4271 return typename __decay_abi<_A0<_Bytes>>::type{};
4275 typename __find_next_valid_abi<_A0, _Bytes, _Tp>::type;
4276 if constexpr (_Bp::template _S_is_valid_v<
4277 _Tp> && _Bp::template _S_size<_Tp> <= _Np)
4281 typename _AbiList<_Rest...>::template _BestAbi<_Tp, _Np>{};
4286 template <
typename _Tp,
int _Np>
4287 using _BestAbi = decltype(_S_determine_best_abi<_Tp, _Np>());
4295 using _AllNativeAbis = _AbiList<simd_abi::_VecBltnBtmsk, simd_abi::_VecBuiltin,
4296 __scalar_abi_wrapper>;
4299 template <
typename _Tp,
typename _Abi>
4300 struct _SimdTraits<_Tp, _Abi,
void_t<typename _Abi::template _IsValid<_Tp>>>
4301 : _Abi::template __traits<_Tp> {};
4305 template <
typename _Tp,
size_t _Np>
4306 struct __deduce_impl<
4307 _Tp, _Np,
enable_if_t<_AllNativeAbis::template _S_has_valid_abi<_Tp, _Np>>>
4308 {
using type = _AllNativeAbis::_FirstValidAbi<_Tp, _Np>; };
4311 template <
typename _Tp,
size_t _Np,
typename =
void>
4312 struct __deduce_fixed_size_fallback {};
4314 template <
typename _Tp,
size_t _Np>
4315 struct __deduce_fixed_size_fallback<_Tp, _Np,
4316 enable_if_t<simd_abi::fixed_size<_Np>::template _S_is_valid_v<_Tp>>>
4317 {
using type = simd_abi::fixed_size<_Np>; };
4319 template <
typename _Tp,
size_t _Np,
typename>
4320 struct __deduce_impl :
public __deduce_fixed_size_fallback<_Tp, _Np> {};
4326 template <
typename _Tp,
typename _Abi>
4327 class simd_mask :
public _SimdTraits<_Tp, _Abi>::_MaskBase
4330 using _Traits = _SimdTraits<_Tp, _Abi>;
4331 using _MemberType =
typename _Traits::_MaskMember;
4337 using _Ip = __int_for_sizeof_t<_Tp>;
4338 static constexpr _Ip* _S_type_tag =
nullptr;
4340 friend typename _Traits::_MaskBase;
4341 friend class simd<_Tp, _Abi>;
4342 friend typename _Traits::_SimdImpl;
4345 using _Impl =
typename _Traits::_MaskImpl;
4350 using value_type = bool;
4351 using reference = _SmartReference<_MemberType, _Impl, value_type>;
4352 using simd_type = simd<_Tp, _Abi>;
4353 using abi_type = _Abi;
4356 static constexpr
size_t size()
4357 {
return __size_or_zero_v<_Tp, _Abi>; }
4361 simd_mask() =
default;
4362 simd_mask(
const simd_mask&) =
default;
4363 simd_mask(simd_mask&&) =
default;
4364 simd_mask& operator=(
const simd_mask&) =
default;
4365 simd_mask& operator=(simd_mask&&) =
default;
4369 _GLIBCXX_SIMD_ALWAYS_INLINE
explicit
4370 simd_mask(
typename _Traits::_MaskCastType __init)
4371 : _M_data{__init} {}
4380 _GLIBCXX_SIMD_ALWAYS_INLINE
static simd_mask
4381 __from_bitset(bitset<size()> bs)
4382 {
return {__bitset_init, bs}; }
4384 _GLIBCXX_SIMD_ALWAYS_INLINE bitset<size()>
4386 {
return _Impl::_S_to_bits(_M_data)._M_to_bitset(); }
4390 _GLIBCXX_SIMD_ALWAYS_INLINE
explicit _GLIBCXX_SIMD_CONSTEXPR
4391 simd_mask(value_type __x)
4392 : _M_data(_Impl::template _S_broadcast<_Ip>(__x)) {}
4396 #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4398 template <
typename _Up,
typename _A2,
4399 typename = enable_if_t<simd_size_v<_Up, _A2> == size()>>
4400 _GLIBCXX_SIMD_ALWAYS_INLINE
explicit(
sizeof(_MemberType)
4401 !=
sizeof(
typename _SimdTraits<_Up, _A2>::_MaskMember))
4402 simd_mask(
const simd_mask<_Up, _A2>& __x)
4403 : simd_mask(__proposed::static_simd_cast<simd_mask>(__x)) {}
4406 template <
typename _Up,
typename =
enable_if_t<conjunction<
4407 is_same<abi_type, simd_abi::fixed_size<size()>>,
4408 is_same<_Up, _Up>>::value>>
4409 _GLIBCXX_SIMD_ALWAYS_INLINE
4410 simd_mask(
const simd_mask<_Up, simd_abi::fixed_size<size()>>& __x)
4411 : _M_data(_Impl::_S_from_bitmask(__data(__x), _S_type_tag)) {}
4416 template <
typename _Flags>
4417 _GLIBCXX_SIMD_ALWAYS_INLINE
4418 simd_mask(
const value_type* __mem, _Flags)
4419 : _M_data(_Impl::template _S_load<_Ip>(
4420 _Flags::template _S_apply<simd_mask>(__mem))) {}
4422 template <
typename _Flags>
4423 _GLIBCXX_SIMD_ALWAYS_INLINE
4424 simd_mask(
const value_type* __mem, simd_mask __k, _Flags)
4428 = _Impl::_S_masked_load(_M_data, __k._M_data,
4429 _Flags::template _S_apply<simd_mask>(__mem));
4434 template <
typename _Flags>
4435 _GLIBCXX_SIMD_ALWAYS_INLINE
void
4436 copy_from(
const value_type* __mem, _Flags)
4438 _M_data = _Impl::template _S_load<_Ip>(
4439 _Flags::template _S_apply<simd_mask>(__mem));
4444 template <
typename _Flags>
4445 _GLIBCXX_SIMD_ALWAYS_INLINE
void
4446 copy_to(value_type* __mem, _Flags)
const
4447 { _Impl::_S_store(_M_data, _Flags::template _S_apply<simd_mask>(__mem)); }
4451 _GLIBCXX_SIMD_ALWAYS_INLINE reference
4452 operator[](
size_t __i)
4455 __invoke_ub(
"Subscript %d is out of range [0, %d]", __i, size() - 1);
4456 return {_M_data, int(__i)};
4459 _GLIBCXX_SIMD_ALWAYS_INLINE value_type
4460 operator[](
size_t __i)
const
4463 __invoke_ub(
"Subscript %d is out of range [0, %d]", __i, size() - 1);
4464 if constexpr (__is_scalar_abi<_Abi>())
4467 return static_cast<
bool>(_M_data[__i]);
4472 _GLIBCXX_SIMD_ALWAYS_INLINE simd_mask
4474 {
return {__private_init, _Impl::_S_bit_not(_M_data)}; }
4478 #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4480 template <
typename _Up,
typename _A2,
4482 = enable_if_t<is_convertible_v<simd_mask<_Up, _A2>, simd_mask>>>
4483 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask
4484 operator&&(
const simd_mask& __x,
const simd_mask<_Up, _A2>& __y)
4486 return {__private_init,
4487 _Impl::_S_logical_and(__x._M_data, simd_mask(__y)._M_data)};
4490 template <
typename _Up,
typename _A2,
4492 = enable_if_t<is_convertible_v<simd_mask<_Up, _A2>, simd_mask>>>
4493 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask
4494 operator||(
const simd_mask& __x,
const simd_mask<_Up, _A2>& __y)
4496 return {__private_init,
4497 _Impl::_S_logical_or(__x._M_data, simd_mask(__y)._M_data)};
4501 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask
4502 operator&&(
const simd_mask& __x,
const simd_mask& __y)
4504 return {__private_init, _Impl::_S_logical_and(__x._M_data, __y._M_data)};
4507 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask
4508 operator||(
const simd_mask& __x,
const simd_mask& __y)
4510 return {__private_init, _Impl::_S_logical_or(__x._M_data, __y._M_data)};
4513 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask
4514 operator&(
const simd_mask& __x,
const simd_mask& __y)
4515 {
return {__private_init, _Impl::_S_bit_and(__x._M_data, __y._M_data)}; }
4517 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask
4518 operator|(
const simd_mask& __x,
const simd_mask& __y)
4519 {
return {__private_init, _Impl::_S_bit_or(__x._M_data, __y._M_data)}; }
4521 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask
4522 operator^(
const simd_mask& __x,
const simd_mask& __y)
4523 {
return {__private_init, _Impl::_S_bit_xor(__x._M_data, __y._M_data)}; }
4525 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask&
4526 operator&=(simd_mask& __x,
const simd_mask& __y)
4528 __x._M_data = _Impl::_S_bit_and(__x._M_data, __y._M_data);
4532 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask&
4533 operator|=(simd_mask& __x,
const simd_mask& __y)
4535 __x._M_data = _Impl::_S_bit_or(__x._M_data, __y._M_data);
4539 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask&
4540 operator^=(simd_mask& __x,
const simd_mask& __y)
4542 __x._M_data = _Impl::_S_bit_xor(__x._M_data, __y._M_data);
4548 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask
4549 operator==(
const simd_mask& __x,
const simd_mask& __y)
4550 {
return !operator!=(__x, __y); }
4552 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask
4553 operator!=(
const simd_mask& __x,
const simd_mask& __y)
4554 {
return {__private_init, _Impl::_S_bit_xor(__x._M_data, __y._M_data)}; }
4558 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
4559 simd_mask(_PrivateInit,
typename _Traits::_MaskMember __init)
4560 : _M_data(__init) {}
4564 template <typename _Fp, typename = decltype(bool(declval<_Fp>()(size_t())))>
4565 _GLIBCXX_SIMD_INTRINSIC constexpr
4566 simd_mask(_PrivateInit, _Fp&& __gen)
4569 __execute_n_times<size()>([&](
auto __i) constexpr {
4570 _Impl::_S_set(_M_data, __i, __gen(__i));
4576 _GLIBCXX_SIMD_INTRINSIC simd_mask(_BitsetInit, bitset<size()> __init)
4578 _Impl::_S_from_bitmask(_SanitizedBitMask<size()>(__init), _S_type_tag))
4589 template <
typename _Up,
typename _A2,
4591 = enable_if_t<simd_size_v<_Up, _A2> == simd_size_v<_Tp, _Abi>>>
4592 operator simd_mask<_Up, _A2>() &&
4594 using namespace std::experimental::__proposed;
4595 return static_simd_cast<simd_mask<_Up, _A2>>(_M_data);
4598 const simd_mask<_Tp, _Abi>& _M_data;
4601 _GLIBCXX_SIMD_INTRINSIC _CvtProxy
4607 #ifdef __GXX_CONDITIONAL_IS_OVERLOADABLE__
4608 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask
4609 operator?:(
const simd_mask& __k,
const simd_mask& __where_true,
4610 const simd_mask& __where_false)
4612 auto __ret = __where_false;
4613 _Impl::_S_masked_assign(__k._M_data, __ret._M_data, __where_true._M_data);
4617 template <
typename _U1,
typename _U2,
4618 typename _Rp = simd<common_type_t<_U1, _U2>, _Abi>,
4620 is_convertible<_U1, _Rp>, is_convertible<_U2, _Rp>,
4621 is_convertible<simd_mask, typename _Rp::mask_type>>>>
4622 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend _Rp
4623 operator?:(
const simd_mask& __k,
const _U1& __where_true,
4624 const _U2& __where_false)
4626 _Rp __ret = __where_false;
4627 _Rp::_Impl::_S_masked_assign(
4628 __data(
static_cast<typename _Rp::mask_type
>(__k)), __data(__ret),
4629 __data(
static_cast<_Rp
>(__where_true)));
4633 #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4634 template <
typename _Kp,
typename _Ak,
typename _Up,
typename _Au,
4636 conjunction_v<is_convertible<simd_mask<_Kp, _Ak>, simd_mask>,
4637 is_convertible<simd_mask<_Up, _Au>, simd_mask>>>>
4638 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask
4639 operator?:(
const simd_mask<_Kp, _Ak>& __k,
const simd_mask& __where_true,
4640 const simd_mask<_Up, _Au>& __where_false)
4642 simd_mask __ret = __where_false;
4643 _Impl::_S_masked_assign(simd_mask(__k)._M_data, __ret._M_data,
4644 __where_true._M_data);
4652 _GLIBCXX_SIMD_INTRINSIC constexpr
bool
4653 _M_is_constprop()
const
4655 if constexpr (__is_scalar_abi<_Abi>())
4656 return __builtin_constant_p(_M_data);
4658 return _M_data._M_is_constprop();
4664 friend const auto& __data<_Tp, abi_type>(const simd_mask&);
4665 friend auto& __data<_Tp, abi_type>(simd_mask&);
4666 alignas(_Traits::_S_mask_align) _MemberType _M_data;
4673 template <typename _Tp, typename _Ap>
4674 _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
4675 __data(const simd_mask<_Tp, _Ap>& __x)
4676 {
return __x._M_data; }
4678 template <
typename _Tp,
typename _Ap>
4679 _GLIBCXX_SIMD_INTRINSIC constexpr
auto&
4680 __data(simd_mask<_Tp, _Ap>& __x)
4681 {
return __x._M_data; }
4687 template <
typename _Tp,
typename _Abi>
4688 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
4689 all_of(
const simd_mask<_Tp, _Abi>& __k) noexcept
4691 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4693 for (
size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
4699 return _Abi::_MaskImpl::_S_all_of(__k);
4702 template <
typename _Tp,
typename _Abi>
4703 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
4704 any_of(
const simd_mask<_Tp, _Abi>& __k) noexcept
4706 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4708 for (
size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
4714 return _Abi::_MaskImpl::_S_any_of(__k);
4717 template <
typename _Tp,
typename _Abi>
4718 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
4719 none_of(
const simd_mask<_Tp, _Abi>& __k) noexcept
4721 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4723 for (
size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
4729 return _Abi::_MaskImpl::_S_none_of(__k);
4732 template <
typename _Tp,
typename _Abi>
4733 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
4734 some_of(
const simd_mask<_Tp, _Abi>& __k) noexcept
4736 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4738 for (
size_t __i = 1; __i < simd_size_v<_Tp, _Abi>; ++__i)
4739 if (__k[__i] != __k[__i - 1])
4744 return _Abi::_MaskImpl::_S_some_of(__k);
4747 template <
typename _Tp,
typename _Abi>
4748 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
int
4749 popcount(
const simd_mask<_Tp, _Abi>& __k) noexcept
4751 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4753 const int __r = __call_with_subscripts<simd_size_v<_Tp, _Abi>>(
4754 __k, [](
auto... __elements) {
return ((__elements != 0) + ...); });
4755 if (__builtin_is_constant_evaluated() || __builtin_constant_p(__r))
4758 return _Abi::_MaskImpl::_S_popcount(__k);
4761 template <
typename _Tp,
typename _Abi>
4762 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
int
4763 find_first_set(
const simd_mask<_Tp, _Abi>& __k)
4765 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4767 constexpr
size_t _Np = simd_size_v<_Tp, _Abi>;
4768 const size_t _Idx = __call_with_n_evaluations<_Np>(
4769 [](
auto... __indexes) {
return std::min({__indexes...}); },
4770 [&](
auto __i) {
return __k[__i] ? +__i : _Np; });
4772 __invoke_ub(
"find_first_set(empty mask) is UB");
4773 if (__builtin_constant_p(_Idx))
4776 return _Abi::_MaskImpl::_S_find_first_set(__k);
4779 template <
typename _Tp,
typename _Abi>
4780 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
int
4781 find_last_set(
const simd_mask<_Tp, _Abi>& __k)
4783 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4785 constexpr
size_t _Np = simd_size_v<_Tp, _Abi>;
4786 const int _Idx = __call_with_n_evaluations<_Np>(
4787 [](
auto... __indexes) {
return std::max({__indexes...}); },
4788 [&](
auto __i) {
return __k[__i] ? int(__i) : -1; });
4790 __invoke_ub(
"find_first_set(empty mask) is UB");
4791 if (__builtin_constant_p(_Idx))
4794 return _Abi::_MaskImpl::_S_find_last_set(__k);
4797 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
4798 all_of(_ExactBool __x) noexcept
4801 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
4802 any_of(_ExactBool __x) noexcept
4805 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
4806 none_of(_ExactBool __x) noexcept
4809 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
4810 some_of(_ExactBool) noexcept
4813 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
int
4817 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
int
4818 find_first_set(_ExactBool)
4821 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
int
4822 find_last_set(_ExactBool)
4829 template <
typename _V,
typename _Impl,
bool>
4830 class _SimdIntOperators {};
4832 template <
typename _V,
typename _Impl>
4833 class _SimdIntOperators<_V, _Impl, true>
4835 _GLIBCXX_SIMD_INTRINSIC
const _V& __derived()
const
4836 {
return *
static_cast<const _V*
>(
this); }
4838 template <
typename _Tp>
4839 _GLIBCXX_SIMD_INTRINSIC
static _GLIBCXX_SIMD_CONSTEXPR _V
4840 _S_make_derived(_Tp&& __d)
4841 {
return {__private_init,
static_cast<_Tp&&
>(__d)}; }
4844 _GLIBCXX_SIMD_CONSTEXPR
friend _V& operator%=(_V& __lhs,
const _V& __x)
4845 {
return __lhs = __lhs % __x; }
4847 _GLIBCXX_SIMD_CONSTEXPR
friend _V& operator&=(_V& __lhs,
const _V& __x)
4848 {
return __lhs = __lhs & __x; }
4850 _GLIBCXX_SIMD_CONSTEXPR
friend _V& operator|=(_V& __lhs,
const _V& __x)
4851 {
return __lhs = __lhs | __x; }
4853 _GLIBCXX_SIMD_CONSTEXPR
friend _V& operator^=(_V& __lhs,
const _V& __x)
4854 {
return __lhs = __lhs ^ __x; }
4856 _GLIBCXX_SIMD_CONSTEXPR
friend _V& operator<<=(_V& __lhs,
const _V& __x)
4857 {
return __lhs = __lhs << __x; }
4859 _GLIBCXX_SIMD_CONSTEXPR
friend _V& operator>>=(_V& __lhs,
const _V& __x)
4860 {
return __lhs = __lhs >> __x; }
4862 _GLIBCXX_SIMD_CONSTEXPR
friend _V& operator<<=(_V& __lhs,
int __x)
4863 {
return __lhs = __lhs << __x; }
4865 _GLIBCXX_SIMD_CONSTEXPR
friend _V& operator>>=(_V& __lhs,
int __x)
4866 {
return __lhs = __lhs >> __x; }
4868 _GLIBCXX_SIMD_CONSTEXPR
friend _V operator%(
const _V& __x,
const _V& __y)
4870 return _SimdIntOperators::_S_make_derived(
4871 _Impl::_S_modulus(__data(__x), __data(__y)));
4874 _GLIBCXX_SIMD_CONSTEXPR
friend _V
operator&(
const _V& __x,
const _V& __y)
4876 return _SimdIntOperators::_S_make_derived(
4877 _Impl::_S_bit_and(__data(__x), __data(__y)));
4880 _GLIBCXX_SIMD_CONSTEXPR
friend _V
operator|(
const _V& __x,
const _V& __y)
4882 return _SimdIntOperators::_S_make_derived(
4883 _Impl::_S_bit_or(__data(__x), __data(__y)));
4886 _GLIBCXX_SIMD_CONSTEXPR
friend _V
operator^(
const _V& __x,
const _V& __y)
4888 return _SimdIntOperators::_S_make_derived(
4889 _Impl::_S_bit_xor(__data(__x), __data(__y)));
4892 _GLIBCXX_SIMD_CONSTEXPR
friend _V
operator<<(
const _V& __x,
const _V& __y)
4894 return _SimdIntOperators::_S_make_derived(
4895 _Impl::_S_bit_shift_left(__data(__x), __data(__y)));
4898 _GLIBCXX_SIMD_CONSTEXPR
friend _V
operator>>(
const _V& __x,
const _V& __y)
4900 return _SimdIntOperators::_S_make_derived(
4901 _Impl::_S_bit_shift_right(__data(__x), __data(__y)));
4904 template <
typename _VV = _V>
4905 _GLIBCXX_SIMD_CONSTEXPR
friend _V
operator<<(
const _V& __x,
int __y)
4907 using _Tp =
typename _VV::value_type;
4909 __invoke_ub(
"The behavior is undefined if the right operand of a "
4910 "shift operation is negative. [expr.shift]\nA shift by "
4913 if (
size_t(__y) >=
sizeof(declval<_Tp>() << __y) * __CHAR_BIT__)
4915 "The behavior is undefined if the right operand of a "
4916 "shift operation is greater than or equal to the width of the "
4917 "promoted left operand. [expr.shift]\nA shift by %d was requested",
4919 return _SimdIntOperators::_S_make_derived(
4920 _Impl::_S_bit_shift_left(__data(__x), __y));
4923 template <
typename _VV = _V>
4924 _GLIBCXX_SIMD_CONSTEXPR
friend _V
operator>>(
const _V& __x,
int __y)
4926 using _Tp =
typename _VV::value_type;
4929 "The behavior is undefined if the right operand of a shift "
4930 "operation is negative. [expr.shift]\nA shift by %d was requested",
4932 if (
size_t(__y) >=
sizeof(declval<_Tp>() << __y) * __CHAR_BIT__)
4934 "The behavior is undefined if the right operand of a shift "
4935 "operation is greater than or equal to the width of the promoted "
4936 "left operand. [expr.shift]\nA shift by %d was requested",
4938 return _SimdIntOperators::_S_make_derived(
4939 _Impl::_S_bit_shift_right(__data(__x), __y));
4943 _GLIBCXX_SIMD_CONSTEXPR _V operator~()
const
4944 {
return {__private_init, _Impl::_S_complement(__derived()._M_data)}; }
4951 template <
typename _Tp,
typename _Abi>
4952 class simd :
public _SimdIntOperators<
4953 simd<_Tp, _Abi>, typename _SimdTraits<_Tp, _Abi>::_SimdImpl,
4954 conjunction<is_integral<_Tp>,
4955 typename _SimdTraits<_Tp, _Abi>::_IsValid>::value>,
4956 public _SimdTraits<_Tp, _Abi>::_SimdBase
4958 using _Traits = _SimdTraits<_Tp, _Abi>;
4959 using _MemberType =
typename _Traits::_SimdMember;
4960 using _CastType =
typename _Traits::_SimdCastType;
4961 static constexpr _Tp* _S_type_tag =
nullptr;
4962 friend typename _Traits::_SimdBase;
4965 using _Impl =
typename _Traits::_SimdImpl;
4967 friend _SimdIntOperators<simd, _Impl, true>;
4969 using value_type = _Tp;
4970 using reference = _SmartReference<_MemberType, _Impl, value_type>;
4971 using mask_type = simd_mask<_Tp, _Abi>;
4972 using abi_type = _Abi;
4974 static constexpr
size_t size()
4975 {
return __size_or_zero_v<_Tp, _Abi>; }
4977 _GLIBCXX_SIMD_CONSTEXPR simd() =
default;
4978 _GLIBCXX_SIMD_CONSTEXPR simd(
const simd&) =
default;
4979 _GLIBCXX_SIMD_CONSTEXPR simd(simd&&) noexcept = default;
4980 _GLIBCXX_SIMD_CONSTEXPR simd& operator=(const simd&) = default;
4981 _GLIBCXX_SIMD_CONSTEXPR simd& operator=(simd&&) noexcept = default;
4984 template <typename _Up,
4985 typename =
enable_if_t<!is_same_v<__remove_cvref_t<_Up>,
bool>>>
4986 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
4987 simd(_ValuePreservingOrInt<_Up, value_type>&& __x)
4989 _Impl::_S_broadcast(static_cast<value_type>(static_cast<_Up&&>(__x))))
4994 template <
typename _Up>
4995 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
4996 simd(
const simd<_Up, simd_abi::fixed_size<size()>>& __x,
4999 is_same<simd_abi::fixed_size<size()>, abi_type>,
5000 negation<__is_narrowing_conversion<_Up, value_type>>,
5001 __converts_to_higher_integer_rank<_Up, value_type>>::value,
5003 : simd{static_cast<array<_Up, size()>>(__x).data(), vector_aligned} {}
5006 #ifdef _GLIBCXX_SIMD_ENABLE_STATIC_CAST
5007 template <
typename _Up,
typename _A2,
5008 typename = decltype(static_simd_cast<simd>(
5009 declval<
const simd<_Up, _A2>&>()))>
5010 _GLIBCXX_SIMD_ALWAYS_INLINE
explicit _GLIBCXX_SIMD_CONSTEXPR
5011 simd(
const simd<_Up, _A2>& __x)
5012 : simd(static_simd_cast<simd>(__x)) {}
5016 template <
typename _Fp>
5017 _GLIBCXX_SIMD_ALWAYS_INLINE
explicit _GLIBCXX_SIMD_CONSTEXPR
5018 simd(_Fp&& __gen, _ValuePreservingOrInt<decltype(declval<_Fp>()(
5019 declval<_SizeConstant<0>&>())),
5020 value_type>* =
nullptr)
5021 : _M_data(_Impl::_S_generator(static_cast<_Fp&&>(__gen), _S_type_tag)) {}
5024 template <
typename _Up,
typename _Flags>
5025 _GLIBCXX_SIMD_ALWAYS_INLINE
5026 simd(
const _Up* __mem, _Flags)
5028 _Impl::_S_load(_Flags::template _S_apply<simd>(__mem), _S_type_tag))
5032 template <
typename _Up,
typename _Flags>
5033 _GLIBCXX_SIMD_ALWAYS_INLINE
void
5034 copy_from(
const _Vectorizable<_Up>* __mem, _Flags)
5036 _M_data =
static_cast<decltype(_M_data)
>(
5037 _Impl::_S_load(_Flags::template _S_apply<simd>(__mem), _S_type_tag));
5041 template <
typename _Up,
typename _Flags>
5042 _GLIBCXX_SIMD_ALWAYS_INLINE
void
5043 copy_to(_Vectorizable<_Up>* __mem, _Flags)
const
5045 _Impl::_S_store(_M_data, _Flags::template _S_apply<simd>(__mem),
5050 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR reference
5051 operator[](
size_t __i)
5052 {
return {_M_data, int(__i)}; }
5054 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR value_type
5055 operator[]([[maybe_unused]]
size_t __i)
const
5057 if constexpr (__is_scalar_abi<_Abi>())
5059 _GLIBCXX_DEBUG_ASSERT(__i == 0);
5063 return _M_data[__i];
5067 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd&
5070 _Impl::_S_increment(_M_data);
5074 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5078 _Impl::_S_increment(_M_data);
5082 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd&
5085 _Impl::_S_decrement(_M_data);
5089 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5093 _Impl::_S_decrement(_M_data);
5098 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR mask_type
5100 {
return {__private_init, _Impl::_S_negate(_M_data)}; }
5102 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5106 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5108 {
return {__private_init, _Impl::_S_unary_minus(_M_data)}; }
5111 _GLIBCXX_SIMD_ALWAYS_INLINE
explicit _GLIBCXX_SIMD_CONSTEXPR
5112 simd(_CastType __init) : _M_data(__init) {}
5115 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd&
5116 operator+=(simd& __lhs,
const simd& __x)
5117 {
return __lhs = __lhs + __x; }
5119 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd&
5120 operator-=(simd& __lhs,
const simd& __x)
5121 {
return __lhs = __lhs - __x; }
5123 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd&
5124 operator*=(simd& __lhs,
const simd& __x)
5125 {
return __lhs = __lhs * __x; }
5127 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd&
5128 operator/=(simd& __lhs,
const simd& __x)
5129 {
return __lhs = __lhs / __x; }
5132 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd
5133 operator+(
const simd& __x,
const simd& __y)
5134 {
return {__private_init, _Impl::_S_plus(__x._M_data, __y._M_data)}; }
5136 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd
5137 operator-(
const simd& __x,
const simd& __y)
5138 {
return {__private_init, _Impl::_S_minus(__x._M_data, __y._M_data)}; }
5140 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd
5141 operator*(
const simd& __x,
const simd& __y)
5142 {
return {__private_init, _Impl::_S_multiplies(__x._M_data, __y._M_data)}; }
5144 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd
5145 operator/(
const simd& __x,
const simd& __y)
5146 {
return {__private_init, _Impl::_S_divides(__x._M_data, __y._M_data)}; }
5149 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend mask_type
5150 operator==(
const simd& __x,
const simd& __y)
5151 {
return simd::_S_make_mask(_Impl::_S_equal_to(__x._M_data, __y._M_data)); }
5153 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend mask_type
5154 operator!=(
const simd& __x,
const simd& __y)
5156 return simd::_S_make_mask(
5157 _Impl::_S_not_equal_to(__x._M_data, __y._M_data));
5160 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend mask_type
5161 operator<(
const simd& __x,
const simd& __y)
5162 {
return simd::_S_make_mask(_Impl::_S_less(__x._M_data, __y._M_data)); }
5164 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend mask_type
5165 operator<=(
const simd& __x,
const simd& __y)
5167 return simd::_S_make_mask(_Impl::_S_less_equal(__x._M_data, __y._M_data));
5170 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend mask_type
5171 operator>(
const simd& __x,
const simd& __y)
5172 {
return simd::_S_make_mask(_Impl::_S_less(__y._M_data, __x._M_data)); }
5174 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend mask_type
5175 operator>=(
const simd& __x,
const simd& __y)
5177 return simd::_S_make_mask(_Impl::_S_less_equal(__y._M_data, __x._M_data));
5181 #ifdef __GXX_CONDITIONAL_IS_OVERLOADABLE__
5182 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd
5183 operator?:(
const mask_type& __k,
const simd& __where_true,
5184 const simd& __where_false)
5186 auto __ret = __where_false;
5187 _Impl::_S_masked_assign(__data(__k), __data(__ret), __data(__where_true));
5195 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
5196 simd(_PrivateInit,
const _MemberType& __init)
5197 : _M_data(__init) {}
5200 _GLIBCXX_SIMD_INTRINSIC
5201 simd(_BitsetInit, bitset<size()> __init) : _M_data()
5202 { where(mask_type(__bitset_init, __init), *
this) = ~*
this; }
5204 _GLIBCXX_SIMD_INTRINSIC constexpr
bool
5205 _M_is_constprop()
const
5207 if constexpr (__is_scalar_abi<_Abi>())
5208 return __builtin_constant_p(_M_data);
5210 return _M_data._M_is_constprop();
5214 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR static mask_type
5215 _S_make_mask(typename mask_type::_MemberType __k)
5216 {
return {__private_init, __k}; }
5218 friend const auto& __data<value_type, abi_type>(
const simd&);
5219 friend auto& __data<value_type, abi_type>(simd&);
5220 alignas(_Traits::_S_simd_align) _MemberType _M_data;
5226 template <
typename _Tp,
typename _Ap>
5227 _GLIBCXX_SIMD_INTRINSIC constexpr
const auto&
5228 __data(
const simd<_Tp, _Ap>& __x)
5229 {
return __x._M_data; }
5231 template <
typename _Tp,
typename _Ap>
5232 _GLIBCXX_SIMD_INTRINSIC constexpr
auto&
5233 __data(simd<_Tp, _Ap>& __x)
5234 {
return __x._M_data; }
5237 namespace __float_bitwise_operators {
5238 template <
typename _Tp,
typename _Ap>
5239 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5240 operator^(
const simd<_Tp, _Ap>& __a,
const simd<_Tp, _Ap>& __b)
5242 return {__private_init,
5243 _Ap::_SimdImpl::_S_bit_xor(__data(__a), __data(__b))};
5246 template <
typename _Tp,
typename _Ap>
5247 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5248 operator|(
const simd<_Tp, _Ap>& __a,
const simd<_Tp, _Ap>& __b)
5250 return {__private_init,
5251 _Ap::_SimdImpl::_S_bit_or(__data(__a), __data(__b))};
5254 template <
typename _Tp,
typename _Ap>
5255 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5256 operator&(
const simd<_Tp, _Ap>& __a,
const simd<_Tp, _Ap>& __b)
5258 return {__private_init,
5259 _Ap::_SimdImpl::_S_bit_and(__data(__a), __data(__b))};
5262 template <
typename _Tp,
typename _Ap>
5263 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
5264 enable_if_t<is_floating_point_v<_Tp>, simd<_Tp, _Ap>>
5265 operator~(
const simd<_Tp, _Ap>& __a)
5266 {
return {__private_init, _Ap::_SimdImpl::_S_complement(__data(__a))}; }
5271 _GLIBCXX_SIMD_END_NAMESPACE
constexpr _If_is_unsigned_integer< _Tp, int > popcount(_Tp __x) noexcept
The number of bits set in x.
constexpr complex< _Tp > operator*(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x times y.
constexpr complex< _Tp > operator/(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x divided by y.
constexpr complex< _Tp > operator-(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x minus y.
constexpr complex< _Tp > operator+(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x plus y.
typename remove_reference< _Tp >::type remove_reference_t
Alias template for remove_reference.
typename make_unsigned< _Tp >::type make_unsigned_t
Alias template for make_unsigned.
void void_t
A metafunction that always yields void, used for detecting valid types.
integral_constant< bool, true > true_type
The type used as a compile-time boolean with true value.
typename conditional< _Cond, _Iftrue, _Iffalse >::type conditional_t
Alias template for conditional.
typename remove_pointer< _Tp >::type remove_pointer_t
Alias template for remove_pointer.
integral_constant< bool, false > false_type
The type used as a compile-time boolean with false value.
typename remove_const< _Tp >::type remove_const_t
Alias template for remove_const.
typename enable_if< _Cond, _Tp >::type enable_if_t
Alias template for enable_if.
constexpr auto tuple_cat(_Tpls &&... __tpls) -> typename __tuple_cat_result< _Tpls... >::__type
tuple_cat
auto declval() noexcept -> decltype(__declval< _Tp >(0))
constexpr std::remove_reference< _Tp >::type && move(_Tp &&__t) noexcept
Convert a value to an rvalue.
constexpr const _Tp & max(const _Tp &, const _Tp &)
This does what you think it does.
constexpr const _Tp & min(const _Tp &, const _Tp &)
This does what you think it does.
constexpr _Tp reduce(_InputIterator __first, _InputIterator __last, _Tp __init, _BinaryOperation __binary_op)
Calculate reduction of values in a range.
std::basic_istream< _CharT, _Traits > & operator>>(std::basic_istream< _CharT, _Traits > &__is, bitset< _Nb > &__x)
Global I/O operators for bitsets.
bitset< _Nb > operator^(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
std::basic_ostream< _CharT, _Traits > & operator<<(std::basic_ostream< _CharT, _Traits > &__os, const bitset< _Nb > &__x)
Global I/O operators for bitsets.
bitset< _Nb > operator|(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
bitset< _Nb > operator&(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.