libstdc++
simd.h
1 // Definition of the public simd interfaces -*- C++ -*-
2 
3 // Copyright (C) 2020-2021 Free Software Foundation, Inc.
4 //
5 // This file is part of the GNU ISO C++ Library. This library is free
6 // software; you can redistribute it and/or modify it under the
7 // terms of the GNU General Public License as published by the
8 // Free Software Foundation; either version 3, or (at your option)
9 // any later version.
10 
11 // This library is distributed in the hope that it will be useful,
12 // but WITHOUT ANY WARRANTY; without even the implied warranty of
13 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 // GNU General Public License for more details.
15 
16 // Under Section 7 of GPL version 3, you are granted additional
17 // permissions described in the GCC Runtime Library Exception, version
18 // 3.1, as published by the Free Software Foundation.
19 
20 // You should have received a copy of the GNU General Public License and
21 // a copy of the GCC Runtime Library Exception along with this program;
22 // see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
23 // <http://www.gnu.org/licenses/>.
24 
25 #ifndef _GLIBCXX_EXPERIMENTAL_SIMD_H
26 #define _GLIBCXX_EXPERIMENTAL_SIMD_H
27 
28 #if __cplusplus >= 201703L
29 
30 #include "simd_detail.h"
31 #include "numeric_traits.h"
32 #include <bit>
33 #include <bitset>
34 #ifdef _GLIBCXX_DEBUG_UB
35 #include <cstdio> // for stderr
36 #endif
37 #include <cstring>
38 #include <functional>
39 #include <iosfwd>
40 #include <utility>
41 
42 #if _GLIBCXX_SIMD_X86INTRIN
43 #include <x86intrin.h>
44 #elif _GLIBCXX_SIMD_HAVE_NEON
45 #include <arm_neon.h>
46 #endif
47 
48 /** @ingroup ts_simd
49  * @{
50  */
51 /* There are several closely related types, with the following naming
52  * convention:
53  * _Tp: vectorizable (arithmetic) type (or any type)
54  * _TV: __vector_type_t<_Tp, _Np>
55  * _TW: _SimdWrapper<_Tp, _Np>
56  * _TI: __intrinsic_type_t<_Tp, _Np>
57  * _TVT: _VectorTraits<_TV> or _VectorTraits<_TW>
58  * If one additional type is needed use _U instead of _T.
59  * Otherwise use _T\d, _TV\d, _TW\d, TI\d, _TVT\d.
60  *
61  * More naming conventions:
62  * _Ap or _Abi: An ABI tag from the simd_abi namespace
63  * _Ip: often used for integer types with sizeof(_Ip) == sizeof(_Tp),
64  * _IV, _IW as for _TV, _TW
65  * _Np: number of elements (not bytes)
66  * _Bytes: number of bytes
67  *
68  * Variable names:
69  * __k: mask object (vector- or bitmask)
70  */
71 _GLIBCXX_SIMD_BEGIN_NAMESPACE
72 
73 #if !_GLIBCXX_SIMD_X86INTRIN
74 using __m128 [[__gnu__::__vector_size__(16)]] = float;
75 using __m128d [[__gnu__::__vector_size__(16)]] = double;
76 using __m128i [[__gnu__::__vector_size__(16)]] = long long;
77 using __m256 [[__gnu__::__vector_size__(32)]] = float;
78 using __m256d [[__gnu__::__vector_size__(32)]] = double;
79 using __m256i [[__gnu__::__vector_size__(32)]] = long long;
80 using __m512 [[__gnu__::__vector_size__(64)]] = float;
81 using __m512d [[__gnu__::__vector_size__(64)]] = double;
82 using __m512i [[__gnu__::__vector_size__(64)]] = long long;
83 #endif
84 
85 namespace simd_abi {
86 // simd_abi forward declarations {{{
87 // implementation details:
88 struct _Scalar;
89 
90 template <int _Np>
91  struct _Fixed;
92 
93 // There are two major ABIs that appear on different architectures.
94 // Both have non-boolean values packed into an N Byte register
95 // -> #elements = N / sizeof(T)
96 // Masks differ:
97 // 1. Use value vector registers for masks (all 0 or all 1)
98 // 2. Use bitmasks (mask registers) with one bit per value in the corresponding
99 // value vector
100 //
101 // Both can be partially used, masking off the rest when doing horizontal
102 // operations or operations that can trap (e.g. FP_INVALID or integer division
103 // by 0). This is encoded as the number of used bytes.
104 template <int _UsedBytes>
105  struct _VecBuiltin;
106 
107 template <int _UsedBytes>
108  struct _VecBltnBtmsk;
109 
110 template <typename _Tp, int _Np>
111  using _VecN = _VecBuiltin<sizeof(_Tp) * _Np>;
112 
113 template <int _UsedBytes = 16>
114  using _Sse = _VecBuiltin<_UsedBytes>;
115 
116 template <int _UsedBytes = 32>
117  using _Avx = _VecBuiltin<_UsedBytes>;
118 
119 template <int _UsedBytes = 64>
120  using _Avx512 = _VecBltnBtmsk<_UsedBytes>;
121 
122 template <int _UsedBytes = 16>
123  using _Neon = _VecBuiltin<_UsedBytes>;
124 
125 // implementation-defined:
126 using __sse = _Sse<>;
127 using __avx = _Avx<>;
128 using __avx512 = _Avx512<>;
129 using __neon = _Neon<>;
130 using __neon128 = _Neon<16>;
131 using __neon64 = _Neon<8>;
132 
133 // standard:
134 template <typename _Tp, size_t _Np, typename...>
135  struct deduce;
136 
137 template <int _Np>
138  using fixed_size = _Fixed<_Np>;
139 
140 using scalar = _Scalar;
141 
142 // }}}
143 } // namespace simd_abi
144 // forward declarations is_simd(_mask), simd(_mask), simd_size {{{
145 template <typename _Tp>
146  struct is_simd;
147 
148 template <typename _Tp>
149  struct is_simd_mask;
150 
151 template <typename _Tp, typename _Abi>
152  class simd;
153 
154 template <typename _Tp, typename _Abi>
155  class simd_mask;
156 
157 template <typename _Tp, typename _Abi>
158  struct simd_size;
159 
160 // }}}
161 // load/store flags {{{
162 struct element_aligned_tag
163 {
164  template <typename _Tp, typename _Up = typename _Tp::value_type>
165  static constexpr size_t _S_alignment = alignof(_Up);
166 
167  template <typename _Tp, typename _Up>
168  _GLIBCXX_SIMD_INTRINSIC static constexpr _Up*
169  _S_apply(_Up* __ptr)
170  { return __ptr; }
171 };
172 
173 struct vector_aligned_tag
174 {
175  template <typename _Tp, typename _Up = typename _Tp::value_type>
176  static constexpr size_t _S_alignment
177  = std::__bit_ceil(sizeof(_Up) * _Tp::size());
178 
179  template <typename _Tp, typename _Up>
180  _GLIBCXX_SIMD_INTRINSIC static constexpr _Up*
181  _S_apply(_Up* __ptr)
182  {
183  return static_cast<_Up*>(
184  __builtin_assume_aligned(__ptr, _S_alignment<_Tp, _Up>));
185  }
186 };
187 
188 template <size_t _Np> struct overaligned_tag
189 {
190  template <typename _Tp, typename _Up = typename _Tp::value_type>
191  static constexpr size_t _S_alignment = _Np;
192 
193  template <typename _Tp, typename _Up>
194  _GLIBCXX_SIMD_INTRINSIC static constexpr _Up*
195  _S_apply(_Up* __ptr)
196  { return static_cast<_Up*>(__builtin_assume_aligned(__ptr, _Np)); }
197 };
198 
199 inline constexpr element_aligned_tag element_aligned = {};
200 
201 inline constexpr vector_aligned_tag vector_aligned = {};
202 
203 template <size_t _Np>
204  inline constexpr overaligned_tag<_Np> overaligned = {};
205 
206 // }}}
207 template <size_t _Xp>
208  using _SizeConstant = integral_constant<size_t, _Xp>;
209 
210 namespace __detail
211 {
212  struct _Minimum
213  {
214  template <typename _Tp>
215  _GLIBCXX_SIMD_INTRINSIC constexpr
216  _Tp
217  operator()(_Tp __a, _Tp __b) const
218  {
219  using std::min;
220  return min(__a, __b);
221  }
222  };
223 
224  struct _Maximum
225  {
226  template <typename _Tp>
227  _GLIBCXX_SIMD_INTRINSIC constexpr
228  _Tp
229  operator()(_Tp __a, _Tp __b) const
230  {
231  using std::max;
232  return max(__a, __b);
233  }
234  };
235 } // namespace __detail
236 
237 // unrolled/pack execution helpers
238 // __execute_n_times{{{
239 template <typename _Fp, size_t... _I>
240  [[__gnu__::__flatten__]] _GLIBCXX_SIMD_INTRINSIC constexpr
241  void
242  __execute_on_index_sequence(_Fp&& __f, index_sequence<_I...>)
243  { ((void)__f(_SizeConstant<_I>()), ...); }
244 
245 template <typename _Fp>
246  _GLIBCXX_SIMD_INTRINSIC constexpr void
247  __execute_on_index_sequence(_Fp&&, index_sequence<>)
248  { }
249 
250 template <size_t _Np, typename _Fp>
251  _GLIBCXX_SIMD_INTRINSIC constexpr void
252  __execute_n_times(_Fp&& __f)
253  {
254  __execute_on_index_sequence(static_cast<_Fp&&>(__f),
255  make_index_sequence<_Np>{});
256  }
257 
258 // }}}
259 // __generate_from_n_evaluations{{{
260 template <typename _R, typename _Fp, size_t... _I>
261  [[__gnu__::__flatten__]] _GLIBCXX_SIMD_INTRINSIC constexpr
262  _R
263  __execute_on_index_sequence_with_return(_Fp&& __f, index_sequence<_I...>)
264  { return _R{__f(_SizeConstant<_I>())...}; }
265 
266 template <size_t _Np, typename _R, typename _Fp>
267  _GLIBCXX_SIMD_INTRINSIC constexpr _R
268  __generate_from_n_evaluations(_Fp&& __f)
269  {
270  return __execute_on_index_sequence_with_return<_R>(
271  static_cast<_Fp&&>(__f), make_index_sequence<_Np>{});
272  }
273 
274 // }}}
275 // __call_with_n_evaluations{{{
276 template <size_t... _I, typename _F0, typename _FArgs>
277  [[__gnu__::__flatten__]] _GLIBCXX_SIMD_INTRINSIC constexpr
278  auto
279  __call_with_n_evaluations(index_sequence<_I...>, _F0&& __f0, _FArgs&& __fargs)
280  { return __f0(__fargs(_SizeConstant<_I>())...); }
281 
282 template <size_t _Np, typename _F0, typename _FArgs>
283  _GLIBCXX_SIMD_INTRINSIC constexpr auto
284  __call_with_n_evaluations(_F0&& __f0, _FArgs&& __fargs)
285  {
286  return __call_with_n_evaluations(make_index_sequence<_Np>{},
287  static_cast<_F0&&>(__f0),
288  static_cast<_FArgs&&>(__fargs));
289  }
290 
291 // }}}
292 // __call_with_subscripts{{{
293 template <size_t _First = 0, size_t... _It, typename _Tp, typename _Fp>
294  [[__gnu__::__flatten__]] _GLIBCXX_SIMD_INTRINSIC constexpr
295  auto
296  __call_with_subscripts(_Tp&& __x, index_sequence<_It...>, _Fp&& __fun)
297  { return __fun(__x[_First + _It]...); }
298 
299 template <size_t _Np, size_t _First = 0, typename _Tp, typename _Fp>
300  _GLIBCXX_SIMD_INTRINSIC constexpr auto
301  __call_with_subscripts(_Tp&& __x, _Fp&& __fun)
302  {
303  return __call_with_subscripts<_First>(static_cast<_Tp&&>(__x),
304  make_index_sequence<_Np>(),
305  static_cast<_Fp&&>(__fun));
306  }
307 
308 // }}}
309 
310 // vvv ---- type traits ---- vvv
311 // integer type aliases{{{
312 using _UChar = unsigned char;
313 using _SChar = signed char;
314 using _UShort = unsigned short;
315 using _UInt = unsigned int;
316 using _ULong = unsigned long;
317 using _ULLong = unsigned long long;
318 using _LLong = long long;
319 
320 //}}}
321 // __first_of_pack{{{
322 template <typename _T0, typename...>
323  struct __first_of_pack
324  { using type = _T0; };
325 
326 template <typename... _Ts>
327  using __first_of_pack_t = typename __first_of_pack<_Ts...>::type;
328 
329 //}}}
330 // __value_type_or_identity_t {{{
331 template <typename _Tp>
332  typename _Tp::value_type
333  __value_type_or_identity_impl(int);
334 
335 template <typename _Tp>
336  _Tp
337  __value_type_or_identity_impl(float);
338 
339 template <typename _Tp>
340  using __value_type_or_identity_t
341  = decltype(__value_type_or_identity_impl<_Tp>(int()));
342 
343 // }}}
344 // __is_vectorizable {{{
345 template <typename _Tp>
346  struct __is_vectorizable : public is_arithmetic<_Tp> {};
347 
348 template <>
349  struct __is_vectorizable<bool> : public false_type {};
350 
351 template <typename _Tp>
352  inline constexpr bool __is_vectorizable_v = __is_vectorizable<_Tp>::value;
353 
354 // Deduces to a vectorizable type
355 template <typename _Tp, typename = enable_if_t<__is_vectorizable_v<_Tp>>>
356  using _Vectorizable = _Tp;
357 
358 // }}}
359 // _LoadStorePtr / __is_possible_loadstore_conversion {{{
360 template <typename _Ptr, typename _ValueType>
361  struct __is_possible_loadstore_conversion
362  : conjunction<__is_vectorizable<_Ptr>, __is_vectorizable<_ValueType>> {};
363 
364 template <>
365  struct __is_possible_loadstore_conversion<bool, bool> : true_type {};
366 
367 // Deduces to a type allowed for load/store with the given value type.
368 template <typename _Ptr, typename _ValueType,
369  typename = enable_if_t<
370  __is_possible_loadstore_conversion<_Ptr, _ValueType>::value>>
371  using _LoadStorePtr = _Ptr;
372 
373 // }}}
374 // __is_bitmask{{{
375 template <typename _Tp, typename = void_t<>>
376  struct __is_bitmask : false_type {};
377 
378 template <typename _Tp>
379  inline constexpr bool __is_bitmask_v = __is_bitmask<_Tp>::value;
380 
381 // the __mmaskXX case:
382 template <typename _Tp>
383  struct __is_bitmask<_Tp,
384  void_t<decltype(declval<unsigned&>() = declval<_Tp>() & 1u)>>
385  : true_type {};
386 
387 // }}}
388 // __int_for_sizeof{{{
389 #pragma GCC diagnostic push
390 #pragma GCC diagnostic ignored "-Wpedantic"
391 template <size_t _Bytes>
392  constexpr auto
393  __int_for_sizeof()
394  {
395  if constexpr (_Bytes == sizeof(int))
396  return int();
397  #ifdef __clang__
398  else if constexpr (_Bytes == sizeof(char))
399  return char();
400  #else
401  else if constexpr (_Bytes == sizeof(_SChar))
402  return _SChar();
403  #endif
404  else if constexpr (_Bytes == sizeof(short))
405  return short();
406  #ifndef __clang__
407  else if constexpr (_Bytes == sizeof(long))
408  return long();
409  #endif
410  else if constexpr (_Bytes == sizeof(_LLong))
411  return _LLong();
412  #ifdef __SIZEOF_INT128__
413  else if constexpr (_Bytes == sizeof(__int128))
414  return __int128();
415  #endif // __SIZEOF_INT128__
416  else if constexpr (_Bytes % sizeof(int) == 0)
417  {
418  constexpr size_t _Np = _Bytes / sizeof(int);
419  struct _Ip
420  {
421  int _M_data[_Np];
422 
423  _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
424  operator&(_Ip __rhs) const
425  {
426  return __generate_from_n_evaluations<_Np, _Ip>(
427  [&](auto __i) { return __rhs._M_data[__i] & _M_data[__i]; });
428  }
429 
430  _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
431  operator|(_Ip __rhs) const
432  {
433  return __generate_from_n_evaluations<_Np, _Ip>(
434  [&](auto __i) { return __rhs._M_data[__i] | _M_data[__i]; });
435  }
436 
437  _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
438  operator^(_Ip __rhs) const
439  {
440  return __generate_from_n_evaluations<_Np, _Ip>(
441  [&](auto __i) { return __rhs._M_data[__i] ^ _M_data[__i]; });
442  }
443 
444  _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
445  operator~() const
446  {
447  return __generate_from_n_evaluations<_Np, _Ip>(
448  [&](auto __i) { return ~_M_data[__i]; });
449  }
450  };
451  return _Ip{};
452  }
453  else
454  static_assert(_Bytes != _Bytes, "this should be unreachable");
455  }
456 #pragma GCC diagnostic pop
457 
458 template <typename _Tp>
459  using __int_for_sizeof_t = decltype(__int_for_sizeof<sizeof(_Tp)>());
460 
461 template <size_t _Np>
462  using __int_with_sizeof_t = decltype(__int_for_sizeof<_Np>());
463 
464 // }}}
465 // __is_fixed_size_abi{{{
466 template <typename _Tp>
467  struct __is_fixed_size_abi : false_type {};
468 
469 template <int _Np>
470  struct __is_fixed_size_abi<simd_abi::fixed_size<_Np>> : true_type {};
471 
472 template <typename _Tp>
473  inline constexpr bool __is_fixed_size_abi_v = __is_fixed_size_abi<_Tp>::value;
474 
475 // }}}
476 // constexpr feature detection{{{
477 constexpr inline bool __have_mmx = _GLIBCXX_SIMD_HAVE_MMX;
478 constexpr inline bool __have_sse = _GLIBCXX_SIMD_HAVE_SSE;
479 constexpr inline bool __have_sse2 = _GLIBCXX_SIMD_HAVE_SSE2;
480 constexpr inline bool __have_sse3 = _GLIBCXX_SIMD_HAVE_SSE3;
481 constexpr inline bool __have_ssse3 = _GLIBCXX_SIMD_HAVE_SSSE3;
482 constexpr inline bool __have_sse4_1 = _GLIBCXX_SIMD_HAVE_SSE4_1;
483 constexpr inline bool __have_sse4_2 = _GLIBCXX_SIMD_HAVE_SSE4_2;
484 constexpr inline bool __have_xop = _GLIBCXX_SIMD_HAVE_XOP;
485 constexpr inline bool __have_avx = _GLIBCXX_SIMD_HAVE_AVX;
486 constexpr inline bool __have_avx2 = _GLIBCXX_SIMD_HAVE_AVX2;
487 constexpr inline bool __have_bmi = _GLIBCXX_SIMD_HAVE_BMI1;
488 constexpr inline bool __have_bmi2 = _GLIBCXX_SIMD_HAVE_BMI2;
489 constexpr inline bool __have_lzcnt = _GLIBCXX_SIMD_HAVE_LZCNT;
490 constexpr inline bool __have_sse4a = _GLIBCXX_SIMD_HAVE_SSE4A;
491 constexpr inline bool __have_fma = _GLIBCXX_SIMD_HAVE_FMA;
492 constexpr inline bool __have_fma4 = _GLIBCXX_SIMD_HAVE_FMA4;
493 constexpr inline bool __have_f16c = _GLIBCXX_SIMD_HAVE_F16C;
494 constexpr inline bool __have_popcnt = _GLIBCXX_SIMD_HAVE_POPCNT;
495 constexpr inline bool __have_avx512f = _GLIBCXX_SIMD_HAVE_AVX512F;
496 constexpr inline bool __have_avx512dq = _GLIBCXX_SIMD_HAVE_AVX512DQ;
497 constexpr inline bool __have_avx512vl = _GLIBCXX_SIMD_HAVE_AVX512VL;
498 constexpr inline bool __have_avx512bw = _GLIBCXX_SIMD_HAVE_AVX512BW;
499 constexpr inline bool __have_avx512dq_vl = __have_avx512dq && __have_avx512vl;
500 constexpr inline bool __have_avx512bw_vl = __have_avx512bw && __have_avx512vl;
501 
502 constexpr inline bool __have_neon = _GLIBCXX_SIMD_HAVE_NEON;
503 constexpr inline bool __have_neon_a32 = _GLIBCXX_SIMD_HAVE_NEON_A32;
504 constexpr inline bool __have_neon_a64 = _GLIBCXX_SIMD_HAVE_NEON_A64;
505 constexpr inline bool __support_neon_float =
506 #if defined __GCC_IEC_559
507  __GCC_IEC_559 == 0;
508 #elif defined __FAST_MATH__
509  true;
510 #else
511  false;
512 #endif
513 
514 #ifdef _ARCH_PWR10
515 constexpr inline bool __have_power10vec = true;
516 #else
517 constexpr inline bool __have_power10vec = false;
518 #endif
519 #ifdef __POWER9_VECTOR__
520 constexpr inline bool __have_power9vec = true;
521 #else
522 constexpr inline bool __have_power9vec = false;
523 #endif
524 #if defined __POWER8_VECTOR__
525 constexpr inline bool __have_power8vec = true;
526 #else
527 constexpr inline bool __have_power8vec = __have_power9vec;
528 #endif
529 #if defined __VSX__
530 constexpr inline bool __have_power_vsx = true;
531 #else
532 constexpr inline bool __have_power_vsx = __have_power8vec;
533 #endif
534 #if defined __ALTIVEC__
535 constexpr inline bool __have_power_vmx = true;
536 #else
537 constexpr inline bool __have_power_vmx = __have_power_vsx;
538 #endif
539 
540 // }}}
541 // __is_scalar_abi {{{
542 template <typename _Abi>
543  constexpr bool
544  __is_scalar_abi()
545  { return is_same_v<simd_abi::scalar, _Abi>; }
546 
547 // }}}
548 // __abi_bytes_v {{{
549 template <template <int> class _Abi, int _Bytes>
550  constexpr int
551  __abi_bytes_impl(_Abi<_Bytes>*)
552  { return _Bytes; }
553 
554 template <typename _Tp>
555  constexpr int
556  __abi_bytes_impl(_Tp*)
557  { return -1; }
558 
559 template <typename _Abi>
560  inline constexpr int __abi_bytes_v
561  = __abi_bytes_impl(static_cast<_Abi*>(nullptr));
562 
563 // }}}
564 // __is_builtin_bitmask_abi {{{
565 template <typename _Abi>
566  constexpr bool
567  __is_builtin_bitmask_abi()
568  { return is_same_v<simd_abi::_VecBltnBtmsk<__abi_bytes_v<_Abi>>, _Abi>; }
569 
570 // }}}
571 // __is_sse_abi {{{
572 template <typename _Abi>
573  constexpr bool
574  __is_sse_abi()
575  {
576  constexpr auto _Bytes = __abi_bytes_v<_Abi>;
577  return _Bytes <= 16 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
578  }
579 
580 // }}}
581 // __is_avx_abi {{{
582 template <typename _Abi>
583  constexpr bool
584  __is_avx_abi()
585  {
586  constexpr auto _Bytes = __abi_bytes_v<_Abi>;
587  return _Bytes > 16 && _Bytes <= 32
588  && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
589  }
590 
591 // }}}
592 // __is_avx512_abi {{{
593 template <typename _Abi>
594  constexpr bool
595  __is_avx512_abi()
596  {
597  constexpr auto _Bytes = __abi_bytes_v<_Abi>;
598  return _Bytes <= 64 && is_same_v<simd_abi::_Avx512<_Bytes>, _Abi>;
599  }
600 
601 // }}}
602 // __is_neon_abi {{{
603 template <typename _Abi>
604  constexpr bool
605  __is_neon_abi()
606  {
607  constexpr auto _Bytes = __abi_bytes_v<_Abi>;
608  return _Bytes <= 16 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
609  }
610 
611 // }}}
612 // __make_dependent_t {{{
613 template <typename, typename _Up>
614  struct __make_dependent
615  { using type = _Up; };
616 
617 template <typename _Tp, typename _Up>
618  using __make_dependent_t = typename __make_dependent<_Tp, _Up>::type;
619 
620 // }}}
621 // ^^^ ---- type traits ---- ^^^
622 
623 // __invoke_ub{{{
624 template <typename... _Args>
625  [[noreturn]] _GLIBCXX_SIMD_ALWAYS_INLINE void
626  __invoke_ub([[maybe_unused]] const char* __msg,
627  [[maybe_unused]] const _Args&... __args)
628  {
629 #ifdef _GLIBCXX_DEBUG_UB
630  __builtin_fprintf(stderr, __msg, __args...);
631  __builtin_trap();
632 #else
633  __builtin_unreachable();
634 #endif
635  }
636 
637 // }}}
638 // __assert_unreachable{{{
639 template <typename _Tp>
640  struct __assert_unreachable
641  { static_assert(!is_same_v<_Tp, _Tp>, "this should be unreachable"); };
642 
643 // }}}
644 // __size_or_zero_v {{{
645 template <typename _Tp, typename _Ap, size_t _Np = simd_size<_Tp, _Ap>::value>
646  constexpr size_t
647  __size_or_zero_dispatch(int)
648  { return _Np; }
649 
650 template <typename _Tp, typename _Ap>
651  constexpr size_t
652  __size_or_zero_dispatch(float)
653  { return 0; }
654 
655 template <typename _Tp, typename _Ap>
656  inline constexpr size_t __size_or_zero_v
657  = __size_or_zero_dispatch<_Tp, _Ap>(0);
658 
659 // }}}
660 // __div_roundup {{{
661 inline constexpr size_t
662 __div_roundup(size_t __a, size_t __b)
663 { return (__a + __b - 1) / __b; }
664 
665 // }}}
666 // _ExactBool{{{
667 class _ExactBool
668 {
669  const bool _M_data;
670 
671 public:
672  _GLIBCXX_SIMD_INTRINSIC constexpr _ExactBool(bool __b) : _M_data(__b) {}
673 
674  _ExactBool(int) = delete;
675 
676  _GLIBCXX_SIMD_INTRINSIC constexpr operator bool() const { return _M_data; }
677 };
678 
679 // }}}
680 // __may_alias{{{
681 /**@internal
682  * Helper __may_alias<_Tp> that turns _Tp into the type to be used for an
683  * aliasing pointer. This adds the __may_alias attribute to _Tp (with compilers
684  * that support it).
685  */
686 template <typename _Tp>
687  using __may_alias [[__gnu__::__may_alias__]] = _Tp;
688 
689 // }}}
690 // _UnsupportedBase {{{
691 // simd and simd_mask base for unsupported <_Tp, _Abi>
692 struct _UnsupportedBase
693 {
694  _UnsupportedBase() = delete;
695  _UnsupportedBase(const _UnsupportedBase&) = delete;
696  _UnsupportedBase& operator=(const _UnsupportedBase&) = delete;
697  ~_UnsupportedBase() = delete;
698 };
699 
700 // }}}
701 // _InvalidTraits {{{
702 /**
703  * @internal
704  * Defines the implementation of __a given <_Tp, _Abi>.
705  *
706  * Implementations must ensure that only valid <_Tp, _Abi> instantiations are
707  * possible. Static assertions in the type definition do not suffice. It is
708  * important that SFINAE works.
709  */
710 struct _InvalidTraits
711 {
712  using _IsValid = false_type;
713  using _SimdBase = _UnsupportedBase;
714  using _MaskBase = _UnsupportedBase;
715 
716  static constexpr size_t _S_full_size = 0;
717  static constexpr bool _S_is_partial = false;
718 
719  static constexpr size_t _S_simd_align = 1;
720  struct _SimdImpl;
721  struct _SimdMember {};
722  struct _SimdCastType;
723 
724  static constexpr size_t _S_mask_align = 1;
725  struct _MaskImpl;
726  struct _MaskMember {};
727  struct _MaskCastType;
728 };
729 
730 // }}}
731 // _SimdTraits {{{
732 template <typename _Tp, typename _Abi, typename = void_t<>>
733  struct _SimdTraits : _InvalidTraits {};
734 
735 // }}}
736 // __private_init, __bitset_init{{{
737 /**
738  * @internal
739  * Tag used for private init constructor of simd and simd_mask
740  */
741 inline constexpr struct _PrivateInit {} __private_init = {};
742 
743 inline constexpr struct _BitsetInit {} __bitset_init = {};
744 
745 // }}}
746 // __is_narrowing_conversion<_From, _To>{{{
747 template <typename _From, typename _To, bool = is_arithmetic_v<_From>,
748  bool = is_arithmetic_v<_To>>
749  struct __is_narrowing_conversion;
750 
751 // ignore "signed/unsigned mismatch" in the following trait.
752 // The implicit conversions will do the right thing here.
753 template <typename _From, typename _To>
754  struct __is_narrowing_conversion<_From, _To, true, true>
755  : public __bool_constant<(
756  __digits_v<_From> > __digits_v<_To>
757  || __finite_max_v<_From> > __finite_max_v<_To>
758  || __finite_min_v<_From> < __finite_min_v<_To>
759  || (is_signed_v<_From> && is_unsigned_v<_To>))> {};
760 
761 template <typename _Tp>
762  struct __is_narrowing_conversion<_Tp, bool, true, true>
763  : public true_type {};
764 
765 template <>
766  struct __is_narrowing_conversion<bool, bool, true, true>
767  : public false_type {};
768 
769 template <typename _Tp>
770  struct __is_narrowing_conversion<_Tp, _Tp, true, true>
771  : public false_type {};
772 
773 template <typename _From, typename _To>
774  struct __is_narrowing_conversion<_From, _To, false, true>
775  : public negation<is_convertible<_From, _To>> {};
776 
777 // }}}
778 // __converts_to_higher_integer_rank{{{
779 template <typename _From, typename _To, bool = (sizeof(_From) < sizeof(_To))>
780  struct __converts_to_higher_integer_rank : public true_type {};
781 
782 // this may fail for char -> short if sizeof(char) == sizeof(short)
783 template <typename _From, typename _To>
784  struct __converts_to_higher_integer_rank<_From, _To, false>
785  : public is_same<decltype(declval<_From>() + declval<_To>()), _To> {};
786 
787 // }}}
788 // __data(simd/simd_mask) {{{
789 template <typename _Tp, typename _Ap>
790  _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
791  __data(const simd<_Tp, _Ap>& __x);
792 
793 template <typename _Tp, typename _Ap>
794  _GLIBCXX_SIMD_INTRINSIC constexpr auto&
795  __data(simd<_Tp, _Ap>& __x);
796 
797 template <typename _Tp, typename _Ap>
798  _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
799  __data(const simd_mask<_Tp, _Ap>& __x);
800 
801 template <typename _Tp, typename _Ap>
802  _GLIBCXX_SIMD_INTRINSIC constexpr auto&
803  __data(simd_mask<_Tp, _Ap>& __x);
804 
805 // }}}
806 // _SimdConverter {{{
807 template <typename _FromT, typename _FromA, typename _ToT, typename _ToA,
808  typename = void>
809  struct _SimdConverter;
810 
811 template <typename _Tp, typename _Ap>
812  struct _SimdConverter<_Tp, _Ap, _Tp, _Ap, void>
813  {
814  template <typename _Up>
815  _GLIBCXX_SIMD_INTRINSIC const _Up&
816  operator()(const _Up& __x)
817  { return __x; }
818  };
819 
820 // }}}
821 // __to_value_type_or_member_type {{{
822 template <typename _V>
823  _GLIBCXX_SIMD_INTRINSIC constexpr auto
824  __to_value_type_or_member_type(const _V& __x) -> decltype(__data(__x))
825  { return __data(__x); }
826 
827 template <typename _V>
828  _GLIBCXX_SIMD_INTRINSIC constexpr const typename _V::value_type&
829  __to_value_type_or_member_type(const typename _V::value_type& __x)
830  { return __x; }
831 
832 // }}}
833 // __bool_storage_member_type{{{
834 template <size_t _Size>
835  struct __bool_storage_member_type;
836 
837 template <size_t _Size>
838  using __bool_storage_member_type_t =
839  typename __bool_storage_member_type<_Size>::type;
840 
841 // }}}
842 // _SimdTuple {{{
843 // why not tuple?
844 // 1. tuple gives no guarantee about the storage order, but I require
845 // storage
846 // equivalent to array<_Tp, _Np>
847 // 2. direct access to the element type (first template argument)
848 // 3. enforces equal element type, only different _Abi types are allowed
849 template <typename _Tp, typename... _Abis>
850  struct _SimdTuple;
851 
852 //}}}
853 // __fixed_size_storage_t {{{
854 template <typename _Tp, int _Np>
855  struct __fixed_size_storage;
856 
857 template <typename _Tp, int _Np>
858  using __fixed_size_storage_t = typename __fixed_size_storage<_Tp, _Np>::type;
859 
860 // }}}
861 // _SimdWrapper fwd decl{{{
862 template <typename _Tp, size_t _Size, typename = void_t<>>
863  struct _SimdWrapper;
864 
865 template <typename _Tp>
866  using _SimdWrapper8 = _SimdWrapper<_Tp, 8 / sizeof(_Tp)>;
867 template <typename _Tp>
868  using _SimdWrapper16 = _SimdWrapper<_Tp, 16 / sizeof(_Tp)>;
869 template <typename _Tp>
870  using _SimdWrapper32 = _SimdWrapper<_Tp, 32 / sizeof(_Tp)>;
871 template <typename _Tp>
872  using _SimdWrapper64 = _SimdWrapper<_Tp, 64 / sizeof(_Tp)>;
873 
874 // }}}
875 // __is_simd_wrapper {{{
876 template <typename _Tp>
877  struct __is_simd_wrapper : false_type {};
878 
879 template <typename _Tp, size_t _Np>
880  struct __is_simd_wrapper<_SimdWrapper<_Tp, _Np>> : true_type {};
881 
882 template <typename _Tp>
883  inline constexpr bool __is_simd_wrapper_v = __is_simd_wrapper<_Tp>::value;
884 
885 // }}}
886 // _BitOps {{{
887 struct _BitOps
888 {
889  // _S_bit_iteration {{{
890  template <typename _Tp, typename _Fp>
891  static void
892  _S_bit_iteration(_Tp __mask, _Fp&& __f)
893  {
894  static_assert(sizeof(_ULLong) >= sizeof(_Tp));
895  conditional_t<sizeof(_Tp) <= sizeof(_UInt), _UInt, _ULLong> __k;
896  if constexpr (is_convertible_v<_Tp, decltype(__k)>)
897  __k = __mask;
898  else
899  __k = __mask.to_ullong();
900  while(__k)
901  {
902  __f(std::__countr_zero(__k));
903  __k &= (__k - 1);
904  }
905  }
906 
907  //}}}
908 };
909 
910 //}}}
911 // __increment, __decrement {{{
912 template <typename _Tp = void>
913  struct __increment
914  { constexpr _Tp operator()(_Tp __a) const { return ++__a; } };
915 
916 template <>
917  struct __increment<void>
918  {
919  template <typename _Tp>
920  constexpr _Tp
921  operator()(_Tp __a) const
922  { return ++__a; }
923  };
924 
925 template <typename _Tp = void>
926  struct __decrement
927  { constexpr _Tp operator()(_Tp __a) const { return --__a; } };
928 
929 template <>
930  struct __decrement<void>
931  {
932  template <typename _Tp>
933  constexpr _Tp
934  operator()(_Tp __a) const
935  { return --__a; }
936  };
937 
938 // }}}
939 // _ValuePreserving(OrInt) {{{
940 template <typename _From, typename _To,
941  typename = enable_if_t<negation<
942  __is_narrowing_conversion<__remove_cvref_t<_From>, _To>>::value>>
943  using _ValuePreserving = _From;
944 
945 template <typename _From, typename _To,
946  typename _DecayedFrom = __remove_cvref_t<_From>,
947  typename = enable_if_t<conjunction<
948  is_convertible<_From, _To>,
949  disjunction<
950  is_same<_DecayedFrom, _To>, is_same<_DecayedFrom, int>,
951  conjunction<is_same<_DecayedFrom, _UInt>, is_unsigned<_To>>,
952  negation<__is_narrowing_conversion<_DecayedFrom, _To>>>>::value>>
953  using _ValuePreservingOrInt = _From;
954 
955 // }}}
956 // __intrinsic_type {{{
957 template <typename _Tp, size_t _Bytes, typename = void_t<>>
958  struct __intrinsic_type;
959 
960 template <typename _Tp, size_t _Size>
961  using __intrinsic_type_t =
962  typename __intrinsic_type<_Tp, _Size * sizeof(_Tp)>::type;
963 
964 template <typename _Tp>
965  using __intrinsic_type2_t = typename __intrinsic_type<_Tp, 2>::type;
966 template <typename _Tp>
967  using __intrinsic_type4_t = typename __intrinsic_type<_Tp, 4>::type;
968 template <typename _Tp>
969  using __intrinsic_type8_t = typename __intrinsic_type<_Tp, 8>::type;
970 template <typename _Tp>
971  using __intrinsic_type16_t = typename __intrinsic_type<_Tp, 16>::type;
972 template <typename _Tp>
973  using __intrinsic_type32_t = typename __intrinsic_type<_Tp, 32>::type;
974 template <typename _Tp>
975  using __intrinsic_type64_t = typename __intrinsic_type<_Tp, 64>::type;
976 
977 // }}}
978 // _BitMask {{{
979 template <size_t _Np, bool _Sanitized = false>
980  struct _BitMask;
981 
982 template <size_t _Np, bool _Sanitized>
983  struct __is_bitmask<_BitMask<_Np, _Sanitized>, void> : true_type {};
984 
985 template <size_t _Np>
986  using _SanitizedBitMask = _BitMask<_Np, true>;
987 
988 template <size_t _Np, bool _Sanitized>
989  struct _BitMask
990  {
991  static_assert(_Np > 0);
992 
993  static constexpr size_t _NBytes = __div_roundup(_Np, __CHAR_BIT__);
994 
995  using _Tp = conditional_t<_Np == 1, bool,
996  make_unsigned_t<__int_with_sizeof_t<std::min(
997  sizeof(_ULLong), std::__bit_ceil(_NBytes))>>>;
998 
999  static constexpr int _S_array_size = __div_roundup(_NBytes, sizeof(_Tp));
1000 
1001  _Tp _M_bits[_S_array_size];
1002 
1003  static constexpr int _S_unused_bits
1004  = _Np == 1 ? 0 : _S_array_size * sizeof(_Tp) * __CHAR_BIT__ - _Np;
1005 
1006  static constexpr _Tp _S_bitmask = +_Tp(~_Tp()) >> _S_unused_bits;
1007 
1008  constexpr _BitMask() noexcept = default;
1009 
1010  constexpr _BitMask(unsigned long long __x) noexcept
1011  : _M_bits{static_cast<_Tp>(__x)} {}
1012 
1013  _BitMask(bitset<_Np> __x) noexcept : _BitMask(__x.to_ullong()) {}
1014 
1015  constexpr _BitMask(const _BitMask&) noexcept = default;
1016 
1017  template <bool _RhsSanitized, typename = enable_if_t<_RhsSanitized == false
1018  && _Sanitized == true>>
1019  constexpr _BitMask(const _BitMask<_Np, _RhsSanitized>& __rhs) noexcept
1020  : _BitMask(__rhs._M_sanitized()) {}
1021 
1022  constexpr operator _SimdWrapper<bool, _Np>() const noexcept
1023  {
1024  static_assert(_S_array_size == 1);
1025  return _M_bits[0];
1026  }
1027 
1028  // precondition: is sanitized
1029  constexpr _Tp
1030  _M_to_bits() const noexcept
1031  {
1032  static_assert(_S_array_size == 1);
1033  return _M_bits[0];
1034  }
1035 
1036  // precondition: is sanitized
1037  constexpr unsigned long long
1038  to_ullong() const noexcept
1039  {
1040  static_assert(_S_array_size == 1);
1041  return _M_bits[0];
1042  }
1043 
1044  // precondition: is sanitized
1045  constexpr unsigned long
1046  to_ulong() const noexcept
1047  {
1048  static_assert(_S_array_size == 1);
1049  return _M_bits[0];
1050  }
1051 
1052  constexpr bitset<_Np>
1053  _M_to_bitset() const noexcept
1054  {
1055  static_assert(_S_array_size == 1);
1056  return _M_bits[0];
1057  }
1058 
1059  constexpr decltype(auto)
1060  _M_sanitized() const noexcept
1061  {
1062  if constexpr (_Sanitized)
1063  return *this;
1064  else if constexpr (_Np == 1)
1065  return _SanitizedBitMask<_Np>(_M_bits[0]);
1066  else
1067  {
1068  _SanitizedBitMask<_Np> __r = {};
1069  for (int __i = 0; __i < _S_array_size; ++__i)
1070  __r._M_bits[__i] = _M_bits[__i];
1071  if constexpr (_S_unused_bits > 0)
1072  __r._M_bits[_S_array_size - 1] &= _S_bitmask;
1073  return __r;
1074  }
1075  }
1076 
1077  template <size_t _Mp, bool _LSanitized>
1078  constexpr _BitMask<_Np + _Mp, _Sanitized>
1079  _M_prepend(_BitMask<_Mp, _LSanitized> __lsb) const noexcept
1080  {
1081  constexpr size_t _RN = _Np + _Mp;
1082  using _Rp = _BitMask<_RN, _Sanitized>;
1083  if constexpr (_Rp::_S_array_size == 1)
1084  {
1085  _Rp __r{{_M_bits[0]}};
1086  __r._M_bits[0] <<= _Mp;
1087  __r._M_bits[0] |= __lsb._M_sanitized()._M_bits[0];
1088  return __r;
1089  }
1090  else
1091  __assert_unreachable<_Rp>();
1092  }
1093 
1094  // Return a new _BitMask with size _NewSize while dropping _DropLsb least
1095  // significant bits. If the operation implicitly produces a sanitized bitmask,
1096  // the result type will have _Sanitized set.
1097  template <size_t _DropLsb, size_t _NewSize = _Np - _DropLsb>
1098  constexpr auto
1099  _M_extract() const noexcept
1100  {
1101  static_assert(_Np > _DropLsb);
1102  static_assert(_DropLsb + _NewSize <= sizeof(_ULLong) * __CHAR_BIT__,
1103  "not implemented for bitmasks larger than one ullong");
1104  if constexpr (_NewSize == 1)
1105  // must sanitize because the return _Tp is bool
1106  return _SanitizedBitMask<1>(_M_bits[0] & (_Tp(1) << _DropLsb));
1107  else
1108  return _BitMask<_NewSize,
1109  ((_NewSize + _DropLsb == sizeof(_Tp) * __CHAR_BIT__
1110  && _NewSize + _DropLsb <= _Np)
1111  || ((_Sanitized || _Np == sizeof(_Tp) * __CHAR_BIT__)
1112  && _NewSize + _DropLsb >= _Np))>(_M_bits[0]
1113  >> _DropLsb);
1114  }
1115 
1116  // True if all bits are set. Implicitly sanitizes if _Sanitized == false.
1117  constexpr bool
1118  all() const noexcept
1119  {
1120  if constexpr (_Np == 1)
1121  return _M_bits[0];
1122  else if constexpr (!_Sanitized)
1123  return _M_sanitized().all();
1124  else
1125  {
1126  constexpr _Tp __allbits = ~_Tp();
1127  for (int __i = 0; __i < _S_array_size - 1; ++__i)
1128  if (_M_bits[__i] != __allbits)
1129  return false;
1130  return _M_bits[_S_array_size - 1] == _S_bitmask;
1131  }
1132  }
1133 
1134  // True if at least one bit is set. Implicitly sanitizes if _Sanitized ==
1135  // false.
1136  constexpr bool
1137  any() const noexcept
1138  {
1139  if constexpr (_Np == 1)
1140  return _M_bits[0];
1141  else if constexpr (!_Sanitized)
1142  return _M_sanitized().any();
1143  else
1144  {
1145  for (int __i = 0; __i < _S_array_size - 1; ++__i)
1146  if (_M_bits[__i] != 0)
1147  return true;
1148  return _M_bits[_S_array_size - 1] != 0;
1149  }
1150  }
1151 
1152  // True if no bit is set. Implicitly sanitizes if _Sanitized == false.
1153  constexpr bool
1154  none() const noexcept
1155  {
1156  if constexpr (_Np == 1)
1157  return !_M_bits[0];
1158  else if constexpr (!_Sanitized)
1159  return _M_sanitized().none();
1160  else
1161  {
1162  for (int __i = 0; __i < _S_array_size - 1; ++__i)
1163  if (_M_bits[__i] != 0)
1164  return false;
1165  return _M_bits[_S_array_size - 1] == 0;
1166  }
1167  }
1168 
1169  // Returns the number of set bits. Implicitly sanitizes if _Sanitized ==
1170  // false.
1171  constexpr int
1172  count() const noexcept
1173  {
1174  if constexpr (_Np == 1)
1175  return _M_bits[0];
1176  else if constexpr (!_Sanitized)
1177  return _M_sanitized().none();
1178  else
1179  {
1180  int __result = __builtin_popcountll(_M_bits[0]);
1181  for (int __i = 1; __i < _S_array_size; ++__i)
1182  __result += __builtin_popcountll(_M_bits[__i]);
1183  return __result;
1184  }
1185  }
1186 
1187  // Returns the bit at offset __i as bool.
1188  constexpr bool
1189  operator[](size_t __i) const noexcept
1190  {
1191  if constexpr (_Np == 1)
1192  return _M_bits[0];
1193  else if constexpr (_S_array_size == 1)
1194  return (_M_bits[0] >> __i) & 1;
1195  else
1196  {
1197  const size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__);
1198  const size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__);
1199  return (_M_bits[__j] >> __shift) & 1;
1200  }
1201  }
1202 
1203  template <size_t __i>
1204  constexpr bool
1205  operator[](_SizeConstant<__i>) const noexcept
1206  {
1207  static_assert(__i < _Np);
1208  constexpr size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__);
1209  constexpr size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__);
1210  return static_cast<bool>(_M_bits[__j] & (_Tp(1) << __shift));
1211  }
1212 
1213  // Set the bit at offset __i to __x.
1214  constexpr void
1215  set(size_t __i, bool __x) noexcept
1216  {
1217  if constexpr (_Np == 1)
1218  _M_bits[0] = __x;
1219  else if constexpr (_S_array_size == 1)
1220  {
1221  _M_bits[0] &= ~_Tp(_Tp(1) << __i);
1222  _M_bits[0] |= _Tp(_Tp(__x) << __i);
1223  }
1224  else
1225  {
1226  const size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__);
1227  const size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__);
1228  _M_bits[__j] &= ~_Tp(_Tp(1) << __shift);
1229  _M_bits[__j] |= _Tp(_Tp(__x) << __shift);
1230  }
1231  }
1232 
1233  template <size_t __i>
1234  constexpr void
1235  set(_SizeConstant<__i>, bool __x) noexcept
1236  {
1237  static_assert(__i < _Np);
1238  if constexpr (_Np == 1)
1239  _M_bits[0] = __x;
1240  else
1241  {
1242  constexpr size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__);
1243  constexpr size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__);
1244  constexpr _Tp __mask = ~_Tp(_Tp(1) << __shift);
1245  _M_bits[__j] &= __mask;
1246  _M_bits[__j] |= _Tp(_Tp(__x) << __shift);
1247  }
1248  }
1249 
1250  // Inverts all bits. Sanitized input leads to sanitized output.
1251  constexpr _BitMask
1252  operator~() const noexcept
1253  {
1254  if constexpr (_Np == 1)
1255  return !_M_bits[0];
1256  else
1257  {
1258  _BitMask __result{};
1259  for (int __i = 0; __i < _S_array_size - 1; ++__i)
1260  __result._M_bits[__i] = ~_M_bits[__i];
1261  if constexpr (_Sanitized)
1262  __result._M_bits[_S_array_size - 1]
1263  = _M_bits[_S_array_size - 1] ^ _S_bitmask;
1264  else
1265  __result._M_bits[_S_array_size - 1] = ~_M_bits[_S_array_size - 1];
1266  return __result;
1267  }
1268  }
1269 
1270  constexpr _BitMask&
1271  operator^=(const _BitMask& __b) & noexcept
1272  {
1273  __execute_n_times<_S_array_size>(
1274  [&](auto __i) { _M_bits[__i] ^= __b._M_bits[__i]; });
1275  return *this;
1276  }
1277 
1278  constexpr _BitMask&
1279  operator|=(const _BitMask& __b) & noexcept
1280  {
1281  __execute_n_times<_S_array_size>(
1282  [&](auto __i) { _M_bits[__i] |= __b._M_bits[__i]; });
1283  return *this;
1284  }
1285 
1286  constexpr _BitMask&
1287  operator&=(const _BitMask& __b) & noexcept
1288  {
1289  __execute_n_times<_S_array_size>(
1290  [&](auto __i) { _M_bits[__i] &= __b._M_bits[__i]; });
1291  return *this;
1292  }
1293 
1294  friend constexpr _BitMask
1295  operator^(const _BitMask& __a, const _BitMask& __b) noexcept
1296  {
1297  _BitMask __r = __a;
1298  __r ^= __b;
1299  return __r;
1300  }
1301 
1302  friend constexpr _BitMask
1303  operator|(const _BitMask& __a, const _BitMask& __b) noexcept
1304  {
1305  _BitMask __r = __a;
1306  __r |= __b;
1307  return __r;
1308  }
1309 
1310  friend constexpr _BitMask
1311  operator&(const _BitMask& __a, const _BitMask& __b) noexcept
1312  {
1313  _BitMask __r = __a;
1314  __r &= __b;
1315  return __r;
1316  }
1317 
1318  _GLIBCXX_SIMD_INTRINSIC
1319  constexpr bool
1320  _M_is_constprop() const
1321  {
1322  if constexpr (_S_array_size == 0)
1323  return __builtin_constant_p(_M_bits[0]);
1324  else
1325  {
1326  for (int __i = 0; __i < _S_array_size; ++__i)
1327  if (!__builtin_constant_p(_M_bits[__i]))
1328  return false;
1329  return true;
1330  }
1331  }
1332  };
1333 
1334 // }}}
1335 
1336 // vvv ---- builtin vector types [[gnu::vector_size(N)]] and operations ---- vvv
1337 // __min_vector_size {{{
1338 template <typename _Tp = void>
1339  static inline constexpr int __min_vector_size = 2 * sizeof(_Tp);
1340 
1341 #if _GLIBCXX_SIMD_HAVE_NEON
1342 template <>
1343  inline constexpr int __min_vector_size<void> = 8;
1344 #else
1345 template <>
1346  inline constexpr int __min_vector_size<void> = 16;
1347 #endif
1348 
1349 // }}}
1350 // __vector_type {{{
1351 template <typename _Tp, size_t _Np, typename = void>
1352  struct __vector_type_n {};
1353 
1354 // substition failure for 0-element case
1355 template <typename _Tp>
1356  struct __vector_type_n<_Tp, 0, void> {};
1357 
1358 // special case 1-element to be _Tp itself
1359 template <typename _Tp>
1360  struct __vector_type_n<_Tp, 1, enable_if_t<__is_vectorizable_v<_Tp>>>
1361  { using type = _Tp; };
1362 
1363 // else, use GNU-style builtin vector types
1364 template <typename _Tp, size_t _Np>
1365  struct __vector_type_n<_Tp, _Np,
1366  enable_if_t<__is_vectorizable_v<_Tp> && _Np >= 2>>
1367  {
1368  static constexpr size_t _S_Np2 = std::__bit_ceil(_Np * sizeof(_Tp));
1369 
1370  static constexpr size_t _S_Bytes =
1371 #ifdef __i386__
1372  // Using [[gnu::vector_size(8)]] would wreak havoc on the FPU because
1373  // those objects are passed via MMX registers and nothing ever calls EMMS.
1374  _S_Np2 == 8 ? 16 :
1375 #endif
1376  _S_Np2 < __min_vector_size<_Tp> ? __min_vector_size<_Tp>
1377  : _S_Np2;
1378 
1379  using type [[__gnu__::__vector_size__(_S_Bytes)]] = _Tp;
1380  };
1381 
1382 template <typename _Tp, size_t _Bytes, size_t = _Bytes % sizeof(_Tp)>
1383  struct __vector_type;
1384 
1385 template <typename _Tp, size_t _Bytes>
1386  struct __vector_type<_Tp, _Bytes, 0>
1387  : __vector_type_n<_Tp, _Bytes / sizeof(_Tp)> {};
1388 
1389 template <typename _Tp, size_t _Size>
1390  using __vector_type_t = typename __vector_type_n<_Tp, _Size>::type;
1391 
1392 template <typename _Tp>
1393  using __vector_type2_t = typename __vector_type<_Tp, 2>::type;
1394 template <typename _Tp>
1395  using __vector_type4_t = typename __vector_type<_Tp, 4>::type;
1396 template <typename _Tp>
1397  using __vector_type8_t = typename __vector_type<_Tp, 8>::type;
1398 template <typename _Tp>
1399  using __vector_type16_t = typename __vector_type<_Tp, 16>::type;
1400 template <typename _Tp>
1401  using __vector_type32_t = typename __vector_type<_Tp, 32>::type;
1402 template <typename _Tp>
1403  using __vector_type64_t = typename __vector_type<_Tp, 64>::type;
1404 
1405 // }}}
1406 // __is_vector_type {{{
1407 template <typename _Tp, typename = void_t<>>
1408  struct __is_vector_type : false_type {};
1409 
1410 template <typename _Tp>
1411  struct __is_vector_type<
1412  _Tp, void_t<typename __vector_type<
1413  remove_reference_t<decltype(declval<_Tp>()[0])>, sizeof(_Tp)>::type>>
1414  : is_same<_Tp, typename __vector_type<
1415  remove_reference_t<decltype(declval<_Tp>()[0])>,
1416  sizeof(_Tp)>::type> {};
1417 
1418 template <typename _Tp>
1419  inline constexpr bool __is_vector_type_v = __is_vector_type<_Tp>::value;
1420 
1421 // }}}
1422 // __is_intrinsic_type {{{
1423 #if _GLIBCXX_SIMD_HAVE_SSE_ABI
1424 template <typename _Tp>
1425  using __is_intrinsic_type = __is_vector_type<_Tp>;
1426 #else // not SSE (x86)
1427 template <typename _Tp, typename = void_t<>>
1428  struct __is_intrinsic_type : false_type {};
1429 
1430 template <typename _Tp>
1431  struct __is_intrinsic_type<
1432  _Tp, void_t<typename __intrinsic_type<
1433  remove_reference_t<decltype(declval<_Tp>()[0])>, sizeof(_Tp)>::type>>
1434  : is_same<_Tp, typename __intrinsic_type<
1435  remove_reference_t<decltype(declval<_Tp>()[0])>,
1436  sizeof(_Tp)>::type> {};
1437 #endif
1438 
1439 template <typename _Tp>
1440  inline constexpr bool __is_intrinsic_type_v = __is_intrinsic_type<_Tp>::value;
1441 
1442 // }}}
1443 // _VectorTraits{{{
1444 template <typename _Tp, typename = void_t<>>
1445  struct _VectorTraitsImpl;
1446 
1447 template <typename _Tp>
1448  struct _VectorTraitsImpl<_Tp, enable_if_t<__is_vector_type_v<_Tp>
1449  || __is_intrinsic_type_v<_Tp>>>
1450  {
1451  using type = _Tp;
1452  using value_type = remove_reference_t<decltype(declval<_Tp>()[0])>;
1453  static constexpr int _S_full_size = sizeof(_Tp) / sizeof(value_type);
1454  using _Wrapper = _SimdWrapper<value_type, _S_full_size>;
1455  template <typename _Up, int _W = _S_full_size>
1456  static constexpr bool _S_is
1457  = is_same_v<value_type, _Up> && _W == _S_full_size;
1458  };
1459 
1460 template <typename _Tp, size_t _Np>
1461  struct _VectorTraitsImpl<_SimdWrapper<_Tp, _Np>,
1462  void_t<__vector_type_t<_Tp, _Np>>>
1463  {
1464  using type = __vector_type_t<_Tp, _Np>;
1465  using value_type = _Tp;
1466  static constexpr int _S_full_size = sizeof(type) / sizeof(value_type);
1467  using _Wrapper = _SimdWrapper<_Tp, _Np>;
1468  static constexpr bool _S_is_partial = (_Np == _S_full_size);
1469  static constexpr int _S_partial_width = _Np;
1470  template <typename _Up, int _W = _S_full_size>
1471  static constexpr bool _S_is
1472  = is_same_v<value_type, _Up>&& _W == _S_full_size;
1473  };
1474 
1475 template <typename _Tp, typename = typename _VectorTraitsImpl<_Tp>::type>
1476  using _VectorTraits = _VectorTraitsImpl<_Tp>;
1477 
1478 // }}}
1479 // __as_vector{{{
1480 template <typename _V>
1481  _GLIBCXX_SIMD_INTRINSIC constexpr auto
1482  __as_vector(_V __x)
1483  {
1484  if constexpr (__is_vector_type_v<_V>)
1485  return __x;
1486  else if constexpr (is_simd<_V>::value || is_simd_mask<_V>::value)
1487  return __data(__x)._M_data;
1488  else if constexpr (__is_vectorizable_v<_V>)
1489  return __vector_type_t<_V, 2>{__x};
1490  else
1491  return __x._M_data;
1492  }
1493 
1494 // }}}
1495 // __as_wrapper{{{
1496 template <size_t _Np = 0, typename _V>
1497  _GLIBCXX_SIMD_INTRINSIC constexpr auto
1498  __as_wrapper(_V __x)
1499  {
1500  if constexpr (__is_vector_type_v<_V>)
1501  return _SimdWrapper<typename _VectorTraits<_V>::value_type,
1502  (_Np > 0 ? _Np : _VectorTraits<_V>::_S_full_size)>(__x);
1503  else if constexpr (is_simd<_V>::value || is_simd_mask<_V>::value)
1504  {
1505  static_assert(_V::size() == _Np);
1506  return __data(__x);
1507  }
1508  else
1509  {
1510  static_assert(_V::_S_size == _Np);
1511  return __x;
1512  }
1513  }
1514 
1515 // }}}
1516 // __intrin_bitcast{{{
1517 template <typename _To, typename _From>
1518  _GLIBCXX_SIMD_INTRINSIC constexpr _To
1519  __intrin_bitcast(_From __v)
1520  {
1521  static_assert((__is_vector_type_v<_From> || __is_intrinsic_type_v<_From>)
1522  && (__is_vector_type_v<_To> || __is_intrinsic_type_v<_To>));
1523  if constexpr (sizeof(_To) == sizeof(_From))
1524  return reinterpret_cast<_To>(__v);
1525  else if constexpr (sizeof(_From) > sizeof(_To))
1526  if constexpr (sizeof(_To) >= 16)
1527  return reinterpret_cast<const __may_alias<_To>&>(__v);
1528  else
1529  {
1530  _To __r;
1531  __builtin_memcpy(&__r, &__v, sizeof(_To));
1532  return __r;
1533  }
1534 #if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1535  else if constexpr (__have_avx && sizeof(_From) == 16 && sizeof(_To) == 32)
1536  return reinterpret_cast<_To>(__builtin_ia32_ps256_ps(
1537  reinterpret_cast<__vector_type_t<float, 4>>(__v)));
1538  else if constexpr (__have_avx512f && sizeof(_From) == 16
1539  && sizeof(_To) == 64)
1540  return reinterpret_cast<_To>(__builtin_ia32_ps512_ps(
1541  reinterpret_cast<__vector_type_t<float, 4>>(__v)));
1542  else if constexpr (__have_avx512f && sizeof(_From) == 32
1543  && sizeof(_To) == 64)
1544  return reinterpret_cast<_To>(__builtin_ia32_ps512_256ps(
1545  reinterpret_cast<__vector_type_t<float, 8>>(__v)));
1546 #endif // _GLIBCXX_SIMD_X86INTRIN
1547  else if constexpr (sizeof(__v) <= 8)
1548  return reinterpret_cast<_To>(
1549  __vector_type_t<__int_for_sizeof_t<_From>, sizeof(_To) / sizeof(_From)>{
1550  reinterpret_cast<__int_for_sizeof_t<_From>>(__v)});
1551  else
1552  {
1553  static_assert(sizeof(_To) > sizeof(_From));
1554  _To __r = {};
1555  __builtin_memcpy(&__r, &__v, sizeof(_From));
1556  return __r;
1557  }
1558  }
1559 
1560 // }}}
1561 // __vector_bitcast{{{
1562 template <typename _To, size_t _NN = 0, typename _From,
1563  typename _FromVT = _VectorTraits<_From>,
1564  size_t _Np = _NN == 0 ? sizeof(_From) / sizeof(_To) : _NN>
1565  _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_To, _Np>
1566  __vector_bitcast(_From __x)
1567  {
1568  using _R = __vector_type_t<_To, _Np>;
1569  return __intrin_bitcast<_R>(__x);
1570  }
1571 
1572 template <typename _To, size_t _NN = 0, typename _Tp, size_t _Nx,
1573  size_t _Np
1574  = _NN == 0 ? sizeof(_SimdWrapper<_Tp, _Nx>) / sizeof(_To) : _NN>
1575  _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_To, _Np>
1576  __vector_bitcast(const _SimdWrapper<_Tp, _Nx>& __x)
1577  {
1578  static_assert(_Np > 1);
1579  return __intrin_bitcast<__vector_type_t<_To, _Np>>(__x._M_data);
1580  }
1581 
1582 // }}}
1583 // __convert_x86 declarations {{{
1584 #ifdef _GLIBCXX_SIMD_WORKAROUND_PR85048
1585 template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1586  _To __convert_x86(_Tp);
1587 
1588 template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1589  _To __convert_x86(_Tp, _Tp);
1590 
1591 template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1592  _To __convert_x86(_Tp, _Tp, _Tp, _Tp);
1593 
1594 template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1595  _To __convert_x86(_Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp);
1596 
1597 template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1598  _To __convert_x86(_Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp,
1599  _Tp, _Tp, _Tp, _Tp);
1600 #endif // _GLIBCXX_SIMD_WORKAROUND_PR85048
1601 
1602 //}}}
1603 // __bit_cast {{{
1604 template <typename _To, typename _From>
1605  _GLIBCXX_SIMD_INTRINSIC constexpr _To
1606  __bit_cast(const _From __x)
1607  {
1608 #if __has_builtin(__builtin_bit_cast)
1609  return __builtin_bit_cast(_To, __x);
1610 #else
1611  static_assert(sizeof(_To) == sizeof(_From));
1612  constexpr bool __to_is_vectorizable
1613  = is_arithmetic_v<_To> || is_enum_v<_To>;
1614  constexpr bool __from_is_vectorizable
1615  = is_arithmetic_v<_From> || is_enum_v<_From>;
1616  if constexpr (__is_vector_type_v<_To> && __is_vector_type_v<_From>)
1617  return reinterpret_cast<_To>(__x);
1618  else if constexpr (__is_vector_type_v<_To> && __from_is_vectorizable)
1619  {
1620  using _FV [[gnu::vector_size(sizeof(_From))]] = _From;
1621  return reinterpret_cast<_To>(_FV{__x});
1622  }
1623  else if constexpr (__to_is_vectorizable && __from_is_vectorizable)
1624  {
1625  using _TV [[gnu::vector_size(sizeof(_To))]] = _To;
1626  using _FV [[gnu::vector_size(sizeof(_From))]] = _From;
1627  return reinterpret_cast<_TV>(_FV{__x})[0];
1628  }
1629  else if constexpr (__to_is_vectorizable && __is_vector_type_v<_From>)
1630  {
1631  using _TV [[gnu::vector_size(sizeof(_To))]] = _To;
1632  return reinterpret_cast<_TV>(__x)[0];
1633  }
1634  else
1635  {
1636  _To __r;
1637  __builtin_memcpy(reinterpret_cast<char*>(&__r),
1638  reinterpret_cast<const char*>(&__x), sizeof(_To));
1639  return __r;
1640  }
1641 #endif
1642  }
1643 
1644 // }}}
1645 // __to_intrin {{{
1646 template <typename _Tp, typename _TVT = _VectorTraits<_Tp>,
1647  typename _R
1648  = __intrinsic_type_t<typename _TVT::value_type, _TVT::_S_full_size>>
1649  _GLIBCXX_SIMD_INTRINSIC constexpr _R
1650  __to_intrin(_Tp __x)
1651  {
1652  static_assert(sizeof(__x) <= sizeof(_R),
1653  "__to_intrin may never drop values off the end");
1654  if constexpr (sizeof(__x) == sizeof(_R))
1655  return reinterpret_cast<_R>(__as_vector(__x));
1656  else
1657  {
1658  using _Up = __int_for_sizeof_t<_Tp>;
1659  return reinterpret_cast<_R>(
1660  __vector_type_t<_Up, sizeof(_R) / sizeof(_Up)>{__bit_cast<_Up>(__x)});
1661  }
1662  }
1663 
1664 // }}}
1665 // __make_vector{{{
1666 template <typename _Tp, typename... _Args>
1667  _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, sizeof...(_Args)>
1668  __make_vector(const _Args&... __args)
1669  {
1670  return __vector_type_t<_Tp, sizeof...(_Args)>{static_cast<_Tp>(__args)...};
1671  }
1672 
1673 // }}}
1674 // __vector_broadcast{{{
1675 template <size_t _Np, typename _Tp>
1676  _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1677  __vector_broadcast(_Tp __x)
1678  {
1679  return __call_with_n_evaluations<_Np>(
1680  [](auto... __xx) { return __vector_type_t<_Tp, _Np>{__xx...}; },
1681  [&__x](int) { return __x; });
1682  }
1683 
1684 // }}}
1685 // __generate_vector{{{
1686  template <typename _Tp, size_t _Np, typename _Gp, size_t... _I>
1687  _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1688  __generate_vector_impl(_Gp&& __gen, index_sequence<_I...>)
1689  {
1690  return __vector_type_t<_Tp, _Np>{
1691  static_cast<_Tp>(__gen(_SizeConstant<_I>()))...};
1692  }
1693 
1694 template <typename _V, typename _VVT = _VectorTraits<_V>, typename _Gp>
1695  _GLIBCXX_SIMD_INTRINSIC constexpr _V
1696  __generate_vector(_Gp&& __gen)
1697  {
1698  if constexpr (__is_vector_type_v<_V>)
1699  return __generate_vector_impl<typename _VVT::value_type,
1700  _VVT::_S_full_size>(
1701  static_cast<_Gp&&>(__gen), make_index_sequence<_VVT::_S_full_size>());
1702  else
1703  return __generate_vector_impl<typename _VVT::value_type,
1704  _VVT::_S_partial_width>(
1705  static_cast<_Gp&&>(__gen),
1706  make_index_sequence<_VVT::_S_partial_width>());
1707  }
1708 
1709 template <typename _Tp, size_t _Np, typename _Gp>
1710  _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1711  __generate_vector(_Gp&& __gen)
1712  {
1713  return __generate_vector_impl<_Tp, _Np>(static_cast<_Gp&&>(__gen),
1714  make_index_sequence<_Np>());
1715  }
1716 
1717 // }}}
1718 // __xor{{{
1719 template <typename _TW>
1720  _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1721  __xor(_TW __a, _TW __b) noexcept
1722  {
1723  if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1724  {
1725  using _Tp = typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1726  _VectorTraitsImpl<_TW>>::value_type;
1727  if constexpr (is_floating_point_v<_Tp>)
1728  {
1729  using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1730  return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1731  ^ __vector_bitcast<_Ip>(__b));
1732  }
1733  else if constexpr (__is_vector_type_v<_TW>)
1734  return __a ^ __b;
1735  else
1736  return __a._M_data ^ __b._M_data;
1737  }
1738  else
1739  return __a ^ __b;
1740  }
1741 
1742 // }}}
1743 // __or{{{
1744 template <typename _TW>
1745  _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1746  __or(_TW __a, _TW __b) noexcept
1747  {
1748  if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1749  {
1750  using _Tp = typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1751  _VectorTraitsImpl<_TW>>::value_type;
1752  if constexpr (is_floating_point_v<_Tp>)
1753  {
1754  using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1755  return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1756  | __vector_bitcast<_Ip>(__b));
1757  }
1758  else if constexpr (__is_vector_type_v<_TW>)
1759  return __a | __b;
1760  else
1761  return __a._M_data | __b._M_data;
1762  }
1763  else
1764  return __a | __b;
1765  }
1766 
1767 // }}}
1768 // __and{{{
1769 template <typename _TW>
1770  _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1771  __and(_TW __a, _TW __b) noexcept
1772  {
1773  if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1774  {
1775  using _Tp = typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1776  _VectorTraitsImpl<_TW>>::value_type;
1777  if constexpr (is_floating_point_v<_Tp>)
1778  {
1779  using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1780  return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1781  & __vector_bitcast<_Ip>(__b));
1782  }
1783  else if constexpr (__is_vector_type_v<_TW>)
1784  return __a & __b;
1785  else
1786  return __a._M_data & __b._M_data;
1787  }
1788  else
1789  return __a & __b;
1790  }
1791 
1792 // }}}
1793 // __andnot{{{
1794 #if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1795 static constexpr struct
1796 {
1797  _GLIBCXX_SIMD_INTRINSIC __v4sf
1798  operator()(__v4sf __a, __v4sf __b) const noexcept
1799  { return __builtin_ia32_andnps(__a, __b); }
1800 
1801  _GLIBCXX_SIMD_INTRINSIC __v2df
1802  operator()(__v2df __a, __v2df __b) const noexcept
1803  { return __builtin_ia32_andnpd(__a, __b); }
1804 
1805  _GLIBCXX_SIMD_INTRINSIC __v2di
1806  operator()(__v2di __a, __v2di __b) const noexcept
1807  { return __builtin_ia32_pandn128(__a, __b); }
1808 
1809  _GLIBCXX_SIMD_INTRINSIC __v8sf
1810  operator()(__v8sf __a, __v8sf __b) const noexcept
1811  { return __builtin_ia32_andnps256(__a, __b); }
1812 
1813  _GLIBCXX_SIMD_INTRINSIC __v4df
1814  operator()(__v4df __a, __v4df __b) const noexcept
1815  { return __builtin_ia32_andnpd256(__a, __b); }
1816 
1817  _GLIBCXX_SIMD_INTRINSIC __v4di
1818  operator()(__v4di __a, __v4di __b) const noexcept
1819  {
1820  if constexpr (__have_avx2)
1821  return __builtin_ia32_andnotsi256(__a, __b);
1822  else
1823  return reinterpret_cast<__v4di>(
1824  __builtin_ia32_andnpd256(reinterpret_cast<__v4df>(__a),
1825  reinterpret_cast<__v4df>(__b)));
1826  }
1827 
1828  _GLIBCXX_SIMD_INTRINSIC __v16sf
1829  operator()(__v16sf __a, __v16sf __b) const noexcept
1830  {
1831  if constexpr (__have_avx512dq)
1832  return _mm512_andnot_ps(__a, __b);
1833  else
1834  return reinterpret_cast<__v16sf>(
1835  _mm512_andnot_si512(reinterpret_cast<__v8di>(__a),
1836  reinterpret_cast<__v8di>(__b)));
1837  }
1838 
1839  _GLIBCXX_SIMD_INTRINSIC __v8df
1840  operator()(__v8df __a, __v8df __b) const noexcept
1841  {
1842  if constexpr (__have_avx512dq)
1843  return _mm512_andnot_pd(__a, __b);
1844  else
1845  return reinterpret_cast<__v8df>(
1846  _mm512_andnot_si512(reinterpret_cast<__v8di>(__a),
1847  reinterpret_cast<__v8di>(__b)));
1848  }
1849 
1850  _GLIBCXX_SIMD_INTRINSIC __v8di
1851  operator()(__v8di __a, __v8di __b) const noexcept
1852  { return _mm512_andnot_si512(__a, __b); }
1853 } _S_x86_andnot;
1854 #endif // _GLIBCXX_SIMD_X86INTRIN && !__clang__
1855 
1856 template <typename _TW>
1857  _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1858  __andnot(_TW __a, _TW __b) noexcept
1859  {
1860  if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1861  {
1862  using _TVT = conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1863  _VectorTraitsImpl<_TW>>;
1864  using _Tp = typename _TVT::value_type;
1865 #if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1866  if constexpr (sizeof(_TW) >= 16)
1867  {
1868  const auto __ai = __to_intrin(__a);
1869  const auto __bi = __to_intrin(__b);
1870  if (!__builtin_is_constant_evaluated()
1871  && !(__builtin_constant_p(__ai) && __builtin_constant_p(__bi)))
1872  {
1873  const auto __r = _S_x86_andnot(__ai, __bi);
1874  if constexpr (is_convertible_v<decltype(__r), _TW>)
1875  return __r;
1876  else
1877  return reinterpret_cast<typename _TVT::type>(__r);
1878  }
1879  }
1880 #endif // _GLIBCXX_SIMD_X86INTRIN
1881  using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1882  return __vector_bitcast<_Tp>(~__vector_bitcast<_Ip>(__a)
1883  & __vector_bitcast<_Ip>(__b));
1884  }
1885  else
1886  return ~__a & __b;
1887  }
1888 
1889 // }}}
1890 // __not{{{
1891 template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1892  _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
1893  __not(_Tp __a) noexcept
1894  {
1895  if constexpr (is_floating_point_v<typename _TVT::value_type>)
1896  return reinterpret_cast<typename _TVT::type>(
1897  ~__vector_bitcast<unsigned>(__a));
1898  else
1899  return ~__a;
1900  }
1901 
1902 // }}}
1903 // __concat{{{
1904 template <typename _Tp, typename _TVT = _VectorTraits<_Tp>,
1905  typename _R = __vector_type_t<typename _TVT::value_type,
1906  _TVT::_S_full_size * 2>>
1907  constexpr _R
1908  __concat(_Tp a_, _Tp b_)
1909  {
1910 #ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_1
1911  using _W
1912  = conditional_t<is_floating_point_v<typename _TVT::value_type>, double,
1913  conditional_t<(sizeof(_Tp) >= 2 * sizeof(long long)),
1914  long long, typename _TVT::value_type>>;
1915  constexpr int input_width = sizeof(_Tp) / sizeof(_W);
1916  const auto __a = __vector_bitcast<_W>(a_);
1917  const auto __b = __vector_bitcast<_W>(b_);
1918  using _Up = __vector_type_t<_W, sizeof(_R) / sizeof(_W)>;
1919 #else
1920  constexpr int input_width = _TVT::_S_full_size;
1921  const _Tp& __a = a_;
1922  const _Tp& __b = b_;
1923  using _Up = _R;
1924 #endif
1925  if constexpr (input_width == 2)
1926  return reinterpret_cast<_R>(_Up{__a[0], __a[1], __b[0], __b[1]});
1927  else if constexpr (input_width == 4)
1928  return reinterpret_cast<_R>(
1929  _Up{__a[0], __a[1], __a[2], __a[3], __b[0], __b[1], __b[2], __b[3]});
1930  else if constexpr (input_width == 8)
1931  return reinterpret_cast<_R>(
1932  _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6], __a[7],
1933  __b[0], __b[1], __b[2], __b[3], __b[4], __b[5], __b[6], __b[7]});
1934  else if constexpr (input_width == 16)
1935  return reinterpret_cast<_R>(
1936  _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6],
1937  __a[7], __a[8], __a[9], __a[10], __a[11], __a[12], __a[13],
1938  __a[14], __a[15], __b[0], __b[1], __b[2], __b[3], __b[4],
1939  __b[5], __b[6], __b[7], __b[8], __b[9], __b[10], __b[11],
1940  __b[12], __b[13], __b[14], __b[15]});
1941  else if constexpr (input_width == 32)
1942  return reinterpret_cast<_R>(
1943  _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6],
1944  __a[7], __a[8], __a[9], __a[10], __a[11], __a[12], __a[13],
1945  __a[14], __a[15], __a[16], __a[17], __a[18], __a[19], __a[20],
1946  __a[21], __a[22], __a[23], __a[24], __a[25], __a[26], __a[27],
1947  __a[28], __a[29], __a[30], __a[31], __b[0], __b[1], __b[2],
1948  __b[3], __b[4], __b[5], __b[6], __b[7], __b[8], __b[9],
1949  __b[10], __b[11], __b[12], __b[13], __b[14], __b[15], __b[16],
1950  __b[17], __b[18], __b[19], __b[20], __b[21], __b[22], __b[23],
1951  __b[24], __b[25], __b[26], __b[27], __b[28], __b[29], __b[30],
1952  __b[31]});
1953  }
1954 
1955 // }}}
1956 // __zero_extend {{{
1957 template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1958  struct _ZeroExtendProxy
1959  {
1960  using value_type = typename _TVT::value_type;
1961  static constexpr size_t _Np = _TVT::_S_full_size;
1962  const _Tp __x;
1963 
1964  template <typename _To, typename _ToVT = _VectorTraits<_To>,
1965  typename
1966  = enable_if_t<is_same_v<typename _ToVT::value_type, value_type>>>
1967  _GLIBCXX_SIMD_INTRINSIC operator _To() const
1968  {
1969  constexpr size_t _ToN = _ToVT::_S_full_size;
1970  if constexpr (_ToN == _Np)
1971  return __x;
1972  else if constexpr (_ToN == 2 * _Np)
1973  {
1974 #ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_3
1975  if constexpr (__have_avx && _TVT::template _S_is<float, 4>)
1976  return __vector_bitcast<value_type>(
1977  _mm256_insertf128_ps(__m256(), __x, 0));
1978  else if constexpr (__have_avx && _TVT::template _S_is<double, 2>)
1979  return __vector_bitcast<value_type>(
1980  _mm256_insertf128_pd(__m256d(), __x, 0));
1981  else if constexpr (__have_avx2 && _Np * sizeof(value_type) == 16)
1982  return __vector_bitcast<value_type>(
1983  _mm256_insertf128_si256(__m256i(), __to_intrin(__x), 0));
1984  else if constexpr (__have_avx512f && _TVT::template _S_is<float, 8>)
1985  {
1986  if constexpr (__have_avx512dq)
1987  return __vector_bitcast<value_type>(
1988  _mm512_insertf32x8(__m512(), __x, 0));
1989  else
1990  return reinterpret_cast<__m512>(
1991  _mm512_insertf64x4(__m512d(),
1992  reinterpret_cast<__m256d>(__x), 0));
1993  }
1994  else if constexpr (__have_avx512f
1995  && _TVT::template _S_is<double, 4>)
1996  return __vector_bitcast<value_type>(
1997  _mm512_insertf64x4(__m512d(), __x, 0));
1998  else if constexpr (__have_avx512f && _Np * sizeof(value_type) == 32)
1999  return __vector_bitcast<value_type>(
2000  _mm512_inserti64x4(__m512i(), __to_intrin(__x), 0));
2001 #endif
2002  return __concat(__x, _Tp());
2003  }
2004  else if constexpr (_ToN == 4 * _Np)
2005  {
2006 #ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_3
2007  if constexpr (__have_avx512dq && _TVT::template _S_is<double, 2>)
2008  {
2009  return __vector_bitcast<value_type>(
2010  _mm512_insertf64x2(__m512d(), __x, 0));
2011  }
2012  else if constexpr (__have_avx512f
2013  && is_floating_point_v<value_type>)
2014  {
2015  return __vector_bitcast<value_type>(
2016  _mm512_insertf32x4(__m512(), reinterpret_cast<__m128>(__x),
2017  0));
2018  }
2019  else if constexpr (__have_avx512f && _Np * sizeof(value_type) == 16)
2020  {
2021  return __vector_bitcast<value_type>(
2022  _mm512_inserti32x4(__m512i(), __to_intrin(__x), 0));
2023  }
2024 #endif
2025  return __concat(__concat(__x, _Tp()),
2026  __vector_type_t<value_type, _Np * 2>());
2027  }
2028  else if constexpr (_ToN == 8 * _Np)
2029  return __concat(operator __vector_type_t<value_type, _Np * 4>(),
2030  __vector_type_t<value_type, _Np * 4>());
2031  else if constexpr (_ToN == 16 * _Np)
2032  return __concat(operator __vector_type_t<value_type, _Np * 8>(),
2033  __vector_type_t<value_type, _Np * 8>());
2034  else
2035  __assert_unreachable<_Tp>();
2036  }
2037  };
2038 
2039 template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
2040  _GLIBCXX_SIMD_INTRINSIC _ZeroExtendProxy<_Tp, _TVT>
2041  __zero_extend(_Tp __x)
2042  { return {__x}; }
2043 
2044 // }}}
2045 // __extract<_Np, By>{{{
2046 template <int _Offset,
2047  int _SplitBy,
2048  typename _Tp,
2049  typename _TVT = _VectorTraits<_Tp>,
2050  typename _R = __vector_type_t<typename _TVT::value_type,
2051  _TVT::_S_full_size / _SplitBy>>
2052  _GLIBCXX_SIMD_INTRINSIC constexpr _R
2053  __extract(_Tp __in)
2054  {
2055  using value_type = typename _TVT::value_type;
2056 #if _GLIBCXX_SIMD_X86INTRIN // {{{
2057  if constexpr (sizeof(_Tp) == 64 && _SplitBy == 4 && _Offset > 0)
2058  {
2059  if constexpr (__have_avx512dq && is_same_v<double, value_type>)
2060  return _mm512_extractf64x2_pd(__to_intrin(__in), _Offset);
2061  else if constexpr (is_floating_point_v<value_type>)
2062  return __vector_bitcast<value_type>(
2063  _mm512_extractf32x4_ps(__intrin_bitcast<__m512>(__in), _Offset));
2064  else
2065  return reinterpret_cast<_R>(
2066  _mm512_extracti32x4_epi32(__intrin_bitcast<__m512i>(__in),
2067  _Offset));
2068  }
2069  else
2070 #endif // _GLIBCXX_SIMD_X86INTRIN }}}
2071  {
2072 #ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_1
2073  using _W = conditional_t<
2074  is_floating_point_v<value_type>, double,
2075  conditional_t<(sizeof(_R) >= 16), long long, value_type>>;
2076  static_assert(sizeof(_R) % sizeof(_W) == 0);
2077  constexpr int __return_width = sizeof(_R) / sizeof(_W);
2078  using _Up = __vector_type_t<_W, __return_width>;
2079  const auto __x = __vector_bitcast<_W>(__in);
2080 #else
2081  constexpr int __return_width = _TVT::_S_full_size / _SplitBy;
2082  using _Up = _R;
2083  const __vector_type_t<value_type, _TVT::_S_full_size>& __x
2084  = __in; // only needed for _Tp = _SimdWrapper<value_type, _Np>
2085 #endif
2086  constexpr int _O = _Offset * __return_width;
2087  return __call_with_subscripts<__return_width, _O>(
2088  __x, [](auto... __entries) {
2089  return reinterpret_cast<_R>(_Up{__entries...});
2090  });
2091  }
2092  }
2093 
2094 // }}}
2095 // __lo/__hi64[z]{{{
2096 template <typename _Tp,
2097  typename _R
2098  = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2099  _GLIBCXX_SIMD_INTRINSIC constexpr _R
2100  __lo64(_Tp __x)
2101  {
2102  _R __r{};
2103  __builtin_memcpy(&__r, &__x, 8);
2104  return __r;
2105  }
2106 
2107 template <typename _Tp,
2108  typename _R
2109  = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2110  _GLIBCXX_SIMD_INTRINSIC constexpr _R
2111  __hi64(_Tp __x)
2112  {
2113  static_assert(sizeof(_Tp) == 16, "use __hi64z if you meant it");
2114  _R __r{};
2115  __builtin_memcpy(&__r, reinterpret_cast<const char*>(&__x) + 8, 8);
2116  return __r;
2117  }
2118 
2119 template <typename _Tp,
2120  typename _R
2121  = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2122  _GLIBCXX_SIMD_INTRINSIC constexpr _R
2123  __hi64z([[maybe_unused]] _Tp __x)
2124  {
2125  _R __r{};
2126  if constexpr (sizeof(_Tp) == 16)
2127  __builtin_memcpy(&__r, reinterpret_cast<const char*>(&__x) + 8, 8);
2128  return __r;
2129  }
2130 
2131 // }}}
2132 // __lo/__hi128{{{
2133 template <typename _Tp>
2134  _GLIBCXX_SIMD_INTRINSIC constexpr auto
2135  __lo128(_Tp __x)
2136  { return __extract<0, sizeof(_Tp) / 16>(__x); }
2137 
2138 template <typename _Tp>
2139  _GLIBCXX_SIMD_INTRINSIC constexpr auto
2140  __hi128(_Tp __x)
2141  {
2142  static_assert(sizeof(__x) == 32);
2143  return __extract<1, 2>(__x);
2144  }
2145 
2146 // }}}
2147 // __lo/__hi256{{{
2148 template <typename _Tp>
2149  _GLIBCXX_SIMD_INTRINSIC constexpr auto
2150  __lo256(_Tp __x)
2151  {
2152  static_assert(sizeof(__x) == 64);
2153  return __extract<0, 2>(__x);
2154  }
2155 
2156 template <typename _Tp>
2157  _GLIBCXX_SIMD_INTRINSIC constexpr auto
2158  __hi256(_Tp __x)
2159  {
2160  static_assert(sizeof(__x) == 64);
2161  return __extract<1, 2>(__x);
2162  }
2163 
2164 // }}}
2165 // __auto_bitcast{{{
2166 template <typename _Tp>
2167  struct _AutoCast
2168  {
2169  static_assert(__is_vector_type_v<_Tp>);
2170 
2171  const _Tp __x;
2172 
2173  template <typename _Up, typename _UVT = _VectorTraits<_Up>>
2174  _GLIBCXX_SIMD_INTRINSIC constexpr operator _Up() const
2175  { return __intrin_bitcast<typename _UVT::type>(__x); }
2176  };
2177 
2178 template <typename _Tp>
2179  _GLIBCXX_SIMD_INTRINSIC constexpr _AutoCast<_Tp>
2180  __auto_bitcast(const _Tp& __x)
2181  { return {__x}; }
2182 
2183 template <typename _Tp, size_t _Np>
2184  _GLIBCXX_SIMD_INTRINSIC constexpr
2185  _AutoCast<typename _SimdWrapper<_Tp, _Np>::_BuiltinType>
2186  __auto_bitcast(const _SimdWrapper<_Tp, _Np>& __x)
2187  { return {__x._M_data}; }
2188 
2189 // }}}
2190 // ^^^ ---- builtin vector types [[gnu::vector_size(N)]] and operations ---- ^^^
2191 
2192 #if _GLIBCXX_SIMD_HAVE_SSE_ABI
2193 // __bool_storage_member_type{{{
2194 #if _GLIBCXX_SIMD_HAVE_AVX512F && _GLIBCXX_SIMD_X86INTRIN
2195 template <size_t _Size>
2196  struct __bool_storage_member_type
2197  {
2198  static_assert((_Size & (_Size - 1)) != 0,
2199  "This trait may only be used for non-power-of-2 sizes. "
2200  "Power-of-2 sizes must be specialized.");
2201  using type =
2202  typename __bool_storage_member_type<std::__bit_ceil(_Size)>::type;
2203  };
2204 
2205 template <>
2206  struct __bool_storage_member_type<1> { using type = bool; };
2207 
2208 template <>
2209  struct __bool_storage_member_type<2> { using type = __mmask8; };
2210 
2211 template <>
2212  struct __bool_storage_member_type<4> { using type = __mmask8; };
2213 
2214 template <>
2215  struct __bool_storage_member_type<8> { using type = __mmask8; };
2216 
2217 template <>
2218  struct __bool_storage_member_type<16> { using type = __mmask16; };
2219 
2220 template <>
2221  struct __bool_storage_member_type<32> { using type = __mmask32; };
2222 
2223 template <>
2224  struct __bool_storage_member_type<64> { using type = __mmask64; };
2225 #endif // _GLIBCXX_SIMD_HAVE_AVX512F
2226 
2227 // }}}
2228 // __intrinsic_type (x86){{{
2229 // the following excludes bool via __is_vectorizable
2230 #if _GLIBCXX_SIMD_HAVE_SSE
2231 template <typename _Tp, size_t _Bytes>
2232  struct __intrinsic_type<_Tp, _Bytes,
2233  enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 64>>
2234  {
2235  static_assert(!is_same_v<_Tp, long double>,
2236  "no __intrinsic_type support for long double on x86");
2237 
2238  static constexpr size_t _S_VBytes = _Bytes <= 16 ? 16
2239  : _Bytes <= 32 ? 32
2240  : 64;
2241 
2242  using type [[__gnu__::__vector_size__(_S_VBytes)]]
2243  = conditional_t<is_integral_v<_Tp>, long long int, _Tp>;
2244  };
2245 #endif // _GLIBCXX_SIMD_HAVE_SSE
2246 
2247 // }}}
2248 #endif // _GLIBCXX_SIMD_HAVE_SSE_ABI
2249 // __intrinsic_type (ARM){{{
2250 #if _GLIBCXX_SIMD_HAVE_NEON
2251 template <>
2252  struct __intrinsic_type<float, 8, void>
2253  { using type = float32x2_t; };
2254 
2255 template <>
2256  struct __intrinsic_type<float, 16, void>
2257  { using type = float32x4_t; };
2258 
2259 #if _GLIBCXX_SIMD_HAVE_NEON_A64
2260 template <>
2261  struct __intrinsic_type<double, 8, void>
2262  { using type = float64x1_t; };
2263 
2264 template <>
2265  struct __intrinsic_type<double, 16, void>
2266  { using type = float64x2_t; };
2267 #endif
2268 
2269 #define _GLIBCXX_SIMD_ARM_INTRIN(_Bits, _Np) \
2270 template <> \
2271  struct __intrinsic_type<__int_with_sizeof_t<_Bits / 8>, \
2272  _Np * _Bits / 8, void> \
2273  { using type = int##_Bits##x##_Np##_t; }; \
2274 template <> \
2275  struct __intrinsic_type<make_unsigned_t<__int_with_sizeof_t<_Bits / 8>>, \
2276  _Np * _Bits / 8, void> \
2277  { using type = uint##_Bits##x##_Np##_t; }
2278 _GLIBCXX_SIMD_ARM_INTRIN(8, 8);
2279 _GLIBCXX_SIMD_ARM_INTRIN(8, 16);
2280 _GLIBCXX_SIMD_ARM_INTRIN(16, 4);
2281 _GLIBCXX_SIMD_ARM_INTRIN(16, 8);
2282 _GLIBCXX_SIMD_ARM_INTRIN(32, 2);
2283 _GLIBCXX_SIMD_ARM_INTRIN(32, 4);
2284 _GLIBCXX_SIMD_ARM_INTRIN(64, 1);
2285 _GLIBCXX_SIMD_ARM_INTRIN(64, 2);
2286 #undef _GLIBCXX_SIMD_ARM_INTRIN
2287 
2288 template <typename _Tp, size_t _Bytes>
2289  struct __intrinsic_type<_Tp, _Bytes,
2290  enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 16>>
2291  {
2292  static constexpr int _SVecBytes = _Bytes <= 8 ? 8 : 16;
2293  using _Ip = __int_for_sizeof_t<_Tp>;
2294  using _Up = conditional_t<
2295  is_floating_point_v<_Tp>, _Tp,
2296  conditional_t<is_unsigned_v<_Tp>, make_unsigned_t<_Ip>, _Ip>>;
2297  static_assert(!is_same_v<_Tp, _Up> || _SVecBytes != _Bytes,
2298  "should use explicit specialization above");
2299  using type = typename __intrinsic_type<_Up, _SVecBytes>::type;
2300  };
2301 #endif // _GLIBCXX_SIMD_HAVE_NEON
2302 
2303 // }}}
2304 // __intrinsic_type (PPC){{{
2305 #ifdef __ALTIVEC__
2306 template <typename _Tp>
2307  struct __intrinsic_type_impl;
2308 
2309 #define _GLIBCXX_SIMD_PPC_INTRIN(_Tp) \
2310  template <> \
2311  struct __intrinsic_type_impl<_Tp> { using type = __vector _Tp; }
2312 _GLIBCXX_SIMD_PPC_INTRIN(float);
2313 _GLIBCXX_SIMD_PPC_INTRIN(double);
2314 _GLIBCXX_SIMD_PPC_INTRIN(signed char);
2315 _GLIBCXX_SIMD_PPC_INTRIN(unsigned char);
2316 _GLIBCXX_SIMD_PPC_INTRIN(signed short);
2317 _GLIBCXX_SIMD_PPC_INTRIN(unsigned short);
2318 _GLIBCXX_SIMD_PPC_INTRIN(signed int);
2319 _GLIBCXX_SIMD_PPC_INTRIN(unsigned int);
2320 _GLIBCXX_SIMD_PPC_INTRIN(signed long);
2321 _GLIBCXX_SIMD_PPC_INTRIN(unsigned long);
2322 _GLIBCXX_SIMD_PPC_INTRIN(signed long long);
2323 _GLIBCXX_SIMD_PPC_INTRIN(unsigned long long);
2324 #undef _GLIBCXX_SIMD_PPC_INTRIN
2325 
2326 template <typename _Tp, size_t _Bytes>
2327  struct __intrinsic_type<_Tp, _Bytes,
2328  enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 16>>
2329  {
2330  static constexpr bool _S_is_ldouble = is_same_v<_Tp, long double>;
2331  // allow _Tp == long double with -mlong-double-64
2332  static_assert(!(_S_is_ldouble && sizeof(long double) > sizeof(double)),
2333  "no __intrinsic_type support for long double on PPC");
2334 #ifndef __VSX__
2335  static_assert(!is_same_v<_Tp, double>,
2336  "no __intrinsic_type support for double on PPC w/o VSX");
2337 #endif
2338  using type =
2339  typename __intrinsic_type_impl<
2340  conditional_t<is_floating_point_v<_Tp>,
2341  conditional_t<_S_is_ldouble, double, _Tp>,
2342  __int_for_sizeof_t<_Tp>>>::type;
2343  };
2344 #endif // __ALTIVEC__
2345 
2346 // }}}
2347 // _SimdWrapper<bool>{{{1
2348 template <size_t _Width>
2349  struct _SimdWrapper<bool, _Width,
2350  void_t<typename __bool_storage_member_type<_Width>::type>>
2351  {
2352  using _BuiltinType = typename __bool_storage_member_type<_Width>::type;
2353  using value_type = bool;
2354 
2355  static constexpr size_t _S_full_size = sizeof(_BuiltinType) * __CHAR_BIT__;
2356 
2357  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<bool, _S_full_size>
2358  __as_full_vector() const { return _M_data; }
2359 
2360  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper() = default;
2361  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper(_BuiltinType __k)
2362  : _M_data(__k) {};
2363 
2364  _GLIBCXX_SIMD_INTRINSIC operator const _BuiltinType&() const
2365  { return _M_data; }
2366 
2367  _GLIBCXX_SIMD_INTRINSIC operator _BuiltinType&()
2368  { return _M_data; }
2369 
2370  _GLIBCXX_SIMD_INTRINSIC _BuiltinType __intrin() const
2371  { return _M_data; }
2372 
2373  _GLIBCXX_SIMD_INTRINSIC constexpr value_type operator[](size_t __i) const
2374  { return _M_data & (_BuiltinType(1) << __i); }
2375 
2376  template <size_t __i>
2377  _GLIBCXX_SIMD_INTRINSIC constexpr value_type
2378  operator[](_SizeConstant<__i>) const
2379  { return _M_data & (_BuiltinType(1) << __i); }
2380 
2381  _GLIBCXX_SIMD_INTRINSIC constexpr void _M_set(size_t __i, value_type __x)
2382  {
2383  if (__x)
2384  _M_data |= (_BuiltinType(1) << __i);
2385  else
2386  _M_data &= ~(_BuiltinType(1) << __i);
2387  }
2388 
2389  _GLIBCXX_SIMD_INTRINSIC
2390  constexpr bool _M_is_constprop() const
2391  { return __builtin_constant_p(_M_data); }
2392 
2393  _GLIBCXX_SIMD_INTRINSIC constexpr bool _M_is_constprop_none_of() const
2394  {
2395  if (__builtin_constant_p(_M_data))
2396  {
2397  constexpr int __nbits = sizeof(_BuiltinType) * __CHAR_BIT__;
2398  constexpr _BuiltinType __active_mask
2399  = ~_BuiltinType() >> (__nbits - _Width);
2400  return (_M_data & __active_mask) == 0;
2401  }
2402  return false;
2403  }
2404 
2405  _GLIBCXX_SIMD_INTRINSIC constexpr bool _M_is_constprop_all_of() const
2406  {
2407  if (__builtin_constant_p(_M_data))
2408  {
2409  constexpr int __nbits = sizeof(_BuiltinType) * __CHAR_BIT__;
2410  constexpr _BuiltinType __active_mask
2411  = ~_BuiltinType() >> (__nbits - _Width);
2412  return (_M_data & __active_mask) == __active_mask;
2413  }
2414  return false;
2415  }
2416 
2417  _BuiltinType _M_data;
2418  };
2419 
2420 // _SimdWrapperBase{{{1
2421 template <bool _MustZeroInitPadding, typename _BuiltinType>
2422  struct _SimdWrapperBase;
2423 
2424 template <typename _BuiltinType>
2425  struct _SimdWrapperBase<false, _BuiltinType> // no padding or no SNaNs
2426  {
2427  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapperBase() = default;
2428  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapperBase(_BuiltinType __init)
2429  : _M_data(__init)
2430  {}
2431 
2432  _BuiltinType _M_data;
2433  };
2434 
2435 template <typename _BuiltinType>
2436  struct _SimdWrapperBase<true, _BuiltinType> // with padding that needs to
2437  // never become SNaN
2438  {
2439  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapperBase() : _M_data() {}
2440  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapperBase(_BuiltinType __init)
2441  : _M_data(__init)
2442  {}
2443 
2444  _BuiltinType _M_data;
2445  };
2446 
2447 // }}}
2448 // _SimdWrapper{{{
2449 template <typename _Tp, size_t _Width>
2450  struct _SimdWrapper<
2451  _Tp, _Width,
2452  void_t<__vector_type_t<_Tp, _Width>, __intrinsic_type_t<_Tp, _Width>>>
2453  : _SimdWrapperBase<__has_iec559_behavior<__signaling_NaN, _Tp>::value
2454  && sizeof(_Tp) * _Width
2455  == sizeof(__vector_type_t<_Tp, _Width>),
2456  __vector_type_t<_Tp, _Width>>
2457  {
2458  using _Base
2459  = _SimdWrapperBase<__has_iec559_behavior<__signaling_NaN, _Tp>::value
2460  && sizeof(_Tp) * _Width
2461  == sizeof(__vector_type_t<_Tp, _Width>),
2462  __vector_type_t<_Tp, _Width>>;
2463 
2464  static_assert(__is_vectorizable_v<_Tp>);
2465  static_assert(_Width >= 2); // 1 doesn't make sense, use _Tp directly then
2466 
2467  using _BuiltinType = __vector_type_t<_Tp, _Width>;
2468  using value_type = _Tp;
2469 
2470  static inline constexpr size_t _S_full_size
2471  = sizeof(_BuiltinType) / sizeof(value_type);
2472  static inline constexpr int _S_size = _Width;
2473  static inline constexpr bool _S_is_partial = _S_full_size != _S_size;
2474 
2475  using _Base::_M_data;
2476 
2477  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<_Tp, _S_full_size>
2478  __as_full_vector() const
2479  { return _M_data; }
2480 
2481  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper(initializer_list<_Tp> __init)
2482  : _Base(__generate_from_n_evaluations<_Width, _BuiltinType>(
2483  [&](auto __i) { return __init.begin()[__i.value]; })) {}
2484 
2485  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper() = default;
2486  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper(const _SimdWrapper&)
2487  = default;
2488  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper(_SimdWrapper&&) = default;
2489 
2490  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper&
2491  operator=(const _SimdWrapper&) = default;
2492  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper&
2493  operator=(_SimdWrapper&&) = default;
2494 
2495  template <typename _V, typename = enable_if_t<disjunction_v<
2496  is_same<_V, __vector_type_t<_Tp, _Width>>,
2497  is_same<_V, __intrinsic_type_t<_Tp, _Width>>>>>
2498  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper(_V __x)
2499  // __vector_bitcast can convert e.g. __m128 to __vector(2) float
2500  : _Base(__vector_bitcast<_Tp, _Width>(__x)) {}
2501 
2502  template <typename... _As,
2503  typename = enable_if_t<((is_same_v<simd_abi::scalar, _As> && ...)
2504  && sizeof...(_As) <= _Width)>>
2505  _GLIBCXX_SIMD_INTRINSIC constexpr
2506  operator _SimdTuple<_Tp, _As...>() const
2507  {
2508  const auto& dd = _M_data; // workaround for GCC7 ICE
2509  return __generate_from_n_evaluations<sizeof...(_As),
2510  _SimdTuple<_Tp, _As...>>([&](
2511  auto __i) constexpr { return dd[int(__i)]; });
2512  }
2513 
2514  _GLIBCXX_SIMD_INTRINSIC constexpr operator const _BuiltinType&() const
2515  { return _M_data; }
2516 
2517  _GLIBCXX_SIMD_INTRINSIC constexpr operator _BuiltinType&()
2518  { return _M_data; }
2519 
2520  _GLIBCXX_SIMD_INTRINSIC constexpr _Tp operator[](size_t __i) const
2521  { return _M_data[__i]; }
2522 
2523  template <size_t __i>
2524  _GLIBCXX_SIMD_INTRINSIC constexpr _Tp operator[](_SizeConstant<__i>) const
2525  { return _M_data[__i]; }
2526 
2527  _GLIBCXX_SIMD_INTRINSIC constexpr void _M_set(size_t __i, _Tp __x)
2528  { _M_data[__i] = __x; }
2529 
2530  _GLIBCXX_SIMD_INTRINSIC
2531  constexpr bool _M_is_constprop() const
2532  { return __builtin_constant_p(_M_data); }
2533 
2534  _GLIBCXX_SIMD_INTRINSIC constexpr bool _M_is_constprop_none_of() const
2535  {
2536  if (__builtin_constant_p(_M_data))
2537  {
2538  bool __r = true;
2539  if constexpr (is_floating_point_v<_Tp>)
2540  {
2541  using _Ip = __int_for_sizeof_t<_Tp>;
2542  const auto __intdata = __vector_bitcast<_Ip>(_M_data);
2543  __execute_n_times<_Width>(
2544  [&](auto __i) { __r &= __intdata[__i.value] == _Ip(); });
2545  }
2546  else
2547  __execute_n_times<_Width>(
2548  [&](auto __i) { __r &= _M_data[__i.value] == _Tp(); });
2549  return __r;
2550  }
2551  return false;
2552  }
2553 
2554  _GLIBCXX_SIMD_INTRINSIC constexpr bool _M_is_constprop_all_of() const
2555  {
2556  if (__builtin_constant_p(_M_data))
2557  {
2558  bool __r = true;
2559  if constexpr (is_floating_point_v<_Tp>)
2560  {
2561  using _Ip = __int_for_sizeof_t<_Tp>;
2562  const auto __intdata = __vector_bitcast<_Ip>(_M_data);
2563  __execute_n_times<_Width>(
2564  [&](auto __i) { __r &= __intdata[__i.value] == ~_Ip(); });
2565  }
2566  else
2567  __execute_n_times<_Width>(
2568  [&](auto __i) { __r &= _M_data[__i.value] == ~_Tp(); });
2569  return __r;
2570  }
2571  return false;
2572  }
2573  };
2574 
2575 // }}}
2576 
2577 // __vectorized_sizeof {{{
2578 template <typename _Tp>
2579  constexpr size_t
2580  __vectorized_sizeof()
2581  {
2582  if constexpr (!__is_vectorizable_v<_Tp>)
2583  return 0;
2584 
2585  if constexpr (sizeof(_Tp) <= 8)
2586  {
2587  // X86:
2588  if constexpr (__have_avx512bw)
2589  return 64;
2590  if constexpr (__have_avx512f && sizeof(_Tp) >= 4)
2591  return 64;
2592  if constexpr (__have_avx2)
2593  return 32;
2594  if constexpr (__have_avx && is_floating_point_v<_Tp>)
2595  return 32;
2596  if constexpr (__have_sse2)
2597  return 16;
2598  if constexpr (__have_sse && is_same_v<_Tp, float>)
2599  return 16;
2600  /* The following is too much trouble because of mixed MMX and x87 code.
2601  * While nothing here explicitly calls MMX instructions of registers,
2602  * they are still emitted but no EMMS cleanup is done.
2603  if constexpr (__have_mmx && sizeof(_Tp) <= 4 && is_integral_v<_Tp>)
2604  return 8;
2605  */
2606 
2607  // PowerPC:
2608  if constexpr (__have_power8vec
2609  || (__have_power_vmx && (sizeof(_Tp) < 8))
2610  || (__have_power_vsx && is_floating_point_v<_Tp>) )
2611  return 16;
2612 
2613  // ARM:
2614  if constexpr (__have_neon_a64
2615  || (__have_neon_a32 && !is_same_v<_Tp, double>) )
2616  return 16;
2617  if constexpr (__have_neon
2618  && sizeof(_Tp) < 8
2619  // Only allow fp if the user allows non-ICE559 fp (e.g.
2620  // via -ffast-math). ARMv7 NEON fp is not conforming to
2621  // IEC559.
2622  && (__support_neon_float || !is_floating_point_v<_Tp>))
2623  return 16;
2624  }
2625 
2626  return sizeof(_Tp);
2627  }
2628 
2629 // }}}
2630 namespace simd_abi {
2631 // most of simd_abi is defined in simd_detail.h
2632 template <typename _Tp>
2633  inline constexpr int max_fixed_size
2634  = (__have_avx512bw && sizeof(_Tp) == 1) ? 64 : 32;
2635 
2636 // compatible {{{
2637 #if defined __x86_64__ || defined __aarch64__
2638 template <typename _Tp>
2639  using compatible = conditional_t<(sizeof(_Tp) <= 8), _VecBuiltin<16>, scalar>;
2640 #elif defined __ARM_NEON
2641 // FIXME: not sure, probably needs to be scalar (or dependent on the hard-float
2642 // ABI?)
2643 template <typename _Tp>
2644  using compatible
2645  = conditional_t<(sizeof(_Tp) < 8
2646  && (__support_neon_float || !is_floating_point_v<_Tp>)),
2647  _VecBuiltin<16>, scalar>;
2648 #else
2649 template <typename>
2650  using compatible = scalar;
2651 #endif
2652 
2653 // }}}
2654 // native {{{
2655 template <typename _Tp>
2656  constexpr auto
2657  __determine_native_abi()
2658  {
2659  constexpr size_t __bytes = __vectorized_sizeof<_Tp>();
2660  if constexpr (__bytes == sizeof(_Tp))
2661  return static_cast<scalar*>(nullptr);
2662  else if constexpr (__have_avx512vl || (__have_avx512f && __bytes == 64))
2663  return static_cast<_VecBltnBtmsk<__bytes>*>(nullptr);
2664  else
2665  return static_cast<_VecBuiltin<__bytes>*>(nullptr);
2666  }
2667 
2668 template <typename _Tp, typename = enable_if_t<__is_vectorizable_v<_Tp>>>
2669  using native = remove_pointer_t<decltype(__determine_native_abi<_Tp>())>;
2670 
2671 // }}}
2672 // __default_abi {{{
2673 #if defined _GLIBCXX_SIMD_DEFAULT_ABI
2674 template <typename _Tp>
2675  using __default_abi = _GLIBCXX_SIMD_DEFAULT_ABI<_Tp>;
2676 #else
2677 template <typename _Tp>
2678  using __default_abi = compatible<_Tp>;
2679 #endif
2680 
2681 // }}}
2682 } // namespace simd_abi
2683 
2684 // traits {{{1
2685 // is_abi_tag {{{2
2686 template <typename _Tp, typename = void_t<>>
2687  struct is_abi_tag : false_type {};
2688 
2689 template <typename _Tp>
2690  struct is_abi_tag<_Tp, void_t<typename _Tp::_IsValidAbiTag>>
2691  : public _Tp::_IsValidAbiTag {};
2692 
2693 template <typename _Tp>
2694  inline constexpr bool is_abi_tag_v = is_abi_tag<_Tp>::value;
2695 
2696 // is_simd(_mask) {{{2
2697 template <typename _Tp>
2698  struct is_simd : public false_type {};
2699 
2700 template <typename _Tp>
2701  inline constexpr bool is_simd_v = is_simd<_Tp>::value;
2702 
2703 template <typename _Tp>
2704  struct is_simd_mask : public false_type {};
2705 
2706 template <typename _Tp>
2707 inline constexpr bool is_simd_mask_v = is_simd_mask<_Tp>::value;
2708 
2709 // simd_size {{{2
2710 template <typename _Tp, typename _Abi, typename = void>
2711  struct __simd_size_impl {};
2712 
2713 template <typename _Tp, typename _Abi>
2714  struct __simd_size_impl<
2715  _Tp, _Abi,
2716  enable_if_t<conjunction_v<__is_vectorizable<_Tp>, is_abi_tag<_Abi>>>>
2717  : _SizeConstant<_Abi::template _S_size<_Tp>> {};
2718 
2719 template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>>
2720  struct simd_size : __simd_size_impl<_Tp, _Abi> {};
2721 
2722 template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>>
2723  inline constexpr size_t simd_size_v = simd_size<_Tp, _Abi>::value;
2724 
2725 // simd_abi::deduce {{{2
2726 template <typename _Tp, size_t _Np, typename = void>
2727  struct __deduce_impl;
2728 
2729 namespace simd_abi {
2730 /**
2731  * @tparam _Tp The requested `value_type` for the elements.
2732  * @tparam _Np The requested number of elements.
2733  * @tparam _Abis This parameter is ignored, since this implementation cannot
2734  * make any use of it. Either __a good native ABI is matched and used as `type`
2735  * alias, or the `fixed_size<_Np>` ABI is used, which internally is built from
2736  * the best matching native ABIs.
2737  */
2738 template <typename _Tp, size_t _Np, typename...>
2739  struct deduce : __deduce_impl<_Tp, _Np> {};
2740 
2741 template <typename _Tp, size_t _Np, typename... _Abis>
2742  using deduce_t = typename deduce<_Tp, _Np, _Abis...>::type;
2743 } // namespace simd_abi
2744 
2745 // }}}2
2746 // rebind_simd {{{2
2747 template <typename _Tp, typename _V, typename = void>
2748  struct rebind_simd;
2749 
2750 template <typename _Tp, typename _Up, typename _Abi>
2751  struct rebind_simd<
2752  _Tp, simd<_Up, _Abi>,
2753  void_t<simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>>
2754  {
2755  using type
2756  = simd<_Tp, simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>;
2757  };
2758 
2759 template <typename _Tp, typename _Up, typename _Abi>
2760  struct rebind_simd<
2761  _Tp, simd_mask<_Up, _Abi>,
2762  void_t<simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>>
2763  {
2764  using type
2765  = simd_mask<_Tp, simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>;
2766  };
2767 
2768 template <typename _Tp, typename _V>
2769  using rebind_simd_t = typename rebind_simd<_Tp, _V>::type;
2770 
2771 // resize_simd {{{2
2772 template <int _Np, typename _V, typename = void>
2773  struct resize_simd;
2774 
2775 template <int _Np, typename _Tp, typename _Abi>
2776  struct resize_simd<_Np, simd<_Tp, _Abi>,
2777  void_t<simd_abi::deduce_t<_Tp, _Np, _Abi>>>
2778  { using type = simd<_Tp, simd_abi::deduce_t<_Tp, _Np, _Abi>>; };
2779 
2780 template <int _Np, typename _Tp, typename _Abi>
2781  struct resize_simd<_Np, simd_mask<_Tp, _Abi>,
2782  void_t<simd_abi::deduce_t<_Tp, _Np, _Abi>>>
2783  { using type = simd_mask<_Tp, simd_abi::deduce_t<_Tp, _Np, _Abi>>; };
2784 
2785 template <int _Np, typename _V>
2786  using resize_simd_t = typename resize_simd<_Np, _V>::type;
2787 
2788 // }}}2
2789 // memory_alignment {{{2
2790 template <typename _Tp, typename _Up = typename _Tp::value_type>
2791  struct memory_alignment
2792  : public _SizeConstant<vector_aligned_tag::_S_alignment<_Tp, _Up>> {};
2793 
2794 template <typename _Tp, typename _Up = typename _Tp::value_type>
2795  inline constexpr size_t memory_alignment_v = memory_alignment<_Tp, _Up>::value;
2796 
2797 // class template simd [simd] {{{1
2798 template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>>
2799  class simd;
2800 
2801 template <typename _Tp, typename _Abi>
2802  struct is_simd<simd<_Tp, _Abi>> : public true_type {};
2803 
2804 template <typename _Tp>
2805  using native_simd = simd<_Tp, simd_abi::native<_Tp>>;
2806 
2807 template <typename _Tp, int _Np>
2808  using fixed_size_simd = simd<_Tp, simd_abi::fixed_size<_Np>>;
2809 
2810 template <typename _Tp, size_t _Np>
2811  using __deduced_simd = simd<_Tp, simd_abi::deduce_t<_Tp, _Np>>;
2812 
2813 // class template simd_mask [simd_mask] {{{1
2814 template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>>
2815  class simd_mask;
2816 
2817 template <typename _Tp, typename _Abi>
2818  struct is_simd_mask<simd_mask<_Tp, _Abi>> : public true_type {};
2819 
2820 template <typename _Tp>
2821  using native_simd_mask = simd_mask<_Tp, simd_abi::native<_Tp>>;
2822 
2823 template <typename _Tp, int _Np>
2824  using fixed_size_simd_mask = simd_mask<_Tp, simd_abi::fixed_size<_Np>>;
2825 
2826 template <typename _Tp, size_t _Np>
2827  using __deduced_simd_mask = simd_mask<_Tp, simd_abi::deduce_t<_Tp, _Np>>;
2828 
2829 // casts [simd.casts] {{{1
2830 // static_simd_cast {{{2
2831 template <typename _Tp, typename _Up, typename _Ap, bool = is_simd_v<_Tp>,
2832  typename = void>
2833  struct __static_simd_cast_return_type;
2834 
2835 template <typename _Tp, typename _A0, typename _Up, typename _Ap>
2836  struct __static_simd_cast_return_type<simd_mask<_Tp, _A0>, _Up, _Ap, false,
2837  void>
2838  : __static_simd_cast_return_type<simd<_Tp, _A0>, _Up, _Ap> {};
2839 
2840 template <typename _Tp, typename _Up, typename _Ap>
2841  struct __static_simd_cast_return_type<
2842  _Tp, _Up, _Ap, true, enable_if_t<_Tp::size() == simd_size_v<_Up, _Ap>>>
2843  { using type = _Tp; };
2844 
2845 template <typename _Tp, typename _Ap>
2846  struct __static_simd_cast_return_type<_Tp, _Tp, _Ap, false,
2847 #ifdef _GLIBCXX_SIMD_FIX_P2TS_ISSUE66
2848  enable_if_t<__is_vectorizable_v<_Tp>>
2849 #else
2850  void
2851 #endif
2852  >
2853  { using type = simd<_Tp, _Ap>; };
2854 
2855 template <typename _Tp, typename = void>
2856  struct __safe_make_signed { using type = _Tp;};
2857 
2858 template <typename _Tp>
2859  struct __safe_make_signed<_Tp, enable_if_t<is_integral_v<_Tp>>>
2860  {
2861  // the extra make_unsigned_t is because of PR85951
2862  using type = make_signed_t<make_unsigned_t<_Tp>>;
2863  };
2864 
2865 template <typename _Tp>
2866  using safe_make_signed_t = typename __safe_make_signed<_Tp>::type;
2867 
2868 template <typename _Tp, typename _Up, typename _Ap>
2869  struct __static_simd_cast_return_type<_Tp, _Up, _Ap, false,
2870 #ifdef _GLIBCXX_SIMD_FIX_P2TS_ISSUE66
2871  enable_if_t<__is_vectorizable_v<_Tp>>
2872 #else
2873  void
2874 #endif
2875  >
2876  {
2877  using type = conditional_t<
2878  (is_integral_v<_Up> && is_integral_v<_Tp> &&
2879 #ifndef _GLIBCXX_SIMD_FIX_P2TS_ISSUE65
2880  is_signed_v<_Up> != is_signed_v<_Tp> &&
2881 #endif
2882  is_same_v<safe_make_signed_t<_Up>, safe_make_signed_t<_Tp>>),
2883  simd<_Tp, _Ap>, fixed_size_simd<_Tp, simd_size_v<_Up, _Ap>>>;
2884  };
2885 
2886 template <typename _Tp, typename _Up, typename _Ap,
2887  typename _R
2888  = typename __static_simd_cast_return_type<_Tp, _Up, _Ap>::type>
2889  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _R
2890  static_simd_cast(const simd<_Up, _Ap>& __x)
2891  {
2892  if constexpr (is_same<_R, simd<_Up, _Ap>>::value)
2893  return __x;
2894  else
2895  {
2896  _SimdConverter<_Up, _Ap, typename _R::value_type, typename _R::abi_type>
2897  __c;
2898  return _R(__private_init, __c(__data(__x)));
2899  }
2900  }
2901 
2902 namespace __proposed {
2903 template <typename _Tp, typename _Up, typename _Ap,
2904  typename _R
2905  = typename __static_simd_cast_return_type<_Tp, _Up, _Ap>::type>
2906  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR typename _R::mask_type
2907  static_simd_cast(const simd_mask<_Up, _Ap>& __x)
2908  {
2909  using _RM = typename _R::mask_type;
2910  return {__private_init, _RM::abi_type::_MaskImpl::template _S_convert<
2911  typename _RM::simd_type::value_type>(__x)};
2912  }
2913 
2914 template <typename _To, typename _Up, typename _Abi>
2915  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
2916  _To
2917  simd_bit_cast(const simd<_Up, _Abi>& __x)
2918  {
2919  using _Tp = typename _To::value_type;
2920  using _ToMember = typename _SimdTraits<_Tp, typename _To::abi_type>::_SimdMember;
2921  using _From = simd<_Up, _Abi>;
2922  using _FromMember = typename _SimdTraits<_Up, _Abi>::_SimdMember;
2923  // with concepts, the following should be constraints
2924  static_assert(sizeof(_To) == sizeof(_From));
2925  static_assert(is_trivially_copyable_v<_Tp> && is_trivially_copyable_v<_Up>);
2926  static_assert(is_trivially_copyable_v<_ToMember> && is_trivially_copyable_v<_FromMember>);
2927 #if __has_builtin(__builtin_bit_cast)
2928  return {__private_init, __builtin_bit_cast(_ToMember, __data(__x))};
2929 #else
2930  return {__private_init, __bit_cast<_ToMember>(__data(__x))};
2931 #endif
2932  }
2933 
2934 template <typename _To, typename _Up, typename _Abi>
2935  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
2936  _To
2937  simd_bit_cast(const simd_mask<_Up, _Abi>& __x)
2938  {
2939  using _From = simd_mask<_Up, _Abi>;
2940  static_assert(sizeof(_To) == sizeof(_From));
2941  static_assert(is_trivially_copyable_v<_From>);
2942  // _To can be simd<T, A>, specifically simd<T, fixed_size<N>> in which case _To is not trivially
2943  // copyable.
2944  if constexpr (is_simd_v<_To>)
2945  {
2946  using _Tp = typename _To::value_type;
2947  using _ToMember = typename _SimdTraits<_Tp, typename _To::abi_type>::_SimdMember;
2948  static_assert(is_trivially_copyable_v<_ToMember>);
2949 #if __has_builtin(__builtin_bit_cast)
2950  return {__private_init, __builtin_bit_cast(_ToMember, __x)};
2951 #else
2952  return {__private_init, __bit_cast<_ToMember>(__x)};
2953 #endif
2954  }
2955  else
2956  {
2957  static_assert(is_trivially_copyable_v<_To>);
2958 #if __has_builtin(__builtin_bit_cast)
2959  return __builtin_bit_cast(_To, __x);
2960 #else
2961  return __bit_cast<_To>(__x);
2962 #endif
2963  }
2964  }
2965 } // namespace __proposed
2966 
2967 // simd_cast {{{2
2968 template <typename _Tp, typename _Up, typename _Ap,
2969  typename _To = __value_type_or_identity_t<_Tp>>
2970  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR auto
2971  simd_cast(const simd<_ValuePreserving<_Up, _To>, _Ap>& __x)
2972  -> decltype(static_simd_cast<_Tp>(__x))
2973  { return static_simd_cast<_Tp>(__x); }
2974 
2975 namespace __proposed {
2976 template <typename _Tp, typename _Up, typename _Ap,
2977  typename _To = __value_type_or_identity_t<_Tp>>
2978  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR auto
2979  simd_cast(const simd_mask<_ValuePreserving<_Up, _To>, _Ap>& __x)
2980  -> decltype(static_simd_cast<_Tp>(__x))
2981  { return static_simd_cast<_Tp>(__x); }
2982 } // namespace __proposed
2983 
2984 // }}}2
2985 // resizing_simd_cast {{{
2986 namespace __proposed {
2987 /* Proposed spec:
2988 
2989 template <class T, class U, class Abi>
2990 T resizing_simd_cast(const simd<U, Abi>& x)
2991 
2992 p1 Constraints:
2993  - is_simd_v<T> is true and
2994  - T::value_type is the same type as U
2995 
2996 p2 Returns:
2997  A simd object with the i^th element initialized to x[i] for all i in the
2998  range of [0, min(T::size(), simd_size_v<U, Abi>)). If T::size() is larger
2999  than simd_size_v<U, Abi>, the remaining elements are value-initialized.
3000 
3001 template <class T, class U, class Abi>
3002 T resizing_simd_cast(const simd_mask<U, Abi>& x)
3003 
3004 p1 Constraints: is_simd_mask_v<T> is true
3005 
3006 p2 Returns:
3007  A simd_mask object with the i^th element initialized to x[i] for all i in
3008 the range of [0, min(T::size(), simd_size_v<U, Abi>)). If T::size() is larger
3009  than simd_size_v<U, Abi>, the remaining elements are initialized to false.
3010 
3011  */
3012 
3013 template <typename _Tp, typename _Up, typename _Ap>
3014  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR enable_if_t<
3015  conjunction_v<is_simd<_Tp>, is_same<typename _Tp::value_type, _Up>>, _Tp>
3016  resizing_simd_cast(const simd<_Up, _Ap>& __x)
3017  {
3018  if constexpr (is_same_v<typename _Tp::abi_type, _Ap>)
3019  return __x;
3020  else if constexpr (simd_size_v<_Up, _Ap> == 1)
3021  {
3022  _Tp __r{};
3023  __r[0] = __x[0];
3024  return __r;
3025  }
3026  else if constexpr (_Tp::size() == 1)
3027  return __x[0];
3028  else if constexpr (sizeof(_Tp) == sizeof(__x)
3029  && !__is_fixed_size_abi_v<_Ap>)
3030  return {__private_init,
3031  __vector_bitcast<typename _Tp::value_type, _Tp::size()>(
3032  _Ap::_S_masked(__data(__x))._M_data)};
3033  else
3034  {
3035  _Tp __r{};
3036  __builtin_memcpy(&__data(__r), &__data(__x),
3037  sizeof(_Up)
3038  * std::min(_Tp::size(), simd_size_v<_Up, _Ap>));
3039  return __r;
3040  }
3041  }
3042 
3043 template <typename _Tp, typename _Up, typename _Ap>
3044  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3045  enable_if_t<is_simd_mask_v<_Tp>, _Tp>
3046  resizing_simd_cast(const simd_mask<_Up, _Ap>& __x)
3047  {
3048  return {__private_init, _Tp::abi_type::_MaskImpl::template _S_convert<
3049  typename _Tp::simd_type::value_type>(__x)};
3050  }
3051 } // namespace __proposed
3052 
3053 // }}}
3054 // to_fixed_size {{{2
3055 template <typename _Tp, int _Np>
3056  _GLIBCXX_SIMD_INTRINSIC fixed_size_simd<_Tp, _Np>
3057  to_fixed_size(const fixed_size_simd<_Tp, _Np>& __x)
3058  { return __x; }
3059 
3060 template <typename _Tp, int _Np>
3061  _GLIBCXX_SIMD_INTRINSIC fixed_size_simd_mask<_Tp, _Np>
3062  to_fixed_size(const fixed_size_simd_mask<_Tp, _Np>& __x)
3063  { return __x; }
3064 
3065 template <typename _Tp, typename _Ap>
3066  _GLIBCXX_SIMD_INTRINSIC auto
3067  to_fixed_size(const simd<_Tp, _Ap>& __x)
3068  {
3069  return simd<_Tp, simd_abi::fixed_size<simd_size_v<_Tp, _Ap>>>([&__x](
3070  auto __i) constexpr { return __x[__i]; });
3071  }
3072 
3073 template <typename _Tp, typename _Ap>
3074  _GLIBCXX_SIMD_INTRINSIC auto
3075  to_fixed_size(const simd_mask<_Tp, _Ap>& __x)
3076  {
3077  constexpr int _Np = simd_mask<_Tp, _Ap>::size();
3078  fixed_size_simd_mask<_Tp, _Np> __r;
3079  __execute_n_times<_Np>([&](auto __i) constexpr { __r[__i] = __x[__i]; });
3080  return __r;
3081  }
3082 
3083 // to_native {{{2
3084 template <typename _Tp, int _Np>
3085  _GLIBCXX_SIMD_INTRINSIC
3086  enable_if_t<(_Np == native_simd<_Tp>::size()), native_simd<_Tp>>
3087  to_native(const fixed_size_simd<_Tp, _Np>& __x)
3088  {
3089  alignas(memory_alignment_v<native_simd<_Tp>>) _Tp __mem[_Np];
3090  __x.copy_to(__mem, vector_aligned);
3091  return {__mem, vector_aligned};
3092  }
3093 
3094 template <typename _Tp, size_t _Np>
3095  _GLIBCXX_SIMD_INTRINSIC
3096  enable_if_t<(_Np == native_simd_mask<_Tp>::size()), native_simd_mask<_Tp>>
3097  to_native(const fixed_size_simd_mask<_Tp, _Np>& __x)
3098  {
3099  return native_simd_mask<_Tp>([&](auto __i) constexpr { return __x[__i]; });
3100  }
3101 
3102 // to_compatible {{{2
3103 template <typename _Tp, size_t _Np>
3104  _GLIBCXX_SIMD_INTRINSIC enable_if_t<(_Np == simd<_Tp>::size()), simd<_Tp>>
3105  to_compatible(const simd<_Tp, simd_abi::fixed_size<_Np>>& __x)
3106  {
3107  alignas(memory_alignment_v<simd<_Tp>>) _Tp __mem[_Np];
3108  __x.copy_to(__mem, vector_aligned);
3109  return {__mem, vector_aligned};
3110  }
3111 
3112 template <typename _Tp, size_t _Np>
3113  _GLIBCXX_SIMD_INTRINSIC
3114  enable_if_t<(_Np == simd_mask<_Tp>::size()), simd_mask<_Tp>>
3115  to_compatible(const simd_mask<_Tp, simd_abi::fixed_size<_Np>>& __x)
3116  { return simd_mask<_Tp>([&](auto __i) constexpr { return __x[__i]; }); }
3117 
3118 // masked assignment [simd_mask.where] {{{1
3119 
3120 // where_expression {{{1
3121 // const_where_expression<M, T> {{{2
3122 template <typename _M, typename _Tp>
3123  class const_where_expression
3124  {
3125  using _V = _Tp;
3126  static_assert(is_same_v<_V, __remove_cvref_t<_Tp>>);
3127 
3128  struct _Wrapper { using value_type = _V; };
3129 
3130  protected:
3131  using _Impl = typename _V::_Impl;
3132 
3133  using value_type =
3134  typename conditional_t<is_arithmetic_v<_V>, _Wrapper, _V>::value_type;
3135 
3136  _GLIBCXX_SIMD_INTRINSIC friend const _M&
3137  __get_mask(const const_where_expression& __x)
3138  { return __x._M_k; }
3139 
3140  _GLIBCXX_SIMD_INTRINSIC friend const _Tp&
3141  __get_lvalue(const const_where_expression& __x)
3142  { return __x._M_value; }
3143 
3144  const _M& _M_k;
3145  _Tp& _M_value;
3146 
3147  public:
3148  const_where_expression(const const_where_expression&) = delete;
3149  const_where_expression& operator=(const const_where_expression&) = delete;
3150 
3151  _GLIBCXX_SIMD_INTRINSIC const_where_expression(const _M& __kk, const _Tp& dd)
3152  : _M_k(__kk), _M_value(const_cast<_Tp&>(dd)) {}
3153 
3154  _GLIBCXX_SIMD_INTRINSIC _V
3155  operator-() const&&
3156  {
3157  return {__private_init,
3158  _Impl::template _S_masked_unary<negate>(__data(_M_k),
3159  __data(_M_value))};
3160  }
3161 
3162  template <typename _Up, typename _Flags>
3163  [[nodiscard]] _GLIBCXX_SIMD_INTRINSIC _V
3164  copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _Flags) const&&
3165  {
3166  return {__private_init,
3167  _Impl::_S_masked_load(__data(_M_value), __data(_M_k),
3168  _Flags::template _S_apply<_V>(__mem))};
3169  }
3170 
3171  template <typename _Up, typename _Flags>
3172  _GLIBCXX_SIMD_INTRINSIC void
3173  copy_to(_LoadStorePtr<_Up, value_type>* __mem, _Flags) const&&
3174  {
3175  _Impl::_S_masked_store(__data(_M_value),
3176  _Flags::template _S_apply<_V>(__mem),
3177  __data(_M_k));
3178  }
3179  };
3180 
3181 // const_where_expression<bool, T> {{{2
3182 template <typename _Tp>
3183  class const_where_expression<bool, _Tp>
3184  {
3185  using _M = bool;
3186  using _V = _Tp;
3187 
3188  static_assert(is_same_v<_V, __remove_cvref_t<_Tp>>);
3189 
3190  struct _Wrapper { using value_type = _V; };
3191 
3192  protected:
3193  using value_type =
3194  typename conditional_t<is_arithmetic_v<_V>, _Wrapper, _V>::value_type;
3195 
3196  _GLIBCXX_SIMD_INTRINSIC friend const _M&
3197  __get_mask(const const_where_expression& __x)
3198  { return __x._M_k; }
3199 
3200  _GLIBCXX_SIMD_INTRINSIC friend const _Tp&
3201  __get_lvalue(const const_where_expression& __x)
3202  { return __x._M_value; }
3203 
3204  const bool _M_k;
3205  _Tp& _M_value;
3206 
3207  public:
3208  const_where_expression(const const_where_expression&) = delete;
3209  const_where_expression& operator=(const const_where_expression&) = delete;
3210 
3211  _GLIBCXX_SIMD_INTRINSIC const_where_expression(const bool __kk, const _Tp& dd)
3212  : _M_k(__kk), _M_value(const_cast<_Tp&>(dd)) {}
3213 
3214  _GLIBCXX_SIMD_INTRINSIC _V operator-() const&&
3215  { return _M_k ? -_M_value : _M_value; }
3216 
3217  template <typename _Up, typename _Flags>
3218  [[nodiscard]] _GLIBCXX_SIMD_INTRINSIC _V
3219  copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _Flags) const&&
3220  { return _M_k ? static_cast<_V>(__mem[0]) : _M_value; }
3221 
3222  template <typename _Up, typename _Flags>
3223  _GLIBCXX_SIMD_INTRINSIC void
3224  copy_to(_LoadStorePtr<_Up, value_type>* __mem, _Flags) const&&
3225  {
3226  if (_M_k)
3227  __mem[0] = _M_value;
3228  }
3229  };
3230 
3231 // where_expression<M, T> {{{2
3232 template <typename _M, typename _Tp>
3233  class where_expression : public const_where_expression<_M, _Tp>
3234  {
3235  using _Impl = typename const_where_expression<_M, _Tp>::_Impl;
3236 
3237  static_assert(!is_const<_Tp>::value,
3238  "where_expression may only be instantiated with __a non-const "
3239  "_Tp parameter");
3240 
3241  using typename const_where_expression<_M, _Tp>::value_type;
3242  using const_where_expression<_M, _Tp>::_M_k;
3243  using const_where_expression<_M, _Tp>::_M_value;
3244 
3245  static_assert(
3246  is_same<typename _M::abi_type, typename _Tp::abi_type>::value, "");
3247  static_assert(_M::size() == _Tp::size(), "");
3248 
3249  _GLIBCXX_SIMD_INTRINSIC friend _Tp& __get_lvalue(where_expression& __x)
3250  { return __x._M_value; }
3251 
3252  public:
3253  where_expression(const where_expression&) = delete;
3254  where_expression& operator=(const where_expression&) = delete;
3255 
3256  _GLIBCXX_SIMD_INTRINSIC where_expression(const _M& __kk, _Tp& dd)
3257  : const_where_expression<_M, _Tp>(__kk, dd) {}
3258 
3259  template <typename _Up>
3260  _GLIBCXX_SIMD_INTRINSIC void operator=(_Up&& __x) &&
3261  {
3262  _Impl::_S_masked_assign(__data(_M_k), __data(_M_value),
3263  __to_value_type_or_member_type<_Tp>(
3264  static_cast<_Up&&>(__x)));
3265  }
3266 
3267 #define _GLIBCXX_SIMD_OP_(__op, __name) \
3268  template <typename _Up> \
3269  _GLIBCXX_SIMD_INTRINSIC void operator __op##=(_Up&& __x)&& \
3270  { \
3271  _Impl::template _S_masked_cassign( \
3272  __data(_M_k), __data(_M_value), \
3273  __to_value_type_or_member_type<_Tp>(static_cast<_Up&&>(__x)), \
3274  [](auto __impl, auto __lhs, auto __rhs) constexpr { \
3275  return __impl.__name(__lhs, __rhs); \
3276  }); \
3277  } \
3278  static_assert(true)
3279  _GLIBCXX_SIMD_OP_(+, _S_plus);
3280  _GLIBCXX_SIMD_OP_(-, _S_minus);
3281  _GLIBCXX_SIMD_OP_(*, _S_multiplies);
3282  _GLIBCXX_SIMD_OP_(/, _S_divides);
3283  _GLIBCXX_SIMD_OP_(%, _S_modulus);
3284  _GLIBCXX_SIMD_OP_(&, _S_bit_and);
3285  _GLIBCXX_SIMD_OP_(|, _S_bit_or);
3286  _GLIBCXX_SIMD_OP_(^, _S_bit_xor);
3287  _GLIBCXX_SIMD_OP_(<<, _S_shift_left);
3288  _GLIBCXX_SIMD_OP_(>>, _S_shift_right);
3289 #undef _GLIBCXX_SIMD_OP_
3290 
3291  _GLIBCXX_SIMD_INTRINSIC void operator++() &&
3292  {
3293  __data(_M_value)
3294  = _Impl::template _S_masked_unary<__increment>(__data(_M_k),
3295  __data(_M_value));
3296  }
3297 
3298  _GLIBCXX_SIMD_INTRINSIC void operator++(int) &&
3299  {
3300  __data(_M_value)
3301  = _Impl::template _S_masked_unary<__increment>(__data(_M_k),
3302  __data(_M_value));
3303  }
3304 
3305  _GLIBCXX_SIMD_INTRINSIC void operator--() &&
3306  {
3307  __data(_M_value)
3308  = _Impl::template _S_masked_unary<__decrement>(__data(_M_k),
3309  __data(_M_value));
3310  }
3311 
3312  _GLIBCXX_SIMD_INTRINSIC void operator--(int) &&
3313  {
3314  __data(_M_value)
3315  = _Impl::template _S_masked_unary<__decrement>(__data(_M_k),
3316  __data(_M_value));
3317  }
3318 
3319  // intentionally hides const_where_expression::copy_from
3320  template <typename _Up, typename _Flags>
3321  _GLIBCXX_SIMD_INTRINSIC void
3322  copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _Flags) &&
3323  {
3324  __data(_M_value)
3325  = _Impl::_S_masked_load(__data(_M_value), __data(_M_k),
3326  _Flags::template _S_apply<_Tp>(__mem));
3327  }
3328  };
3329 
3330 // where_expression<bool, T> {{{2
3331 template <typename _Tp>
3332  class where_expression<bool, _Tp> : public const_where_expression<bool, _Tp>
3333  {
3334  using _M = bool;
3335  using typename const_where_expression<_M, _Tp>::value_type;
3336  using const_where_expression<_M, _Tp>::_M_k;
3337  using const_where_expression<_M, _Tp>::_M_value;
3338 
3339  public:
3340  where_expression(const where_expression&) = delete;
3341  where_expression& operator=(const where_expression&) = delete;
3342 
3343  _GLIBCXX_SIMD_INTRINSIC where_expression(const _M& __kk, _Tp& dd)
3344  : const_where_expression<_M, _Tp>(__kk, dd) {}
3345 
3346 #define _GLIBCXX_SIMD_OP_(__op) \
3347  template <typename _Up> \
3348  _GLIBCXX_SIMD_INTRINSIC void operator __op(_Up&& __x)&& \
3349  { if (_M_k) _M_value __op static_cast<_Up&&>(__x); }
3350 
3351  _GLIBCXX_SIMD_OP_(=)
3352  _GLIBCXX_SIMD_OP_(+=)
3353  _GLIBCXX_SIMD_OP_(-=)
3354  _GLIBCXX_SIMD_OP_(*=)
3355  _GLIBCXX_SIMD_OP_(/=)
3356  _GLIBCXX_SIMD_OP_(%=)
3357  _GLIBCXX_SIMD_OP_(&=)
3358  _GLIBCXX_SIMD_OP_(|=)
3359  _GLIBCXX_SIMD_OP_(^=)
3360  _GLIBCXX_SIMD_OP_(<<=)
3361  _GLIBCXX_SIMD_OP_(>>=)
3362  #undef _GLIBCXX_SIMD_OP_
3363 
3364  _GLIBCXX_SIMD_INTRINSIC void operator++() &&
3365  { if (_M_k) ++_M_value; }
3366 
3367  _GLIBCXX_SIMD_INTRINSIC void operator++(int) &&
3368  { if (_M_k) ++_M_value; }
3369 
3370  _GLIBCXX_SIMD_INTRINSIC void operator--() &&
3371  { if (_M_k) --_M_value; }
3372 
3373  _GLIBCXX_SIMD_INTRINSIC void operator--(int) &&
3374  { if (_M_k) --_M_value; }
3375 
3376  // intentionally hides const_where_expression::copy_from
3377  template <typename _Up, typename _Flags>
3378  _GLIBCXX_SIMD_INTRINSIC void
3379  copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _Flags) &&
3380  { if (_M_k) _M_value = __mem[0]; }
3381  };
3382 
3383 // where {{{1
3384 template <typename _Tp, typename _Ap>
3385  _GLIBCXX_SIMD_INTRINSIC where_expression<simd_mask<_Tp, _Ap>, simd<_Tp, _Ap>>
3386  where(const typename simd<_Tp, _Ap>::mask_type& __k, simd<_Tp, _Ap>& __value)
3387  { return {__k, __value}; }
3388 
3389 template <typename _Tp, typename _Ap>
3390  _GLIBCXX_SIMD_INTRINSIC
3391  const_where_expression<simd_mask<_Tp, _Ap>, simd<_Tp, _Ap>>
3392  where(const typename simd<_Tp, _Ap>::mask_type& __k,
3393  const simd<_Tp, _Ap>& __value)
3394  { return {__k, __value}; }
3395 
3396 template <typename _Tp, typename _Ap>
3397  _GLIBCXX_SIMD_INTRINSIC
3398  where_expression<simd_mask<_Tp, _Ap>, simd_mask<_Tp, _Ap>>
3399  where(const remove_const_t<simd_mask<_Tp, _Ap>>& __k,
3400  simd_mask<_Tp, _Ap>& __value)
3401  { return {__k, __value}; }
3402 
3403 template <typename _Tp, typename _Ap>
3404  _GLIBCXX_SIMD_INTRINSIC
3405  const_where_expression<simd_mask<_Tp, _Ap>, simd_mask<_Tp, _Ap>>
3406  where(const remove_const_t<simd_mask<_Tp, _Ap>>& __k,
3407  const simd_mask<_Tp, _Ap>& __value)
3408  { return {__k, __value}; }
3409 
3410 template <typename _Tp>
3411  _GLIBCXX_SIMD_INTRINSIC where_expression<bool, _Tp>
3412  where(_ExactBool __k, _Tp& __value)
3413  { return {__k, __value}; }
3414 
3415 template <typename _Tp>
3416  _GLIBCXX_SIMD_INTRINSIC const_where_expression<bool, _Tp>
3417  where(_ExactBool __k, const _Tp& __value)
3418  { return {__k, __value}; }
3419 
3420  template <typename _Tp, typename _Ap>
3421  void where(bool __k, simd<_Tp, _Ap>& __value) = delete;
3422 
3423  template <typename _Tp, typename _Ap>
3424  void where(bool __k, const simd<_Tp, _Ap>& __value) = delete;
3425 
3426 // proposed mask iterations {{{1
3427 namespace __proposed {
3428 template <size_t _Np>
3429  class where_range
3430  {
3431  const bitset<_Np> __bits;
3432 
3433  public:
3434  where_range(bitset<_Np> __b) : __bits(__b) {}
3435 
3436  class iterator
3437  {
3438  size_t __mask;
3439  size_t __bit;
3440 
3441  _GLIBCXX_SIMD_INTRINSIC void __next_bit()
3442  { __bit = __builtin_ctzl(__mask); }
3443 
3444  _GLIBCXX_SIMD_INTRINSIC void __reset_lsb()
3445  {
3446  // 01100100 - 1 = 01100011
3447  __mask &= (__mask - 1);
3448  // __asm__("btr %1,%0" : "+r"(__mask) : "r"(__bit));
3449  }
3450 
3451  public:
3452  iterator(decltype(__mask) __m) : __mask(__m) { __next_bit(); }
3453  iterator(const iterator&) = default;
3454  iterator(iterator&&) = default;
3455 
3456  _GLIBCXX_SIMD_ALWAYS_INLINE size_t operator->() const
3457  { return __bit; }
3458 
3459  _GLIBCXX_SIMD_ALWAYS_INLINE size_t operator*() const
3460  { return __bit; }
3461 
3462  _GLIBCXX_SIMD_ALWAYS_INLINE iterator& operator++()
3463  {
3464  __reset_lsb();
3465  __next_bit();
3466  return *this;
3467  }
3468 
3469  _GLIBCXX_SIMD_ALWAYS_INLINE iterator operator++(int)
3470  {
3471  iterator __tmp = *this;
3472  __reset_lsb();
3473  __next_bit();
3474  return __tmp;
3475  }
3476 
3477  _GLIBCXX_SIMD_ALWAYS_INLINE bool operator==(const iterator& __rhs) const
3478  { return __mask == __rhs.__mask; }
3479 
3480  _GLIBCXX_SIMD_ALWAYS_INLINE bool operator!=(const iterator& __rhs) const
3481  { return __mask != __rhs.__mask; }
3482  };
3483 
3484  iterator begin() const
3485  { return __bits.to_ullong(); }
3486 
3487  iterator end() const
3488  { return 0; }
3489  };
3490 
3491 template <typename _Tp, typename _Ap>
3492  where_range<simd_size_v<_Tp, _Ap>>
3493  where(const simd_mask<_Tp, _Ap>& __k)
3494  { return __k.__to_bitset(); }
3495 
3496 } // namespace __proposed
3497 
3498 // }}}1
3499 // reductions [simd.reductions] {{{1
3500 template <typename _Tp, typename _Abi, typename _BinaryOperation = plus<>>
3501  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3502  reduce(const simd<_Tp, _Abi>& __v,
3503  _BinaryOperation __binary_op = _BinaryOperation())
3504  { return _Abi::_SimdImpl::_S_reduce(__v, __binary_op); }
3505 
3506 template <typename _M, typename _V, typename _BinaryOperation = plus<>>
3507  _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3508  reduce(const const_where_expression<_M, _V>& __x,
3509  typename _V::value_type __identity_element,
3510  _BinaryOperation __binary_op)
3511  {
3512  if (__builtin_expect(none_of(__get_mask(__x)), false))
3513  return __identity_element;
3514 
3515  _V __tmp = __identity_element;
3516  _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3517  __data(__get_lvalue(__x)));
3518  return reduce(__tmp, __binary_op);
3519  }
3520 
3521 template <typename _M, typename _V>
3522  _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3523  reduce(const const_where_expression<_M, _V>& __x, plus<> __binary_op = {})
3524  { return reduce(__x, 0, __binary_op); }
3525 
3526 template <typename _M, typename _V>
3527  _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3528  reduce(const const_where_expression<_M, _V>& __x, multiplies<> __binary_op)
3529  { return reduce(__x, 1, __binary_op); }
3530 
3531 template <typename _M, typename _V>
3532  _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3533  reduce(const const_where_expression<_M, _V>& __x, bit_and<> __binary_op)
3534  { return reduce(__x, ~typename _V::value_type(), __binary_op); }
3535 
3536 template <typename _M, typename _V>
3537  _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3538  reduce(const const_where_expression<_M, _V>& __x, bit_or<> __binary_op)
3539  { return reduce(__x, 0, __binary_op); }
3540 
3541 template <typename _M, typename _V>
3542  _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3543  reduce(const const_where_expression<_M, _V>& __x, bit_xor<> __binary_op)
3544  { return reduce(__x, 0, __binary_op); }
3545 
3546 template <typename _Tp, typename _Abi>
3547  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3548  hmin(const simd<_Tp, _Abi>& __v) noexcept
3549  {
3550  return _Abi::_SimdImpl::_S_reduce(__v, __detail::_Minimum());
3551  }
3552 
3553 template <typename _Tp, typename _Abi>
3554  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3555  hmax(const simd<_Tp, _Abi>& __v) noexcept
3556  {
3557  return _Abi::_SimdImpl::_S_reduce(__v, __detail::_Maximum());
3558  }
3559 
3560 template <typename _M, typename _V>
3561  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3562  typename _V::value_type
3563  hmin(const const_where_expression<_M, _V>& __x) noexcept
3564  {
3565  using _Tp = typename _V::value_type;
3566  constexpr _Tp __id_elem =
3567 #ifdef __FINITE_MATH_ONLY__
3568  __finite_max_v<_Tp>;
3569 #else
3570  __value_or<__infinity, _Tp>(__finite_max_v<_Tp>);
3571 #endif
3572  _V __tmp = __id_elem;
3573  _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3574  __data(__get_lvalue(__x)));
3575  return _V::abi_type::_SimdImpl::_S_reduce(__tmp, __detail::_Minimum());
3576  }
3577 
3578 template <typename _M, typename _V>
3579  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3580  typename _V::value_type
3581  hmax(const const_where_expression<_M, _V>& __x) noexcept
3582  {
3583  using _Tp = typename _V::value_type;
3584  constexpr _Tp __id_elem =
3585 #ifdef __FINITE_MATH_ONLY__
3586  __finite_min_v<_Tp>;
3587 #else
3588  [] {
3589  if constexpr (__value_exists_v<__infinity, _Tp>)
3590  return -__infinity_v<_Tp>;
3591  else
3592  return __finite_min_v<_Tp>;
3593  }();
3594 #endif
3595  _V __tmp = __id_elem;
3596  _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3597  __data(__get_lvalue(__x)));
3598  return _V::abi_type::_SimdImpl::_S_reduce(__tmp, __detail::_Maximum());
3599  }
3600 
3601 // }}}1
3602 // algorithms [simd.alg] {{{
3603 template <typename _Tp, typename _Ap>
3604  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3605  min(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
3606  { return {__private_init, _Ap::_SimdImpl::_S_min(__data(__a), __data(__b))}; }
3607 
3608 template <typename _Tp, typename _Ap>
3609  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3610  max(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
3611  { return {__private_init, _Ap::_SimdImpl::_S_max(__data(__a), __data(__b))}; }
3612 
3613 template <typename _Tp, typename _Ap>
3614  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3615  pair<simd<_Tp, _Ap>, simd<_Tp, _Ap>>
3616  minmax(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
3617  {
3618  const auto pair_of_members
3619  = _Ap::_SimdImpl::_S_minmax(__data(__a), __data(__b));
3620  return {simd<_Tp, _Ap>(__private_init, pair_of_members.first),
3621  simd<_Tp, _Ap>(__private_init, pair_of_members.second)};
3622  }
3623 
3624 template <typename _Tp, typename _Ap>
3625  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3626  clamp(const simd<_Tp, _Ap>& __v, const simd<_Tp, _Ap>& __lo,
3627  const simd<_Tp, _Ap>& __hi)
3628  {
3629  using _Impl = typename _Ap::_SimdImpl;
3630  return {__private_init,
3631  _Impl::_S_min(__data(__hi),
3632  _Impl::_S_max(__data(__lo), __data(__v)))};
3633  }
3634 
3635 // }}}
3636 
3637 template <size_t... _Sizes, typename _Tp, typename _Ap,
3638  typename = enable_if_t<((_Sizes + ...) == simd<_Tp, _Ap>::size())>>
3639  inline tuple<simd<_Tp, simd_abi::deduce_t<_Tp, _Sizes>>...>
3640  split(const simd<_Tp, _Ap>&);
3641 
3642 // __extract_part {{{
3643 template <int _Index, int _Total, int _Combine = 1, typename _Tp, size_t _Np>
3644  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_CONST
3645  _SimdWrapper<_Tp, _Np / _Total * _Combine>
3646  __extract_part(const _SimdWrapper<_Tp, _Np> __x);
3647 
3648 template <int Index, int Parts, int _Combine = 1, typename _Tp, typename _A0,
3649  typename... _As>
3650  _GLIBCXX_SIMD_INTRINSIC auto
3651  __extract_part(const _SimdTuple<_Tp, _A0, _As...>& __x);
3652 
3653 // }}}
3654 // _SizeList {{{
3655 template <size_t _V0, size_t... _Values>
3656  struct _SizeList
3657  {
3658  template <size_t _I>
3659  static constexpr size_t _S_at(_SizeConstant<_I> = {})
3660  {
3661  if constexpr (_I == 0)
3662  return _V0;
3663  else
3664  return _SizeList<_Values...>::template _S_at<_I - 1>();
3665  }
3666 
3667  template <size_t _I>
3668  static constexpr auto _S_before(_SizeConstant<_I> = {})
3669  {
3670  if constexpr (_I == 0)
3671  return _SizeConstant<0>();
3672  else
3673  return _SizeConstant<
3674  _V0 + _SizeList<_Values...>::template _S_before<_I - 1>()>();
3675  }
3676 
3677  template <size_t _Np>
3678  static constexpr auto _S_pop_front(_SizeConstant<_Np> = {})
3679  {
3680  if constexpr (_Np == 0)
3681  return _SizeList();
3682  else
3683  return _SizeList<_Values...>::template _S_pop_front<_Np - 1>();
3684  }
3685  };
3686 
3687 // }}}
3688 // __extract_center {{{
3689 template <typename _Tp, size_t _Np>
3690  _GLIBCXX_SIMD_INTRINSIC _SimdWrapper<_Tp, _Np / 2>
3691  __extract_center(_SimdWrapper<_Tp, _Np> __x)
3692  {
3693  static_assert(_Np >= 4);
3694  static_assert(_Np % 4 == 0); // x0 - x1 - x2 - x3 -> return {x1, x2}
3695 #if _GLIBCXX_SIMD_X86INTRIN // {{{
3696  if constexpr (__have_avx512f && sizeof(_Tp) * _Np == 64)
3697  {
3698  const auto __intrin = __to_intrin(__x);
3699  if constexpr (is_integral_v<_Tp>)
3700  return __vector_bitcast<_Tp>(_mm512_castsi512_si256(
3701  _mm512_shuffle_i32x4(__intrin, __intrin,
3702  1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
3703  else if constexpr (sizeof(_Tp) == 4)
3704  return __vector_bitcast<_Tp>(_mm512_castps512_ps256(
3705  _mm512_shuffle_f32x4(__intrin, __intrin,
3706  1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
3707  else if constexpr (sizeof(_Tp) == 8)
3708  return __vector_bitcast<_Tp>(_mm512_castpd512_pd256(
3709  _mm512_shuffle_f64x2(__intrin, __intrin,
3710  1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
3711  else
3712  __assert_unreachable<_Tp>();
3713  }
3714  else if constexpr (sizeof(_Tp) * _Np == 32 && is_floating_point_v<_Tp>)
3715  return __vector_bitcast<_Tp>(
3716  _mm_shuffle_pd(__lo128(__vector_bitcast<double>(__x)),
3717  __hi128(__vector_bitcast<double>(__x)), 1));
3718  else if constexpr (sizeof(__x) == 32 && sizeof(_Tp) * _Np <= 32)
3719  return __vector_bitcast<_Tp>(
3720  _mm_alignr_epi8(__hi128(__vector_bitcast<_LLong>(__x)),
3721  __lo128(__vector_bitcast<_LLong>(__x)),
3722  sizeof(_Tp) * _Np / 4));
3723  else
3724 #endif // _GLIBCXX_SIMD_X86INTRIN }}}
3725  {
3726  __vector_type_t<_Tp, _Np / 2> __r;
3727  __builtin_memcpy(&__r,
3728  reinterpret_cast<const char*>(&__x)
3729  + sizeof(_Tp) * _Np / 4,
3730  sizeof(_Tp) * _Np / 2);
3731  return __r;
3732  }
3733  }
3734 
3735 template <typename _Tp, typename _A0, typename... _As>
3736  _GLIBCXX_SIMD_INTRINSIC
3737  _SimdWrapper<_Tp, _SimdTuple<_Tp, _A0, _As...>::_S_size() / 2>
3738  __extract_center(const _SimdTuple<_Tp, _A0, _As...>& __x)
3739  {
3740  if constexpr (sizeof...(_As) == 0)
3741  return __extract_center(__x.first);
3742  else
3743  return __extract_part<1, 4, 2>(__x);
3744  }
3745 
3746 // }}}
3747 // __split_wrapper {{{
3748 template <size_t... _Sizes, typename _Tp, typename... _As>
3749  auto
3750  __split_wrapper(_SizeList<_Sizes...>, const _SimdTuple<_Tp, _As...>& __x)
3751  {
3752  return split<_Sizes...>(
3753  fixed_size_simd<_Tp, _SimdTuple<_Tp, _As...>::_S_size()>(__private_init,
3754  __x));
3755  }
3756 
3757 // }}}
3758 
3759 // split<simd>(simd) {{{
3760 template <typename _V, typename _Ap,
3761  size_t Parts = simd_size_v<typename _V::value_type, _Ap> / _V::size()>
3762  enable_if_t<simd_size_v<typename _V::value_type, _Ap> == Parts * _V::size()
3763  && is_simd_v<_V>, array<_V, Parts>>
3764  split(const simd<typename _V::value_type, _Ap>& __x)
3765  {
3766  using _Tp = typename _V::value_type;
3767  if constexpr (Parts == 1)
3768  {
3769  return {simd_cast<_V>(__x)};
3770  }
3771  else if (__x._M_is_constprop())
3772  {
3773  return __generate_from_n_evaluations<Parts, array<_V, Parts>>([&](
3774  auto __i) constexpr {
3775  return _V([&](auto __j) constexpr {
3776  return __x[__i * _V::size() + __j];
3777  });
3778  });
3779  }
3780  else if constexpr (
3781  __is_fixed_size_abi_v<_Ap>
3782  && (is_same_v<typename _V::abi_type, simd_abi::scalar>
3783  || (__is_fixed_size_abi_v<typename _V::abi_type>
3784  && sizeof(_V) == sizeof(_Tp) * _V::size() // _V doesn't have padding
3785  )))
3786  {
3787  // fixed_size -> fixed_size (w/o padding) or scalar
3788 #ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
3789  const __may_alias<_Tp>* const __element_ptr
3790  = reinterpret_cast<const __may_alias<_Tp>*>(&__data(__x));
3791  return __generate_from_n_evaluations<Parts, array<_V, Parts>>([&](
3792  auto __i) constexpr {
3793  return _V(__element_ptr + __i * _V::size(), vector_aligned);
3794  });
3795 #else
3796  const auto& __xx = __data(__x);
3797  return __generate_from_n_evaluations<Parts, array<_V, Parts>>([&](
3798  auto __i) constexpr {
3799  [[maybe_unused]] constexpr size_t __offset
3800  = decltype(__i)::value * _V::size();
3801  return _V([&](auto __j) constexpr {
3802  constexpr _SizeConstant<__j + __offset> __k;
3803  return __xx[__k];
3804  });
3805  });
3806 #endif
3807  }
3808  else if constexpr (is_same_v<typename _V::abi_type, simd_abi::scalar>)
3809  {
3810  // normally memcpy should work here as well
3811  return __generate_from_n_evaluations<Parts, array<_V, Parts>>([&](
3812  auto __i) constexpr { return __x[__i]; });
3813  }
3814  else
3815  {
3816  return __generate_from_n_evaluations<Parts, array<_V, Parts>>([&](
3817  auto __i) constexpr {
3818  if constexpr (__is_fixed_size_abi_v<typename _V::abi_type>)
3819  return _V([&](auto __j) constexpr {
3820  return __x[__i * _V::size() + __j];
3821  });
3822  else
3823  return _V(__private_init,
3824  __extract_part<decltype(__i)::value, Parts>(__data(__x)));
3825  });
3826  }
3827  }
3828 
3829 // }}}
3830 // split<simd_mask>(simd_mask) {{{
3831 template <typename _V, typename _Ap,
3832  size_t _Parts
3833  = simd_size_v<typename _V::simd_type::value_type, _Ap> / _V::size()>
3834  enable_if_t<is_simd_mask_v<_V> && simd_size_v<typename
3835  _V::simd_type::value_type, _Ap> == _Parts * _V::size(), array<_V, _Parts>>
3836  split(const simd_mask<typename _V::simd_type::value_type, _Ap>& __x)
3837  {
3838  if constexpr (is_same_v<_Ap, typename _V::abi_type>)
3839  return {__x};
3840  else if constexpr (_Parts == 1)
3841  return {__proposed::static_simd_cast<_V>(__x)};
3842  else if constexpr (_Parts == 2 && __is_sse_abi<typename _V::abi_type>()
3843  && __is_avx_abi<_Ap>())
3844  return {_V(__private_init, __lo128(__data(__x))),
3845  _V(__private_init, __hi128(__data(__x)))};
3846  else if constexpr (_V::size() <= __CHAR_BIT__ * sizeof(_ULLong))
3847  {
3848  const bitset __bits = __x.__to_bitset();
3849  return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>([&](
3850  auto __i) constexpr {
3851  constexpr size_t __offset = __i * _V::size();
3852  return _V(__bitset_init, (__bits >> __offset).to_ullong());
3853  });
3854  }
3855  else
3856  {
3857  return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>([&](
3858  auto __i) constexpr {
3859  constexpr size_t __offset = __i * _V::size();
3860  return _V(
3861  __private_init, [&](auto __j) constexpr {
3862  return __x[__j + __offset];
3863  });
3864  });
3865  }
3866  }
3867 
3868 // }}}
3869 // split<_Sizes...>(simd) {{{
3870 template <size_t... _Sizes, typename _Tp, typename _Ap, typename>
3871  _GLIBCXX_SIMD_ALWAYS_INLINE
3872  tuple<simd<_Tp, simd_abi::deduce_t<_Tp, _Sizes>>...>
3873  split(const simd<_Tp, _Ap>& __x)
3874  {
3875  using _SL = _SizeList<_Sizes...>;
3876  using _Tuple = tuple<__deduced_simd<_Tp, _Sizes>...>;
3877  constexpr size_t _Np = simd_size_v<_Tp, _Ap>;
3878  constexpr size_t _N0 = _SL::template _S_at<0>();
3879  using _V = __deduced_simd<_Tp, _N0>;
3880 
3881  if (__x._M_is_constprop())
3882  return __generate_from_n_evaluations<sizeof...(_Sizes), _Tuple>([&](
3883  auto __i) constexpr {
3884  using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
3885  constexpr size_t __offset = _SL::_S_before(__i);
3886  return _Vi([&](auto __j) constexpr { return __x[__offset + __j]; });
3887  });
3888  else if constexpr (_Np == _N0)
3889  {
3890  static_assert(sizeof...(_Sizes) == 1);
3891  return {simd_cast<_V>(__x)};
3892  }
3893  else if constexpr // split from fixed_size, such that __x::first.size == _N0
3894  (__is_fixed_size_abi_v<
3895  _Ap> && __fixed_size_storage_t<_Tp, _Np>::_S_first_size == _N0)
3896  {
3897  static_assert(
3898  !__is_fixed_size_abi_v<typename _V::abi_type>,
3899  "How can <_Tp, _Np> be __a single _SimdTuple entry but __a "
3900  "fixed_size_simd "
3901  "when deduced?");
3902  // extract first and recurse (__split_wrapper is needed to deduce a new
3903  // _Sizes pack)
3904  return tuple_cat(make_tuple(_V(__private_init, __data(__x).first)),
3905  __split_wrapper(_SL::template _S_pop_front<1>(),
3906  __data(__x).second));
3907  }
3908  else if constexpr ((!is_same_v<simd_abi::scalar,
3909  simd_abi::deduce_t<_Tp, _Sizes>> && ...)
3910  && (!__is_fixed_size_abi_v<
3911  simd_abi::deduce_t<_Tp, _Sizes>> && ...))
3912  {
3913  if constexpr (((_Sizes * 2 == _Np) && ...))
3914  return {{__private_init, __extract_part<0, 2>(__data(__x))},
3915  {__private_init, __extract_part<1, 2>(__data(__x))}};
3916  else if constexpr (is_same_v<_SizeList<_Sizes...>,
3917  _SizeList<_Np / 3, _Np / 3, _Np / 3>>)
3918  return {{__private_init, __extract_part<0, 3>(__data(__x))},
3919  {__private_init, __extract_part<1, 3>(__data(__x))},
3920  {__private_init, __extract_part<2, 3>(__data(__x))}};
3921  else if constexpr (is_same_v<_SizeList<_Sizes...>,
3922  _SizeList<2 * _Np / 3, _Np / 3>>)
3923  return {{__private_init, __extract_part<0, 3, 2>(__data(__x))},
3924  {__private_init, __extract_part<2, 3>(__data(__x))}};
3925  else if constexpr (is_same_v<_SizeList<_Sizes...>,
3926  _SizeList<_Np / 3, 2 * _Np / 3>>)
3927  return {{__private_init, __extract_part<0, 3>(__data(__x))},
3928  {__private_init, __extract_part<1, 3, 2>(__data(__x))}};
3929  else if constexpr (is_same_v<_SizeList<_Sizes...>,
3930  _SizeList<_Np / 2, _Np / 4, _Np / 4>>)
3931  return {{__private_init, __extract_part<0, 2>(__data(__x))},
3932  {__private_init, __extract_part<2, 4>(__data(__x))},
3933  {__private_init, __extract_part<3, 4>(__data(__x))}};
3934  else if constexpr (is_same_v<_SizeList<_Sizes...>,
3935  _SizeList<_Np / 4, _Np / 4, _Np / 2>>)
3936  return {{__private_init, __extract_part<0, 4>(__data(__x))},
3937  {__private_init, __extract_part<1, 4>(__data(__x))},
3938  {__private_init, __extract_part<1, 2>(__data(__x))}};
3939  else if constexpr (is_same_v<_SizeList<_Sizes...>,
3940  _SizeList<_Np / 4, _Np / 2, _Np / 4>>)
3941  return {{__private_init, __extract_part<0, 4>(__data(__x))},
3942  {__private_init, __extract_center(__data(__x))},
3943  {__private_init, __extract_part<3, 4>(__data(__x))}};
3944  else if constexpr (((_Sizes * 4 == _Np) && ...))
3945  return {{__private_init, __extract_part<0, 4>(__data(__x))},
3946  {__private_init, __extract_part<1, 4>(__data(__x))},
3947  {__private_init, __extract_part<2, 4>(__data(__x))},
3948  {__private_init, __extract_part<3, 4>(__data(__x))}};
3949  // else fall through
3950  }
3951 #ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
3952  const __may_alias<_Tp>* const __element_ptr
3953  = reinterpret_cast<const __may_alias<_Tp>*>(&__x);
3954  return __generate_from_n_evaluations<sizeof...(_Sizes), _Tuple>([&](
3955  auto __i) constexpr {
3956  using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
3957  constexpr size_t __offset = _SL::_S_before(__i);
3958  constexpr size_t __base_align = alignof(simd<_Tp, _Ap>);
3959  constexpr size_t __a
3960  = __base_align - ((__offset * sizeof(_Tp)) % __base_align);
3961  constexpr size_t __b = ((__a - 1) & __a) ^ __a;
3962  constexpr size_t __alignment = __b == 0 ? __a : __b;
3963  return _Vi(__element_ptr + __offset, overaligned<__alignment>);
3964  });
3965 #else
3966  return __generate_from_n_evaluations<sizeof...(_Sizes), _Tuple>([&](
3967  auto __i) constexpr {
3968  using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
3969  const auto& __xx = __data(__x);
3970  using _Offset = decltype(_SL::_S_before(__i));
3971  return _Vi([&](auto __j) constexpr {
3972  constexpr _SizeConstant<_Offset::value + __j> __k;
3973  return __xx[__k];
3974  });
3975  });
3976 #endif
3977  }
3978 
3979 // }}}
3980 
3981 // __subscript_in_pack {{{
3982 template <size_t _I, typename _Tp, typename _Ap, typename... _As>
3983  _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
3984  __subscript_in_pack(const simd<_Tp, _Ap>& __x, const simd<_Tp, _As>&... __xs)
3985  {
3986  if constexpr (_I < simd_size_v<_Tp, _Ap>)
3987  return __x[_I];
3988  else
3989  return __subscript_in_pack<_I - simd_size_v<_Tp, _Ap>>(__xs...);
3990  }
3991 
3992 // }}}
3993 // __store_pack_of_simd {{{
3994 template <typename _Tp, typename _A0, typename... _As>
3995  _GLIBCXX_SIMD_INTRINSIC void
3996  __store_pack_of_simd(char* __mem, const simd<_Tp, _A0>& __x0,
3997  const simd<_Tp, _As>&... __xs)
3998  {
3999  constexpr size_t __n_bytes = sizeof(_Tp) * simd_size_v<_Tp, _A0>;
4000  __builtin_memcpy(__mem, &__data(__x0), __n_bytes);
4001  if constexpr (sizeof...(__xs) > 0)
4002  __store_pack_of_simd(__mem + __n_bytes, __xs...);
4003  }
4004 
4005 // }}}
4006 // concat(simd...) {{{
4007 template <typename _Tp, typename... _As>
4008  inline _GLIBCXX_SIMD_CONSTEXPR
4009  simd<_Tp, simd_abi::deduce_t<_Tp, (simd_size_v<_Tp, _As> + ...)>>
4010  concat(const simd<_Tp, _As>&... __xs)
4011  {
4012  using _Rp = __deduced_simd<_Tp, (simd_size_v<_Tp, _As> + ...)>;
4013  if constexpr (sizeof...(__xs) == 1)
4014  return simd_cast<_Rp>(__xs...);
4015  else if ((... && __xs._M_is_constprop()))
4016  return simd<_Tp,
4017  simd_abi::deduce_t<_Tp, (simd_size_v<_Tp, _As> + ...)>>([&](
4018  auto __i) constexpr { return __subscript_in_pack<__i>(__xs...); });
4019  else
4020  {
4021  _Rp __r{};
4022  __store_pack_of_simd(reinterpret_cast<char*>(&__data(__r)), __xs...);
4023  return __r;
4024  }
4025  }
4026 
4027 // }}}
4028 // concat(array<simd>) {{{
4029 template <typename _Tp, typename _Abi, size_t _Np>
4030  _GLIBCXX_SIMD_ALWAYS_INLINE
4031  _GLIBCXX_SIMD_CONSTEXPR __deduced_simd<_Tp, simd_size_v<_Tp, _Abi> * _Np>
4032  concat(const array<simd<_Tp, _Abi>, _Np>& __x)
4033  {
4034  return __call_with_subscripts<_Np>(__x, [](const auto&... __xs) {
4035  return concat(__xs...);
4036  });
4037  }
4038 
4039 // }}}
4040 
4041 /// @cond undocumented
4042 // _SmartReference {{{
4043 template <typename _Up, typename _Accessor = _Up,
4044  typename _ValueType = typename _Up::value_type>
4045  class _SmartReference
4046  {
4047  friend _Accessor;
4048  int _M_index;
4049  _Up& _M_obj;
4050 
4051  _GLIBCXX_SIMD_INTRINSIC constexpr _ValueType _M_read() const noexcept
4052  {
4053  if constexpr (is_arithmetic_v<_Up>)
4054  return _M_obj;
4055  else
4056  return _M_obj[_M_index];
4057  }
4058 
4059  template <typename _Tp>
4060  _GLIBCXX_SIMD_INTRINSIC constexpr void _M_write(_Tp&& __x) const
4061  { _Accessor::_S_set(_M_obj, _M_index, static_cast<_Tp&&>(__x)); }
4062 
4063  public:
4064  _GLIBCXX_SIMD_INTRINSIC constexpr
4065  _SmartReference(_Up& __o, int __i) noexcept
4066  : _M_index(__i), _M_obj(__o) {}
4067 
4068  using value_type = _ValueType;
4069 
4070  _GLIBCXX_SIMD_INTRINSIC _SmartReference(const _SmartReference&) = delete;
4071 
4072  _GLIBCXX_SIMD_INTRINSIC constexpr operator value_type() const noexcept
4073  { return _M_read(); }
4074 
4075  template <typename _Tp,
4076  typename
4077  = _ValuePreservingOrInt<__remove_cvref_t<_Tp>, value_type>>
4078  _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference operator=(_Tp&& __x) &&
4079  {
4080  _M_write(static_cast<_Tp&&>(__x));
4081  return {_M_obj, _M_index};
4082  }
4083 
4084 #define _GLIBCXX_SIMD_OP_(__op) \
4085  template <typename _Tp, \
4086  typename _TT \
4087  = decltype(declval<value_type>() __op declval<_Tp>()), \
4088  typename = _ValuePreservingOrInt<__remove_cvref_t<_Tp>, _TT>, \
4089  typename = _ValuePreservingOrInt<_TT, value_type>> \
4090  _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference \
4091  operator __op##=(_Tp&& __x) && \
4092  { \
4093  const value_type& __lhs = _M_read(); \
4094  _M_write(__lhs __op __x); \
4095  return {_M_obj, _M_index}; \
4096  }
4097  _GLIBCXX_SIMD_ALL_ARITHMETICS(_GLIBCXX_SIMD_OP_);
4098  _GLIBCXX_SIMD_ALL_SHIFTS(_GLIBCXX_SIMD_OP_);
4099  _GLIBCXX_SIMD_ALL_BINARY(_GLIBCXX_SIMD_OP_);
4100 #undef _GLIBCXX_SIMD_OP_
4101 
4102  template <typename _Tp = void,
4103  typename
4104  = decltype(++declval<conditional_t<true, value_type, _Tp>&>())>
4105  _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference operator++() &&
4106  {
4107  value_type __x = _M_read();
4108  _M_write(++__x);
4109  return {_M_obj, _M_index};
4110  }
4111 
4112  template <typename _Tp = void,
4113  typename
4114  = decltype(declval<conditional_t<true, value_type, _Tp>&>()++)>
4115  _GLIBCXX_SIMD_INTRINSIC constexpr value_type operator++(int) &&
4116  {
4117  const value_type __r = _M_read();
4118  value_type __x = __r;
4119  _M_write(++__x);
4120  return __r;
4121  }
4122 
4123  template <typename _Tp = void,
4124  typename
4125  = decltype(--declval<conditional_t<true, value_type, _Tp>&>())>
4126  _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference operator--() &&
4127  {
4128  value_type __x = _M_read();
4129  _M_write(--__x);
4130  return {_M_obj, _M_index};
4131  }
4132 
4133  template <typename _Tp = void,
4134  typename
4135  = decltype(declval<conditional_t<true, value_type, _Tp>&>()--)>
4136  _GLIBCXX_SIMD_INTRINSIC constexpr value_type operator--(int) &&
4137  {
4138  const value_type __r = _M_read();
4139  value_type __x = __r;
4140  _M_write(--__x);
4141  return __r;
4142  }
4143 
4144  _GLIBCXX_SIMD_INTRINSIC friend void
4145  swap(_SmartReference&& __a, _SmartReference&& __b) noexcept(
4146  conjunction<
4147  is_nothrow_constructible<value_type, _SmartReference&&>,
4148  is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4149  {
4150  value_type __tmp = static_cast<_SmartReference&&>(__a);
4151  static_cast<_SmartReference&&>(__a) = static_cast<value_type>(__b);
4152  static_cast<_SmartReference&&>(__b) = std::move(__tmp);
4153  }
4154 
4155  _GLIBCXX_SIMD_INTRINSIC friend void
4156  swap(value_type& __a, _SmartReference&& __b) noexcept(
4157  conjunction<
4158  is_nothrow_constructible<value_type, value_type&&>,
4159  is_nothrow_assignable<value_type&, value_type&&>,
4160  is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4161  {
4162  value_type __tmp(std::move(__a));
4163  __a = static_cast<value_type>(__b);
4164  static_cast<_SmartReference&&>(__b) = std::move(__tmp);
4165  }
4166 
4167  _GLIBCXX_SIMD_INTRINSIC friend void
4168  swap(_SmartReference&& __a, value_type& __b) noexcept(
4169  conjunction<
4170  is_nothrow_constructible<value_type, _SmartReference&&>,
4171  is_nothrow_assignable<value_type&, value_type&&>,
4172  is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4173  {
4174  value_type __tmp(__a);
4175  static_cast<_SmartReference&&>(__a) = std::move(__b);
4176  __b = std::move(__tmp);
4177  }
4178  };
4179 
4180 // }}}
4181 // __scalar_abi_wrapper {{{
4182 template <int _Bytes>
4183  struct __scalar_abi_wrapper
4184  {
4185  template <typename _Tp> static constexpr size_t _S_full_size = 1;
4186  template <typename _Tp> static constexpr size_t _S_size = 1;
4187  template <typename _Tp> static constexpr size_t _S_is_partial = false;
4188 
4189  template <typename _Tp, typename _Abi = simd_abi::scalar>
4190  static constexpr bool _S_is_valid_v
4191  = _Abi::template _IsValid<_Tp>::value && sizeof(_Tp) == _Bytes;
4192  };
4193 
4194 // }}}
4195 // __decay_abi metafunction {{{
4196 template <typename _Tp>
4197  struct __decay_abi { using type = _Tp; };
4198 
4199 template <int _Bytes>
4200  struct __decay_abi<__scalar_abi_wrapper<_Bytes>>
4201  { using type = simd_abi::scalar; };
4202 
4203 // }}}
4204 // __find_next_valid_abi metafunction {{{1
4205 // Given an ABI tag A<N>, find an N2 < N such that A<N2>::_S_is_valid_v<_Tp> ==
4206 // true, N2 is a power-of-2, and A<N2>::_S_is_partial<_Tp> is false. Break
4207 // recursion at 2 elements in the resulting ABI tag. In this case
4208 // type::_S_is_valid_v<_Tp> may be false.
4209 template <template <int> class _Abi, int _Bytes, typename _Tp>
4210  struct __find_next_valid_abi
4211  {
4212  static constexpr auto _S_choose()
4213  {
4214  constexpr int _NextBytes = std::__bit_ceil(_Bytes) / 2;
4215  using _NextAbi = _Abi<_NextBytes>;
4216  if constexpr (_NextBytes < sizeof(_Tp) * 2) // break recursion
4217  return _Abi<_Bytes>();
4218  else if constexpr (_NextAbi::template _S_is_partial<_Tp> == false
4219  && _NextAbi::template _S_is_valid_v<_Tp>)
4220  return _NextAbi();
4221  else
4222  return __find_next_valid_abi<_Abi, _NextBytes, _Tp>::_S_choose();
4223  }
4224 
4225  using type = decltype(_S_choose());
4226  };
4227 
4228 template <int _Bytes, typename _Tp>
4229  struct __find_next_valid_abi<__scalar_abi_wrapper, _Bytes, _Tp>
4230  { using type = simd_abi::scalar; };
4231 
4232 // _AbiList {{{1
4233 template <template <int> class...>
4234  struct _AbiList
4235  {
4236  template <typename, int> static constexpr bool _S_has_valid_abi = false;
4237  template <typename, int> using _FirstValidAbi = void;
4238  template <typename, int> using _BestAbi = void;
4239  };
4240 
4241 template <template <int> class _A0, template <int> class... _Rest>
4242  struct _AbiList<_A0, _Rest...>
4243  {
4244  template <typename _Tp, int _Np>
4245  static constexpr bool _S_has_valid_abi
4246  = _A0<sizeof(_Tp) * _Np>::template _S_is_valid_v<
4247  _Tp> || _AbiList<_Rest...>::template _S_has_valid_abi<_Tp, _Np>;
4248 
4249  template <typename _Tp, int _Np>
4250  using _FirstValidAbi = conditional_t<
4251  _A0<sizeof(_Tp) * _Np>::template _S_is_valid_v<_Tp>,
4252  typename __decay_abi<_A0<sizeof(_Tp) * _Np>>::type,
4253  typename _AbiList<_Rest...>::template _FirstValidAbi<_Tp, _Np>>;
4254 
4255  template <typename _Tp, int _Np>
4256  static constexpr auto _S_determine_best_abi()
4257  {
4258  static_assert(_Np >= 1);
4259  constexpr int _Bytes = sizeof(_Tp) * _Np;
4260  if constexpr (_Np == 1)
4261  return __make_dependent_t<_Tp, simd_abi::scalar>{};
4262  else
4263  {
4264  constexpr int __fullsize = _A0<_Bytes>::template _S_full_size<_Tp>;
4265  // _A0<_Bytes> is good if:
4266  // 1. The ABI tag is valid for _Tp
4267  // 2. The storage overhead is no more than padding to fill the next
4268  // power-of-2 number of bytes
4269  if constexpr (_A0<_Bytes>::template _S_is_valid_v<
4270  _Tp> && __fullsize / 2 < _Np)
4271  return typename __decay_abi<_A0<_Bytes>>::type{};
4272  else
4273  {
4274  using _Bp =
4275  typename __find_next_valid_abi<_A0, _Bytes, _Tp>::type;
4276  if constexpr (_Bp::template _S_is_valid_v<
4277  _Tp> && _Bp::template _S_size<_Tp> <= _Np)
4278  return _Bp{};
4279  else
4280  return
4281  typename _AbiList<_Rest...>::template _BestAbi<_Tp, _Np>{};
4282  }
4283  }
4284  }
4285 
4286  template <typename _Tp, int _Np>
4287  using _BestAbi = decltype(_S_determine_best_abi<_Tp, _Np>());
4288  };
4289 
4290 // }}}1
4291 
4292 // the following lists all native ABIs, which makes them accessible to
4293 // simd_abi::deduce and select_best_vector_type_t (for fixed_size). Order
4294 // matters: Whatever comes first has higher priority.
4295 using _AllNativeAbis = _AbiList<simd_abi::_VecBltnBtmsk, simd_abi::_VecBuiltin,
4296  __scalar_abi_wrapper>;
4297 
4298 // valid _SimdTraits specialization {{{1
4299 template <typename _Tp, typename _Abi>
4300  struct _SimdTraits<_Tp, _Abi, void_t<typename _Abi::template _IsValid<_Tp>>>
4301  : _Abi::template __traits<_Tp> {};
4302 
4303 // __deduce_impl specializations {{{1
4304 // try all native ABIs (including scalar) first
4305 template <typename _Tp, size_t _Np>
4306  struct __deduce_impl<
4307  _Tp, _Np, enable_if_t<_AllNativeAbis::template _S_has_valid_abi<_Tp, _Np>>>
4308  { using type = _AllNativeAbis::_FirstValidAbi<_Tp, _Np>; };
4309 
4310 // fall back to fixed_size only if scalar and native ABIs don't match
4311 template <typename _Tp, size_t _Np, typename = void>
4312  struct __deduce_fixed_size_fallback {};
4313 
4314 template <typename _Tp, size_t _Np>
4315  struct __deduce_fixed_size_fallback<_Tp, _Np,
4316  enable_if_t<simd_abi::fixed_size<_Np>::template _S_is_valid_v<_Tp>>>
4317  { using type = simd_abi::fixed_size<_Np>; };
4318 
4319 template <typename _Tp, size_t _Np, typename>
4320  struct __deduce_impl : public __deduce_fixed_size_fallback<_Tp, _Np> {};
4321 
4322 //}}}1
4323 /// @endcond
4324 
4325 // simd_mask {{{
4326 template <typename _Tp, typename _Abi>
4327  class simd_mask : public _SimdTraits<_Tp, _Abi>::_MaskBase
4328  {
4329  // types, tags, and friends {{{
4330  using _Traits = _SimdTraits<_Tp, _Abi>;
4331  using _MemberType = typename _Traits::_MaskMember;
4332 
4333  // We map all masks with equal element sizeof to a single integer type, the
4334  // one given by __int_for_sizeof_t<_Tp>. This is the approach
4335  // [[gnu::vector_size(N)]] types take as well and it reduces the number of
4336  // template specializations in the implementation classes.
4337  using _Ip = __int_for_sizeof_t<_Tp>;
4338  static constexpr _Ip* _S_type_tag = nullptr;
4339 
4340  friend typename _Traits::_MaskBase;
4341  friend class simd<_Tp, _Abi>; // to construct masks on return
4342  friend typename _Traits::_SimdImpl; // to construct masks on return and
4343  // inspect data on masked operations
4344  public:
4345  using _Impl = typename _Traits::_MaskImpl;
4346  friend _Impl;
4347 
4348  // }}}
4349  // member types {{{
4350  using value_type = bool;
4351  using reference = _SmartReference<_MemberType, _Impl, value_type>;
4352  using simd_type = simd<_Tp, _Abi>;
4353  using abi_type = _Abi;
4354 
4355  // }}}
4356  static constexpr size_t size() // {{{
4357  { return __size_or_zero_v<_Tp, _Abi>; }
4358 
4359  // }}}
4360  // constructors & assignment {{{
4361  simd_mask() = default;
4362  simd_mask(const simd_mask&) = default;
4363  simd_mask(simd_mask&&) = default;
4364  simd_mask& operator=(const simd_mask&) = default;
4365  simd_mask& operator=(simd_mask&&) = default;
4366 
4367  // }}}
4368  // access to internal representation (optional feature) {{{
4369  _GLIBCXX_SIMD_ALWAYS_INLINE explicit
4370  simd_mask(typename _Traits::_MaskCastType __init)
4371  : _M_data{__init} {}
4372  // conversions to internal type is done in _MaskBase
4373 
4374  // }}}
4375  // bitset interface (extension to be proposed) {{{
4376  // TS_FEEDBACK:
4377  // Conversion of simd_mask to and from bitset makes it much easier to
4378  // interface with other facilities. I suggest adding `static
4379  // simd_mask::from_bitset` and `simd_mask::to_bitset`.
4380  _GLIBCXX_SIMD_ALWAYS_INLINE static simd_mask
4381  __from_bitset(bitset<size()> bs)
4382  { return {__bitset_init, bs}; }
4383 
4384  _GLIBCXX_SIMD_ALWAYS_INLINE bitset<size()>
4385  __to_bitset() const
4386  { return _Impl::_S_to_bits(_M_data)._M_to_bitset(); }
4387 
4388  // }}}
4389  // explicit broadcast constructor {{{
4390  _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR
4391  simd_mask(value_type __x)
4392  : _M_data(_Impl::template _S_broadcast<_Ip>(__x)) {}
4393 
4394  // }}}
4395  // implicit type conversion constructor {{{
4396  #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4397  // proposed improvement
4398  template <typename _Up, typename _A2,
4399  typename = enable_if_t<simd_size_v<_Up, _A2> == size()>>
4400  _GLIBCXX_SIMD_ALWAYS_INLINE explicit(sizeof(_MemberType)
4401  != sizeof(typename _SimdTraits<_Up, _A2>::_MaskMember))
4402  simd_mask(const simd_mask<_Up, _A2>& __x)
4403  : simd_mask(__proposed::static_simd_cast<simd_mask>(__x)) {}
4404  #else
4405  // conforming to ISO/IEC 19570:2018
4406  template <typename _Up, typename = enable_if_t<conjunction<
4407  is_same<abi_type, simd_abi::fixed_size<size()>>,
4408  is_same<_Up, _Up>>::value>>
4409  _GLIBCXX_SIMD_ALWAYS_INLINE
4410  simd_mask(const simd_mask<_Up, simd_abi::fixed_size<size()>>& __x)
4411  : _M_data(_Impl::_S_from_bitmask(__data(__x), _S_type_tag)) {}
4412  #endif
4413 
4414  // }}}
4415  // load constructor {{{
4416  template <typename _Flags>
4417  _GLIBCXX_SIMD_ALWAYS_INLINE
4418  simd_mask(const value_type* __mem, _Flags)
4419  : _M_data(_Impl::template _S_load<_Ip>(
4420  _Flags::template _S_apply<simd_mask>(__mem))) {}
4421 
4422  template <typename _Flags>
4423  _GLIBCXX_SIMD_ALWAYS_INLINE
4424  simd_mask(const value_type* __mem, simd_mask __k, _Flags)
4425  : _M_data{}
4426  {
4427  _M_data
4428  = _Impl::_S_masked_load(_M_data, __k._M_data,
4429  _Flags::template _S_apply<simd_mask>(__mem));
4430  }
4431 
4432  // }}}
4433  // loads [simd_mask.load] {{{
4434  template <typename _Flags>
4435  _GLIBCXX_SIMD_ALWAYS_INLINE void
4436  copy_from(const value_type* __mem, _Flags)
4437  {
4438  _M_data = _Impl::template _S_load<_Ip>(
4439  _Flags::template _S_apply<simd_mask>(__mem));
4440  }
4441 
4442  // }}}
4443  // stores [simd_mask.store] {{{
4444  template <typename _Flags>
4445  _GLIBCXX_SIMD_ALWAYS_INLINE void
4446  copy_to(value_type* __mem, _Flags) const
4447  { _Impl::_S_store(_M_data, _Flags::template _S_apply<simd_mask>(__mem)); }
4448 
4449  // }}}
4450  // scalar access {{{
4451  _GLIBCXX_SIMD_ALWAYS_INLINE reference
4452  operator[](size_t __i)
4453  {
4454  if (__i >= size())
4455  __invoke_ub("Subscript %d is out of range [0, %d]", __i, size() - 1);
4456  return {_M_data, int(__i)};
4457  }
4458 
4459  _GLIBCXX_SIMD_ALWAYS_INLINE value_type
4460  operator[](size_t __i) const
4461  {
4462  if (__i >= size())
4463  __invoke_ub("Subscript %d is out of range [0, %d]", __i, size() - 1);
4464  if constexpr (__is_scalar_abi<_Abi>())
4465  return _M_data;
4466  else
4467  return static_cast<bool>(_M_data[__i]);
4468  }
4469 
4470  // }}}
4471  // negation {{{
4472  _GLIBCXX_SIMD_ALWAYS_INLINE simd_mask
4473  operator!() const
4474  { return {__private_init, _Impl::_S_bit_not(_M_data)}; }
4475 
4476  // }}}
4477  // simd_mask binary operators [simd_mask.binary] {{{
4478  #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4479  // simd_mask<int> && simd_mask<uint> needs disambiguation
4480  template <typename _Up, typename _A2,
4481  typename
4482  = enable_if_t<is_convertible_v<simd_mask<_Up, _A2>, simd_mask>>>
4483  _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask
4484  operator&&(const simd_mask& __x, const simd_mask<_Up, _A2>& __y)
4485  {
4486  return {__private_init,
4487  _Impl::_S_logical_and(__x._M_data, simd_mask(__y)._M_data)};
4488  }
4489 
4490  template <typename _Up, typename _A2,
4491  typename
4492  = enable_if_t<is_convertible_v<simd_mask<_Up, _A2>, simd_mask>>>
4493  _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask
4494  operator||(const simd_mask& __x, const simd_mask<_Up, _A2>& __y)
4495  {
4496  return {__private_init,
4497  _Impl::_S_logical_or(__x._M_data, simd_mask(__y)._M_data)};
4498  }
4499  #endif // _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4500 
4501  _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask
4502  operator&&(const simd_mask& __x, const simd_mask& __y)
4503  {
4504  return {__private_init, _Impl::_S_logical_and(__x._M_data, __y._M_data)};
4505  }
4506 
4507  _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask
4508  operator||(const simd_mask& __x, const simd_mask& __y)
4509  {
4510  return {__private_init, _Impl::_S_logical_or(__x._M_data, __y._M_data)};
4511  }
4512 
4513  _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask
4514  operator&(const simd_mask& __x, const simd_mask& __y)
4515  { return {__private_init, _Impl::_S_bit_and(__x._M_data, __y._M_data)}; }
4516 
4517  _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask
4518  operator|(const simd_mask& __x, const simd_mask& __y)
4519  { return {__private_init, _Impl::_S_bit_or(__x._M_data, __y._M_data)}; }
4520 
4521  _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask
4522  operator^(const simd_mask& __x, const simd_mask& __y)
4523  { return {__private_init, _Impl::_S_bit_xor(__x._M_data, __y._M_data)}; }
4524 
4525  _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask&
4526  operator&=(simd_mask& __x, const simd_mask& __y)
4527  {
4528  __x._M_data = _Impl::_S_bit_and(__x._M_data, __y._M_data);
4529  return __x;
4530  }
4531 
4532  _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask&
4533  operator|=(simd_mask& __x, const simd_mask& __y)
4534  {
4535  __x._M_data = _Impl::_S_bit_or(__x._M_data, __y._M_data);
4536  return __x;
4537  }
4538 
4539  _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask&
4540  operator^=(simd_mask& __x, const simd_mask& __y)
4541  {
4542  __x._M_data = _Impl::_S_bit_xor(__x._M_data, __y._M_data);
4543  return __x;
4544  }
4545 
4546  // }}}
4547  // simd_mask compares [simd_mask.comparison] {{{
4548  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4549  operator==(const simd_mask& __x, const simd_mask& __y)
4550  { return !operator!=(__x, __y); }
4551 
4552  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4553  operator!=(const simd_mask& __x, const simd_mask& __y)
4554  { return {__private_init, _Impl::_S_bit_xor(__x._M_data, __y._M_data)}; }
4555 
4556  // }}}
4557  // private_init ctor {{{
4558  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
4559  simd_mask(_PrivateInit, typename _Traits::_MaskMember __init)
4560  : _M_data(__init) {}
4561 
4562  // }}}
4563  // private_init generator ctor {{{
4564  template <typename _Fp, typename = decltype(bool(declval<_Fp>()(size_t())))>
4565  _GLIBCXX_SIMD_INTRINSIC constexpr
4566  simd_mask(_PrivateInit, _Fp&& __gen)
4567  : _M_data()
4568  {
4569  __execute_n_times<size()>([&](auto __i) constexpr {
4570  _Impl::_S_set(_M_data, __i, __gen(__i));
4571  });
4572  }
4573 
4574  // }}}
4575  // bitset_init ctor {{{
4576  _GLIBCXX_SIMD_INTRINSIC simd_mask(_BitsetInit, bitset<size()> __init)
4577  : _M_data(
4578  _Impl::_S_from_bitmask(_SanitizedBitMask<size()>(__init), _S_type_tag))
4579  {}
4580 
4581  // }}}
4582  // __cvt {{{
4583  // TS_FEEDBACK:
4584  // The conversion operator this implements should be a ctor on simd_mask.
4585  // Once you call .__cvt() on a simd_mask it converts conveniently.
4586  // A useful variation: add `explicit(sizeof(_Tp) != sizeof(_Up))`
4587  struct _CvtProxy
4588  {
4589  template <typename _Up, typename _A2,
4590  typename
4591  = enable_if_t<simd_size_v<_Up, _A2> == simd_size_v<_Tp, _Abi>>>
4592  operator simd_mask<_Up, _A2>() &&
4593  {
4594  using namespace std::experimental::__proposed;
4595  return static_simd_cast<simd_mask<_Up, _A2>>(_M_data);
4596  }
4597 
4598  const simd_mask<_Tp, _Abi>& _M_data;
4599  };
4600 
4601  _GLIBCXX_SIMD_INTRINSIC _CvtProxy
4602  __cvt() const
4603  { return {*this}; }
4604 
4605  // }}}
4606  // operator?: overloads (suggested extension) {{{
4607  #ifdef __GXX_CONDITIONAL_IS_OVERLOADABLE__
4608  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4609  operator?:(const simd_mask& __k, const simd_mask& __where_true,
4610  const simd_mask& __where_false)
4611  {
4612  auto __ret = __where_false;
4613  _Impl::_S_masked_assign(__k._M_data, __ret._M_data, __where_true._M_data);
4614  return __ret;
4615  }
4616 
4617  template <typename _U1, typename _U2,
4618  typename _Rp = simd<common_type_t<_U1, _U2>, _Abi>,
4619  typename = enable_if_t<conjunction_v<
4620  is_convertible<_U1, _Rp>, is_convertible<_U2, _Rp>,
4621  is_convertible<simd_mask, typename _Rp::mask_type>>>>
4622  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend _Rp
4623  operator?:(const simd_mask& __k, const _U1& __where_true,
4624  const _U2& __where_false)
4625  {
4626  _Rp __ret = __where_false;
4627  _Rp::_Impl::_S_masked_assign(
4628  __data(static_cast<typename _Rp::mask_type>(__k)), __data(__ret),
4629  __data(static_cast<_Rp>(__where_true)));
4630  return __ret;
4631  }
4632 
4633  #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4634  template <typename _Kp, typename _Ak, typename _Up, typename _Au,
4635  typename = enable_if_t<
4636  conjunction_v<is_convertible<simd_mask<_Kp, _Ak>, simd_mask>,
4637  is_convertible<simd_mask<_Up, _Au>, simd_mask>>>>
4638  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4639  operator?:(const simd_mask<_Kp, _Ak>& __k, const simd_mask& __where_true,
4640  const simd_mask<_Up, _Au>& __where_false)
4641  {
4642  simd_mask __ret = __where_false;
4643  _Impl::_S_masked_assign(simd_mask(__k)._M_data, __ret._M_data,
4644  __where_true._M_data);
4645  return __ret;
4646  }
4647  #endif // _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4648  #endif // __GXX_CONDITIONAL_IS_OVERLOADABLE__
4649 
4650  // }}}
4651  // _M_is_constprop {{{
4652  _GLIBCXX_SIMD_INTRINSIC constexpr bool
4653  _M_is_constprop() const
4654  {
4655  if constexpr (__is_scalar_abi<_Abi>())
4656  return __builtin_constant_p(_M_data);
4657  else
4658  return _M_data._M_is_constprop();
4659  }
4660 
4661  // }}}
4662 
4663  private:
4664  friend const auto& __data<_Tp, abi_type>(const simd_mask&);
4665  friend auto& __data<_Tp, abi_type>(simd_mask&);
4666  alignas(_Traits::_S_mask_align) _MemberType _M_data;
4667  };
4668 
4669 // }}}
4670 
4671 /// @cond undocumented
4672 // __data(simd_mask) {{{
4673 template <typename _Tp, typename _Ap>
4674  _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
4675  __data(const simd_mask<_Tp, _Ap>& __x)
4676  { return __x._M_data; }
4677 
4678 template <typename _Tp, typename _Ap>
4679  _GLIBCXX_SIMD_INTRINSIC constexpr auto&
4680  __data(simd_mask<_Tp, _Ap>& __x)
4681  { return __x._M_data; }
4682 
4683 // }}}
4684 /// @endcond
4685 
4686 // simd_mask reductions [simd_mask.reductions] {{{
4687 template <typename _Tp, typename _Abi>
4688  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4689  all_of(const simd_mask<_Tp, _Abi>& __k) noexcept
4690  {
4691  if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4692  {
4693  for (size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
4694  if (!__k[__i])
4695  return false;
4696  return true;
4697  }
4698  else
4699  return _Abi::_MaskImpl::_S_all_of(__k);
4700  }
4701 
4702 template <typename _Tp, typename _Abi>
4703  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4704  any_of(const simd_mask<_Tp, _Abi>& __k) noexcept
4705  {
4706  if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4707  {
4708  for (size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
4709  if (__k[__i])
4710  return true;
4711  return false;
4712  }
4713  else
4714  return _Abi::_MaskImpl::_S_any_of(__k);
4715  }
4716 
4717 template <typename _Tp, typename _Abi>
4718  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4719  none_of(const simd_mask<_Tp, _Abi>& __k) noexcept
4720  {
4721  if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4722  {
4723  for (size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
4724  if (__k[__i])
4725  return false;
4726  return true;
4727  }
4728  else
4729  return _Abi::_MaskImpl::_S_none_of(__k);
4730  }
4731 
4732 template <typename _Tp, typename _Abi>
4733  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4734  some_of(const simd_mask<_Tp, _Abi>& __k) noexcept
4735  {
4736  if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4737  {
4738  for (size_t __i = 1; __i < simd_size_v<_Tp, _Abi>; ++__i)
4739  if (__k[__i] != __k[__i - 1])
4740  return true;
4741  return false;
4742  }
4743  else
4744  return _Abi::_MaskImpl::_S_some_of(__k);
4745  }
4746 
4747 template <typename _Tp, typename _Abi>
4748  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4749  popcount(const simd_mask<_Tp, _Abi>& __k) noexcept
4750  {
4751  if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4752  {
4753  const int __r = __call_with_subscripts<simd_size_v<_Tp, _Abi>>(
4754  __k, [](auto... __elements) { return ((__elements != 0) + ...); });
4755  if (__builtin_is_constant_evaluated() || __builtin_constant_p(__r))
4756  return __r;
4757  }
4758  return _Abi::_MaskImpl::_S_popcount(__k);
4759  }
4760 
4761 template <typename _Tp, typename _Abi>
4762  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4763  find_first_set(const simd_mask<_Tp, _Abi>& __k)
4764  {
4765  if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4766  {
4767  constexpr size_t _Np = simd_size_v<_Tp, _Abi>;
4768  const size_t _Idx = __call_with_n_evaluations<_Np>(
4769  [](auto... __indexes) { return std::min({__indexes...}); },
4770  [&](auto __i) { return __k[__i] ? +__i : _Np; });
4771  if (_Idx >= _Np)
4772  __invoke_ub("find_first_set(empty mask) is UB");
4773  if (__builtin_constant_p(_Idx))
4774  return _Idx;
4775  }
4776  return _Abi::_MaskImpl::_S_find_first_set(__k);
4777  }
4778 
4779 template <typename _Tp, typename _Abi>
4780  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4781  find_last_set(const simd_mask<_Tp, _Abi>& __k)
4782  {
4783  if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4784  {
4785  constexpr size_t _Np = simd_size_v<_Tp, _Abi>;
4786  const int _Idx = __call_with_n_evaluations<_Np>(
4787  [](auto... __indexes) { return std::max({__indexes...}); },
4788  [&](auto __i) { return __k[__i] ? int(__i) : -1; });
4789  if (_Idx < 0)
4790  __invoke_ub("find_first_set(empty mask) is UB");
4791  if (__builtin_constant_p(_Idx))
4792  return _Idx;
4793  }
4794  return _Abi::_MaskImpl::_S_find_last_set(__k);
4795  }
4796 
4797 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4798 all_of(_ExactBool __x) noexcept
4799 { return __x; }
4800 
4801 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4802 any_of(_ExactBool __x) noexcept
4803 { return __x; }
4804 
4805 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4806 none_of(_ExactBool __x) noexcept
4807 { return !__x; }
4808 
4809 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4810 some_of(_ExactBool) noexcept
4811 { return false; }
4812 
4813 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4814 popcount(_ExactBool __x) noexcept
4815 { return __x; }
4816 
4817 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4818 find_first_set(_ExactBool)
4819 { return 0; }
4820 
4821 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4822 find_last_set(_ExactBool)
4823 { return 0; }
4824 
4825 // }}}
4826 
4827 /// @cond undocumented
4828 // _SimdIntOperators{{{1
4829 template <typename _V, typename _Impl, bool>
4830  class _SimdIntOperators {};
4831 
4832 template <typename _V, typename _Impl>
4833  class _SimdIntOperators<_V, _Impl, true>
4834  {
4835  _GLIBCXX_SIMD_INTRINSIC const _V& __derived() const
4836  { return *static_cast<const _V*>(this); }
4837 
4838  template <typename _Tp>
4839  _GLIBCXX_SIMD_INTRINSIC static _GLIBCXX_SIMD_CONSTEXPR _V
4840  _S_make_derived(_Tp&& __d)
4841  { return {__private_init, static_cast<_Tp&&>(__d)}; }
4842 
4843  public:
4844  _GLIBCXX_SIMD_CONSTEXPR friend _V& operator%=(_V& __lhs, const _V& __x)
4845  { return __lhs = __lhs % __x; }
4846 
4847  _GLIBCXX_SIMD_CONSTEXPR friend _V& operator&=(_V& __lhs, const _V& __x)
4848  { return __lhs = __lhs & __x; }
4849 
4850  _GLIBCXX_SIMD_CONSTEXPR friend _V& operator|=(_V& __lhs, const _V& __x)
4851  { return __lhs = __lhs | __x; }
4852 
4853  _GLIBCXX_SIMD_CONSTEXPR friend _V& operator^=(_V& __lhs, const _V& __x)
4854  { return __lhs = __lhs ^ __x; }
4855 
4856  _GLIBCXX_SIMD_CONSTEXPR friend _V& operator<<=(_V& __lhs, const _V& __x)
4857  { return __lhs = __lhs << __x; }
4858 
4859  _GLIBCXX_SIMD_CONSTEXPR friend _V& operator>>=(_V& __lhs, const _V& __x)
4860  { return __lhs = __lhs >> __x; }
4861 
4862  _GLIBCXX_SIMD_CONSTEXPR friend _V& operator<<=(_V& __lhs, int __x)
4863  { return __lhs = __lhs << __x; }
4864 
4865  _GLIBCXX_SIMD_CONSTEXPR friend _V& operator>>=(_V& __lhs, int __x)
4866  { return __lhs = __lhs >> __x; }
4867 
4868  _GLIBCXX_SIMD_CONSTEXPR friend _V operator%(const _V& __x, const _V& __y)
4869  {
4870  return _SimdIntOperators::_S_make_derived(
4871  _Impl::_S_modulus(__data(__x), __data(__y)));
4872  }
4873 
4874  _GLIBCXX_SIMD_CONSTEXPR friend _V operator&(const _V& __x, const _V& __y)
4875  {
4876  return _SimdIntOperators::_S_make_derived(
4877  _Impl::_S_bit_and(__data(__x), __data(__y)));
4878  }
4879 
4880  _GLIBCXX_SIMD_CONSTEXPR friend _V operator|(const _V& __x, const _V& __y)
4881  {
4882  return _SimdIntOperators::_S_make_derived(
4883  _Impl::_S_bit_or(__data(__x), __data(__y)));
4884  }
4885 
4886  _GLIBCXX_SIMD_CONSTEXPR friend _V operator^(const _V& __x, const _V& __y)
4887  {
4888  return _SimdIntOperators::_S_make_derived(
4889  _Impl::_S_bit_xor(__data(__x), __data(__y)));
4890  }
4891 
4892  _GLIBCXX_SIMD_CONSTEXPR friend _V operator<<(const _V& __x, const _V& __y)
4893  {
4894  return _SimdIntOperators::_S_make_derived(
4895  _Impl::_S_bit_shift_left(__data(__x), __data(__y)));
4896  }
4897 
4898  _GLIBCXX_SIMD_CONSTEXPR friend _V operator>>(const _V& __x, const _V& __y)
4899  {
4900  return _SimdIntOperators::_S_make_derived(
4901  _Impl::_S_bit_shift_right(__data(__x), __data(__y)));
4902  }
4903 
4904  template <typename _VV = _V>
4905  _GLIBCXX_SIMD_CONSTEXPR friend _V operator<<(const _V& __x, int __y)
4906  {
4907  using _Tp = typename _VV::value_type;
4908  if (__y < 0)
4909  __invoke_ub("The behavior is undefined if the right operand of a "
4910  "shift operation is negative. [expr.shift]\nA shift by "
4911  "%d was requested",
4912  __y);
4913  if (size_t(__y) >= sizeof(declval<_Tp>() << __y) * __CHAR_BIT__)
4914  __invoke_ub(
4915  "The behavior is undefined if the right operand of a "
4916  "shift operation is greater than or equal to the width of the "
4917  "promoted left operand. [expr.shift]\nA shift by %d was requested",
4918  __y);
4919  return _SimdIntOperators::_S_make_derived(
4920  _Impl::_S_bit_shift_left(__data(__x), __y));
4921  }
4922 
4923  template <typename _VV = _V>
4924  _GLIBCXX_SIMD_CONSTEXPR friend _V operator>>(const _V& __x, int __y)
4925  {
4926  using _Tp = typename _VV::value_type;
4927  if (__y < 0)
4928  __invoke_ub(
4929  "The behavior is undefined if the right operand of a shift "
4930  "operation is negative. [expr.shift]\nA shift by %d was requested",
4931  __y);
4932  if (size_t(__y) >= sizeof(declval<_Tp>() << __y) * __CHAR_BIT__)
4933  __invoke_ub(
4934  "The behavior is undefined if the right operand of a shift "
4935  "operation is greater than or equal to the width of the promoted "
4936  "left operand. [expr.shift]\nA shift by %d was requested",
4937  __y);
4938  return _SimdIntOperators::_S_make_derived(
4939  _Impl::_S_bit_shift_right(__data(__x), __y));
4940  }
4941 
4942  // unary operators (for integral _Tp)
4943  _GLIBCXX_SIMD_CONSTEXPR _V operator~() const
4944  { return {__private_init, _Impl::_S_complement(__derived()._M_data)}; }
4945  };
4946 
4947 //}}}1
4948 /// @endcond
4949 
4950 // simd {{{
4951 template <typename _Tp, typename _Abi>
4952  class simd : public _SimdIntOperators<
4953  simd<_Tp, _Abi>, typename _SimdTraits<_Tp, _Abi>::_SimdImpl,
4954  conjunction<is_integral<_Tp>,
4955  typename _SimdTraits<_Tp, _Abi>::_IsValid>::value>,
4956  public _SimdTraits<_Tp, _Abi>::_SimdBase
4957  {
4958  using _Traits = _SimdTraits<_Tp, _Abi>;
4959  using _MemberType = typename _Traits::_SimdMember;
4960  using _CastType = typename _Traits::_SimdCastType;
4961  static constexpr _Tp* _S_type_tag = nullptr;
4962  friend typename _Traits::_SimdBase;
4963 
4964  public:
4965  using _Impl = typename _Traits::_SimdImpl;
4966  friend _Impl;
4967  friend _SimdIntOperators<simd, _Impl, true>;
4968 
4969  using value_type = _Tp;
4970  using reference = _SmartReference<_MemberType, _Impl, value_type>;
4971  using mask_type = simd_mask<_Tp, _Abi>;
4972  using abi_type = _Abi;
4973 
4974  static constexpr size_t size()
4975  { return __size_or_zero_v<_Tp, _Abi>; }
4976 
4977  _GLIBCXX_SIMD_CONSTEXPR simd() = default;
4978  _GLIBCXX_SIMD_CONSTEXPR simd(const simd&) = default;
4979  _GLIBCXX_SIMD_CONSTEXPR simd(simd&&) noexcept = default;
4980  _GLIBCXX_SIMD_CONSTEXPR simd& operator=(const simd&) = default;
4981  _GLIBCXX_SIMD_CONSTEXPR simd& operator=(simd&&) noexcept = default;
4982 
4983  // implicit broadcast constructor
4984  template <typename _Up,
4985  typename = enable_if_t<!is_same_v<__remove_cvref_t<_Up>, bool>>>
4986  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
4987  simd(_ValuePreservingOrInt<_Up, value_type>&& __x)
4988  : _M_data(
4989  _Impl::_S_broadcast(static_cast<value_type>(static_cast<_Up&&>(__x))))
4990  {}
4991 
4992  // implicit type conversion constructor (convert from fixed_size to
4993  // fixed_size)
4994  template <typename _Up>
4995  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
4996  simd(const simd<_Up, simd_abi::fixed_size<size()>>& __x,
4997  enable_if_t<
4998  conjunction<
4999  is_same<simd_abi::fixed_size<size()>, abi_type>,
5000  negation<__is_narrowing_conversion<_Up, value_type>>,
5001  __converts_to_higher_integer_rank<_Up, value_type>>::value,
5002  void*> = nullptr)
5003  : simd{static_cast<array<_Up, size()>>(__x).data(), vector_aligned} {}
5004 
5005  // explicit type conversion constructor
5006 #ifdef _GLIBCXX_SIMD_ENABLE_STATIC_CAST
5007  template <typename _Up, typename _A2,
5008  typename = decltype(static_simd_cast<simd>(
5009  declval<const simd<_Up, _A2>&>()))>
5010  _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR
5011  simd(const simd<_Up, _A2>& __x)
5012  : simd(static_simd_cast<simd>(__x)) {}
5013 #endif // _GLIBCXX_SIMD_ENABLE_STATIC_CAST
5014 
5015  // generator constructor
5016  template <typename _Fp>
5017  _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR
5018  simd(_Fp&& __gen, _ValuePreservingOrInt<decltype(declval<_Fp>()(
5019  declval<_SizeConstant<0>&>())),
5020  value_type>* = nullptr)
5021  : _M_data(_Impl::_S_generator(static_cast<_Fp&&>(__gen), _S_type_tag)) {}
5022 
5023  // load constructor
5024  template <typename _Up, typename _Flags>
5025  _GLIBCXX_SIMD_ALWAYS_INLINE
5026  simd(const _Up* __mem, _Flags)
5027  : _M_data(
5028  _Impl::_S_load(_Flags::template _S_apply<simd>(__mem), _S_type_tag))
5029  {}
5030 
5031  // loads [simd.load]
5032  template <typename _Up, typename _Flags>
5033  _GLIBCXX_SIMD_ALWAYS_INLINE void
5034  copy_from(const _Vectorizable<_Up>* __mem, _Flags)
5035  {
5036  _M_data = static_cast<decltype(_M_data)>(
5037  _Impl::_S_load(_Flags::template _S_apply<simd>(__mem), _S_type_tag));
5038  }
5039 
5040  // stores [simd.store]
5041  template <typename _Up, typename _Flags>
5042  _GLIBCXX_SIMD_ALWAYS_INLINE void
5043  copy_to(_Vectorizable<_Up>* __mem, _Flags) const
5044  {
5045  _Impl::_S_store(_M_data, _Flags::template _S_apply<simd>(__mem),
5046  _S_type_tag);
5047  }
5048 
5049  // scalar access
5050  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR reference
5051  operator[](size_t __i)
5052  { return {_M_data, int(__i)}; }
5053 
5054  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR value_type
5055  operator[]([[maybe_unused]] size_t __i) const
5056  {
5057  if constexpr (__is_scalar_abi<_Abi>())
5058  {
5059  _GLIBCXX_DEBUG_ASSERT(__i == 0);
5060  return _M_data;
5061  }
5062  else
5063  return _M_data[__i];
5064  }
5065 
5066  // increment and decrement:
5067  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd&
5068  operator++()
5069  {
5070  _Impl::_S_increment(_M_data);
5071  return *this;
5072  }
5073 
5074  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5075  operator++(int)
5076  {
5077  simd __r = *this;
5078  _Impl::_S_increment(_M_data);
5079  return __r;
5080  }
5081 
5082  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd&
5083  operator--()
5084  {
5085  _Impl::_S_decrement(_M_data);
5086  return *this;
5087  }
5088 
5089  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5090  operator--(int)
5091  {
5092  simd __r = *this;
5093  _Impl::_S_decrement(_M_data);
5094  return __r;
5095  }
5096 
5097  // unary operators (for any _Tp)
5098  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR mask_type
5099  operator!() const
5100  { return {__private_init, _Impl::_S_negate(_M_data)}; }
5101 
5102  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5103  operator+() const
5104  { return *this; }
5105 
5106  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5107  operator-() const
5108  { return {__private_init, _Impl::_S_unary_minus(_M_data)}; }
5109 
5110  // access to internal representation (suggested extension)
5111  _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR
5112  simd(_CastType __init) : _M_data(__init) {}
5113 
5114  // compound assignment [simd.cassign]
5115  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd&
5116  operator+=(simd& __lhs, const simd& __x)
5117  { return __lhs = __lhs + __x; }
5118 
5119  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd&
5120  operator-=(simd& __lhs, const simd& __x)
5121  { return __lhs = __lhs - __x; }
5122 
5123  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd&
5124  operator*=(simd& __lhs, const simd& __x)
5125  { return __lhs = __lhs * __x; }
5126 
5127  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd&
5128  operator/=(simd& __lhs, const simd& __x)
5129  { return __lhs = __lhs / __x; }
5130 
5131  // binary operators [simd.binary]
5132  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5133  operator+(const simd& __x, const simd& __y)
5134  { return {__private_init, _Impl::_S_plus(__x._M_data, __y._M_data)}; }
5135 
5136  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5137  operator-(const simd& __x, const simd& __y)
5138  { return {__private_init, _Impl::_S_minus(__x._M_data, __y._M_data)}; }
5139 
5140  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5141  operator*(const simd& __x, const simd& __y)
5142  { return {__private_init, _Impl::_S_multiplies(__x._M_data, __y._M_data)}; }
5143 
5144  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5145  operator/(const simd& __x, const simd& __y)
5146  { return {__private_init, _Impl::_S_divides(__x._M_data, __y._M_data)}; }
5147 
5148  // compares [simd.comparison]
5149  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5150  operator==(const simd& __x, const simd& __y)
5151  { return simd::_S_make_mask(_Impl::_S_equal_to(__x._M_data, __y._M_data)); }
5152 
5153  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5154  operator!=(const simd& __x, const simd& __y)
5155  {
5156  return simd::_S_make_mask(
5157  _Impl::_S_not_equal_to(__x._M_data, __y._M_data));
5158  }
5159 
5160  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5161  operator<(const simd& __x, const simd& __y)
5162  { return simd::_S_make_mask(_Impl::_S_less(__x._M_data, __y._M_data)); }
5163 
5164  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5165  operator<=(const simd& __x, const simd& __y)
5166  {
5167  return simd::_S_make_mask(_Impl::_S_less_equal(__x._M_data, __y._M_data));
5168  }
5169 
5170  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5171  operator>(const simd& __x, const simd& __y)
5172  { return simd::_S_make_mask(_Impl::_S_less(__y._M_data, __x._M_data)); }
5173 
5174  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5175  operator>=(const simd& __x, const simd& __y)
5176  {
5177  return simd::_S_make_mask(_Impl::_S_less_equal(__y._M_data, __x._M_data));
5178  }
5179 
5180  // operator?: overloads (suggested extension) {{{
5181 #ifdef __GXX_CONDITIONAL_IS_OVERLOADABLE__
5182  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5183  operator?:(const mask_type& __k, const simd& __where_true,
5184  const simd& __where_false)
5185  {
5186  auto __ret = __where_false;
5187  _Impl::_S_masked_assign(__data(__k), __data(__ret), __data(__where_true));
5188  return __ret;
5189  }
5190 
5191 #endif // __GXX_CONDITIONAL_IS_OVERLOADABLE__
5192  // }}}
5193 
5194  // "private" because of the first arguments's namespace
5195  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
5196  simd(_PrivateInit, const _MemberType& __init)
5197  : _M_data(__init) {}
5198 
5199  // "private" because of the first arguments's namespace
5200  _GLIBCXX_SIMD_INTRINSIC
5201  simd(_BitsetInit, bitset<size()> __init) : _M_data()
5202  { where(mask_type(__bitset_init, __init), *this) = ~*this; }
5203 
5204  _GLIBCXX_SIMD_INTRINSIC constexpr bool
5205  _M_is_constprop() const
5206  {
5207  if constexpr (__is_scalar_abi<_Abi>())
5208  return __builtin_constant_p(_M_data);
5209  else
5210  return _M_data._M_is_constprop();
5211  }
5212 
5213  private:
5214  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR static mask_type
5215  _S_make_mask(typename mask_type::_MemberType __k)
5216  { return {__private_init, __k}; }
5217 
5218  friend const auto& __data<value_type, abi_type>(const simd&);
5219  friend auto& __data<value_type, abi_type>(simd&);
5220  alignas(_Traits::_S_simd_align) _MemberType _M_data;
5221  };
5222 
5223 // }}}
5224 /// @cond undocumented
5225 // __data {{{
5226 template <typename _Tp, typename _Ap>
5227  _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
5228  __data(const simd<_Tp, _Ap>& __x)
5229  { return __x._M_data; }
5230 
5231 template <typename _Tp, typename _Ap>
5232  _GLIBCXX_SIMD_INTRINSIC constexpr auto&
5233  __data(simd<_Tp, _Ap>& __x)
5234  { return __x._M_data; }
5235 
5236 // }}}
5237 namespace __float_bitwise_operators { //{{{
5238 template <typename _Tp, typename _Ap>
5239  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5240  operator^(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
5241  {
5242  return {__private_init,
5243  _Ap::_SimdImpl::_S_bit_xor(__data(__a), __data(__b))};
5244  }
5245 
5246 template <typename _Tp, typename _Ap>
5247  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5248  operator|(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
5249  {
5250  return {__private_init,
5251  _Ap::_SimdImpl::_S_bit_or(__data(__a), __data(__b))};
5252  }
5253 
5254 template <typename _Tp, typename _Ap>
5255  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5256  operator&(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
5257  {
5258  return {__private_init,
5259  _Ap::_SimdImpl::_S_bit_and(__data(__a), __data(__b))};
5260  }
5261 
5262 template <typename _Tp, typename _Ap>
5263  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
5264  enable_if_t<is_floating_point_v<_Tp>, simd<_Tp, _Ap>>
5265  operator~(const simd<_Tp, _Ap>& __a)
5266  { return {__private_init, _Ap::_SimdImpl::_S_complement(__data(__a))}; }
5267 } // namespace __float_bitwise_operators }}}
5268 /// @endcond
5269 
5270 /// @}
5271 _GLIBCXX_SIMD_END_NAMESPACE
5272 
5273 #endif // __cplusplus >= 201703L
5274 #endif // _GLIBCXX_EXPERIMENTAL_SIMD_H
5275 
5276 // vim: foldmethod=marker foldmarker={{{,}}}
constexpr _If_is_unsigned_integer< _Tp, int > popcount(_Tp __x) noexcept
The number of bits set in x.
Definition: bit:361
constexpr complex< _Tp > operator*(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x times y.
Definition: complex:392
constexpr complex< _Tp > operator/(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x divided by y.
Definition: complex:422
constexpr complex< _Tp > operator-(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x minus y.
Definition: complex:362
constexpr complex< _Tp > operator+(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x plus y.
Definition: complex:332
typename remove_reference< _Tp >::type remove_reference_t
Alias template for remove_reference.
Definition: type_traits:1670
typename make_unsigned< _Tp >::type make_unsigned_t
Alias template for make_unsigned.
Definition: type_traits:2009
void void_t
A metafunction that always yields void, used for detecting valid types.
Definition: type_traits:2636
integral_constant< bool, true > true_type
The type used as a compile-time boolean with true value.
Definition: type_traits:82
typename conditional< _Cond, _Iftrue, _Iffalse >::type conditional_t
Alias template for conditional.
Definition: type_traits:2618
typename remove_pointer< _Tp >::type remove_pointer_t
Alias template for remove_pointer.
Definition: type_traits:2084
integral_constant< bool, false > false_type
The type used as a compile-time boolean with false value.
Definition: type_traits:85
typename remove_const< _Tp >::type remove_const_t
Alias template for remove_const.
Definition: type_traits:1601
typename enable_if< _Cond, _Tp >::type enable_if_t
Alias template for enable_if.
Definition: type_traits:2614
constexpr auto tuple_cat(_Tpls &&... __tpls) -> typename __tuple_cat_result< _Tpls... >::__type
tuple_cat
Definition: tuple:1730
auto declval() noexcept -> decltype(__declval< _Tp >(0))
Definition: type_traits:2393
constexpr std::remove_reference< _Tp >::type && move(_Tp &&__t) noexcept
Convert a value to an rvalue.
Definition: move.h:104
constexpr const _Tp & max(const _Tp &, const _Tp &)
This does what you think it does.
Definition: stl_algobase.h:254
constexpr const _Tp & min(const _Tp &, const _Tp &)
This does what you think it does.
Definition: stl_algobase.h:230
constexpr _Tp reduce(_InputIterator __first, _InputIterator __last, _Tp __init, _BinaryOperation __binary_op)
Calculate reduction of values in a range.
Definition: numeric:278
std::basic_istream< _CharT, _Traits > & operator>>(std::basic_istream< _CharT, _Traits > &__is, bitset< _Nb > &__x)
Global I/O operators for bitsets.
Definition: bitset:1472
bitset< _Nb > operator^(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
Definition: bitset:1453
std::basic_ostream< _CharT, _Traits > & operator<<(std::basic_ostream< _CharT, _Traits > &__os, const bitset< _Nb > &__x)
Global I/O operators for bitsets.
Definition: bitset:1540
bitset< _Nb > operator|(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
Definition: bitset:1444
bitset< _Nb > operator&(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
Definition: bitset:1435