28#if defined(HIGHWAY_HWY_TESTS_TEST_UTIL_INL_H_) == \
29 defined(HWY_TARGET_TOGGLE)
30#ifdef HIGHWAY_HWY_TESTS_TEST_UTIL_INL_H_
31#undef HIGHWAY_HWY_TESTS_TEST_UTIL_INL_H_
33#define HIGHWAY_HWY_TESTS_TEST_UTIL_INL_H_
43template <
class D,
typename T = TFromD<D>,
class V = Vec<D>>
45 const char* filename,
const int line) {
47 auto actual_lanes = AllocateAligned<T>(
N);
48 Store(actual,
d, actual_lanes.get());
50 const auto info = hwy::detail::MakeTypeInfo<T>();
53 target_name, filename,
line);
59template <
class D,
typename T = TFromD<D>,
class V = Vec<D>>
61 const char* filename,
int line) {
62 auto expected_lanes = AllocateAligned<T>(
Lanes(
d));
63 Store(expected,
d, expected_lanes.get());
70 const char* filename,
int line) {
82#if HWY_TARGET == HWY_SCALAR
87 const size_t N8 =
Lanes(d8);
88 auto bits_a = AllocateAligned<uint8_t>(
HWY_MAX(
size_t{8}, N8));
89 auto bits_b = AllocateAligned<uint8_t>(
size_t{
HWY_MAX(8, N8)});
90 memset(bits_a.get(), 0, N8);
91 memset(bits_b.get(), 0, N8);
97 for (; i <
N / 8; ++i) {
98 if (bits_a[i] != bits_b[i]) {
99 fprintf(stderr,
"Mismatch in byte %d: %d != %d\n",
static_cast<int>(i),
100 bits_a[i], bits_b[i]);
101 Print(d8,
"expect",
Load(d8, bits_a.get()), 0, N8);
102 Print(d8,
"actual",
Load(d8, bits_b.get()), 0, N8);
103 hwy::Abort(filename,
line,
"Masks not equal");
107 const size_t remainder =
N % 8;
108 if (remainder != 0) {
109 const int mask = (1 << remainder) - 1;
110 const int valid_a = bits_a[i] & mask;
111 const int valid_b = bits_b[i] & mask;
112 if (valid_a != valid_b) {
113 fprintf(stderr,
"Mismatch in last byte %d: %d != %d\n",
114 static_cast<int>(i), valid_a, valid_b);
115 Print(d8,
"expect",
Load(d8, bits_a.get()), 0, N8);
116 Print(d8,
"actual",
Load(d8, bits_b.get()), 0, N8);
117 hwy::Abort(filename,
line,
"Masks not equal");
139#define HWY_ASSERT_EQ(expected, actual) \
140 hwy::AssertEqual(expected, actual, hwy::TargetName(HWY_TARGET), __FILE__, \
143#define HWY_ASSERT_ARRAY_EQ(expected, actual, count) \
144 hwy::AssertArrayEqual(expected, actual, count, hwy::TargetName(HWY_TARGET), \
147#define HWY_ASSERT_STRING_EQ(expected, actual) \
148 hwy::AssertStringEqual(expected, actual, hwy::TargetName(HWY_TARGET), \
151#define HWY_ASSERT_VEC_EQ(d, expected, actual) \
152 AssertVecEqual(d, expected, actual, __FILE__, __LINE__)
154#define HWY_ASSERT_MASK_EQ(d, expected, actual) \
155 AssertMaskEqual(d, expected, actual, __FILE__, __LINE__)
167template <
typename T,
size_t kMul,
size_t kMinArg,
class Test>
169 static void Do(
size_t min_lanes,
size_t max_lanes) {
173 const size_t lanes =
Lanes(
d);
174 if (lanes < min_lanes)
return;
176 if (lanes <= max_lanes) {
184template <
typename T,
size_t kMinArg,
class Test>
186 static void Do(
size_t,
size_t) {}
192constexpr int MinPow2() {
201template <
typename T,
int kPow2,
int kAddPow2,
class Test>
202struct ForeachShiftR {
203 static void Do(
size_t min_lanes) {
204 const ScalableTag<T, kPow2 + kAddPow2>
d;
208 if (
Lanes(
d) >= min_lanes) {
211 fprintf(stderr,
"%d lanes < %d: T=%d pow=%d\n",
212 static_cast<int>(
Lanes(
d)),
static_cast<int>(min_lanes),
213 static_cast<int>(
sizeof(T)), kPow2 + kAddPow2);
217 ForeachShiftR<T, kPow2 + 1, kAddPow2, Test>::Do(min_lanes);
222template <
typename T,
int kAddPow2,
class Test>
223struct ForeachShiftR<T, 4, kAddPow2, Test> {
224 static void Do(
size_t) {}
242template <
class Test,
int kPow2 = 1>
249 HWY_ABORT(
"Test is incorrect, ensure operator() is called");
253 template <
typename T>
256 constexpr size_t kMaxCapped =
HWY_LANES(T);
261#if HWY_TARGET == HWY_SCALAR
265#if HWY_TARGET == HWY_RVV
267 detail::ForeachShiftR<T, detail::MinPow2<T>() + kPow2, -kPow2, Test>::Do(1);
268#elif HWY_HAVE_SCALABLE
270 detail::ForeachShiftR<T, detail::MinPow2<T>() + kPow2 + 3, -kPow2 - 3,
279template <
class Test,
int kPow2 = 1>
286 HWY_ABORT(
"Test is incorrect, ensure operator() is called");
290 template <
typename T>
293 constexpr size_t kMinLanes =
size_t{1} << kPow2;
294 constexpr size_t kMaxCapped =
HWY_LANES(T);
296 constexpr size_t max_lanes = kMaxCapped;
301#if HWY_TARGET == HWY_SCALAR
305 kMinLanes, max_lanes);
306#if HWY_TARGET == HWY_RVV
308 detail::ForeachShiftR<T, detail::MinPow2<T>() + kPow2, 0, Test>::Do(
310#elif HWY_HAVE_SCALABLE
312 detail::ForeachShiftR<T, detail::MinPow2<T>() + kPow2 + 3, -3, Test>::Do(
321template <
size_t kMinBits,
class Test>
328 HWY_ABORT(
"Test is incorrect, ensure operator() is called");
332 template <
typename T>
335 constexpr size_t kMaxCapped =
HWY_LANES(T);
336 constexpr size_t kMinLanes = kMinBits / 8 /
sizeof(T);
338 constexpr size_t max_lanes = kMaxCapped;
340#if HWY_TARGET == HWY_SCALAR
344 kMinLanes, max_lanes);
345#if HWY_TARGET == HWY_RVV
347 constexpr size_t kRatio = 128 / kMinBits;
348 constexpr int kMinPow2 =
349 kRatio == 0 ? 0 : -
static_cast<int>(
CeilLog2(kRatio));
351 detail::ForeachShiftR<T, kMinPow2, 0, Test>::Do(kMinLanes);
352#elif HWY_HAVE_SCALABLE
354 constexpr size_t kRatio = 128 / kMinBits;
355 constexpr int kMinPow2 =
356 kRatio == 0 ? 0 : -
static_cast<int>(
CeilLog2(kRatio));
358 detail::ForeachShiftR<T, kMinPow2 + 3, -3, Test>::Do(kMinLanes);
369template <
class Test,
int kPow2 = 1>
376 HWY_ABORT(
"Test is incorrect, ensure operator() is called");
380 template <
typename T>
383 constexpr size_t kFactor =
size_t{1} << kPow2;
384 static_assert(kFactor >= 2 && kFactor *
sizeof(T) <=
sizeof(uint64_t),
"");
385 constexpr size_t kMaxCapped =
HWY_LANES(T);
386 constexpr size_t kMinLanes = kFactor;
392#if HWY_TARGET == HWY_SCALAR
398#if HWY_TARGET == HWY_RVV
400 detail::ForeachShiftR<T, detail::MinPow2<T>() + kPow2, -kPow2, Test>::Do(
402#elif HWY_HAVE_SCALABLE
404 detail::ForeachShiftR<T, detail::MinPow2<T>() + kPow2 + 3, -kPow2 - 3,
405 Test>::Do(kMinLanes);
413template <
class Test,
int kPow2 = 1>
420 HWY_ABORT(
"Test is incorrect, ensure operator() is called");
424 template <
typename T>
427 constexpr size_t kMinLanes =
size_t{1} << kPow2;
428 constexpr size_t kMaxCapped =
HWY_LANES(T);
430 constexpr size_t max_lanes = kMaxCapped;
435#if HWY_TARGET == HWY_SCALAR
439 kMinLanes, max_lanes);
442#if HWY_TARGET == HWY_RVV
444 detail::ForeachShiftR<T, detail::MinPow2<T>() + kPow2, 0, Test>::Do(
446#elif HWY_HAVE_SCALABLE
448 detail::ForeachShiftR<T, detail::MinPow2<T>() + kPow2 + 3, -3, Test>::Do(
456template <
class Test,
int kPow2 = 1>
463 HWY_ABORT(
"Test is incorrect, ensure operator() is called");
467 template <
typename T>
470#if HWY_TARGET == HWY_SCALAR
473 constexpr size_t kMinLanes =
size_t{1} << kPow2;
475 constexpr size_t kMaxCapped =
HWY_LANES(T);
477 kMinLanes, kMaxCapped);
480#if HWY_TARGET == HWY_RVV
482 detail::ForeachShiftR<T, detail::MinPow2<T>() + kPow2, 0, Test>::Do(
484#elif HWY_HAVE_SCALABLE
486 detail::ForeachShiftR<T, detail::MinPow2<T>() + kPow2 + 3, -3, Test>::Do(
502 HWY_ABORT(
"Test is incorrect, ensure operator() is called");
506 template <
typename T>
509#if HWY_TARGET == HWY_SCALAR
525#if HWY_HAVE_INTEGER64
535#if HWY_HAVE_INTEGER64
594#if HWY_HAVE_INTEGER64
636 return HWY_MAX(max_reps / 32, 2);
637#elif HWY_IS_DEBUG_BUILD
638 return HWY_MAX(max_reps / 8, 2);
640 return HWY_MAX(max_reps / 4, 2);
650 return HWY_MIN(max_pow2 - 4, max_pow2);
651#elif HWY_IS_DEBUG_BUILD
652 return HWY_MIN(max_pow2 - 1, max_pow2);
654 return HWY_MIN(max_pow2 - 1, max_pow2);
#define HWY_MAX(a, b)
Definition: base.h:135
#define HWY_NOINLINE
Definition: base.h:72
#define HWY_MIN(a, b)
Definition: base.h:134
#define HWY_ABORT(format,...)
Definition: base.h:188
#define HWY_INLINE
Definition: base.h:70
#define HWY_ASSERT(condition)
Definition: base.h:192
Definition: test_util-inl.h:414
~ForDemoteVectors()
Definition: test_util-inl.h:418
void operator()(T) const
Definition: test_util-inl.h:425
bool called_
Definition: test_util-inl.h:415
Definition: test_util-inl.h:243
void operator()(T) const
Definition: test_util-inl.h:254
bool called_
Definition: test_util-inl.h:244
~ForExtendableVectors()
Definition: test_util-inl.h:247
Definition: test_util-inl.h:322
bool called_
Definition: test_util-inl.h:323
~ForGEVectors()
Definition: test_util-inl.h:326
void operator()(T) const
Definition: test_util-inl.h:333
Definition: test_util-inl.h:457
~ForHalfVectors()
Definition: test_util-inl.h:461
bool called_
Definition: test_util-inl.h:458
void operator()(T) const
Definition: test_util-inl.h:468
Definition: test_util-inl.h:496
bool called_
Definition: test_util-inl.h:497
void operator()(T t) const
Definition: test_util-inl.h:507
~ForPartialVectors()
Definition: test_util-inl.h:500
Definition: test_util-inl.h:280
void operator()(T) const
Definition: test_util-inl.h:291
bool called_
Definition: test_util-inl.h:281
~ForShrinkableVectors()
Definition: test_util-inl.h:284
#define HWY_TARGET
Definition: detect_targets.h:380
d
Definition: rvv-inl.h:1998
V VecArg
Definition: ops/shared-inl.h:324
HWY_API Mask128< TTo, N > RebindMask(Simd< TTo, N, 0 > dto, Mask128< TFrom, N > m)
Definition: arm_neon-inl.h:2230
constexpr size_t AdjustedReps(size_t max_reps)
Definition: test_util-inl.h:634
HWY_API bool AllTrue(const Full128< T > d, const Mask128< T > m)
Definition: arm_neon-inl.h:5716
void ForUIF32(const Func &func)
Definition: test_util-inl.h:587
void ForUI163264(const Func &func)
Definition: test_util-inl.h:621
HWY_API Mask128< T, N > FirstN(const Simd< T, N, 0 > d, size_t num)
Definition: arm_neon-inl.h:2456
HWY_API size_t StoreMaskBits(Simd< T, N, 0 >, const Mask128< T, N > mask, uint8_t *bits)
Definition: arm_neon-inl.h:5701
void ForUIF3264(const Func &func)
Definition: test_util-inl.h:615
void ForUIF163264(const Func &func)
Definition: test_util-inl.h:627
constexpr size_t AdjustedLog2Reps(size_t max_pow2)
Definition: test_util-inl.h:647
typename detail::CappedTagChecker< T, kLimit >::type CappedTag
Definition: ops/shared-inl.h:184
void ForUI32(const Func &func)
Definition: test_util-inl.h:581
void ForAllTypes(const Func &func)
Definition: test_util-inl.h:555
Rebind< MakeSigned< TFromD< D > >, D > RebindToSigned
Definition: ops/shared-inl.h:210
void ForFloatTypes(const Func &func)
Definition: test_util-inl.h:547
void Print(const D d, const char *caption, VecArg< V > v, size_t lane_u=0, size_t max_lanes=7)
Definition: print-inl.h:39
HWY_NOINLINE void AssertMaskEqual(D d, VecArg< Mask< D > > a, VecArg< Mask< D > > b, const char *filename, int line)
Definition: test_util-inl.h:69
HWY_API size_t CountTrue(Full128< T >, const Mask128< T > mask)
Definition: arm_neon-inl.h:5671
HWY_API Vec128< T, N > VecFromMask(Simd< T, N, 0 > d, const Mask128< T, N > v)
Definition: arm_neon-inl.h:2223
HWY_INLINE void AssertVecEqual(D d, const T *expected, VecArg< V > actual, const char *filename, const int line)
Definition: test_util-inl.h:44
void ForIntegerTypes(const Func &func)
Definition: test_util-inl.h:541
HWY_API constexpr size_t Lanes(Simd< T, N, kPow2 >)
Definition: arm_sve-inl.h:243
HWY_API Vec128< T, N > Load(Simd< T, N, 0 > d, const T *HWY_RESTRICT p)
Definition: arm_neon-inl.h:2753
HWY_INLINE Mask< D > MaskFalse(const D d)
Definition: test_util-inl.h:132
void ForUI8(const Func &func)
Definition: test_util-inl.h:561
void ForUI3264(const Func &func)
Definition: test_util-inl.h:609
typename detail::ScalableTagChecker< T, kPow2 >::type ScalableTag
Definition: ops/shared-inl.h:173
HWY_API bool AllFalse(const Simd< T, N, 0 > d, const Mask128< T, N > m)
Definition: arm_neon-inl.h:5710
void ForUIF64(const Func &func)
Definition: test_util-inl.h:601
void ForUI16(const Func &func)
Definition: test_util-inl.h:567
typename D::template Rebind< T > Rebind
Definition: ops/shared-inl.h:207
HWY_API Vec128< T, N > Zero(Simd< T, N, 0 > d)
Definition: arm_neon-inl.h:1020
void ForUI64(const Func &func)
Definition: test_util-inl.h:593
void ForSignedTypes(const Func &func)
Definition: test_util-inl.h:521
void ForUIF16(const Func &func)
Definition: test_util-inl.h:573
typename D::template Repartition< T > Repartition
Definition: ops/shared-inl.h:218
decltype(MaskFromVec(Zero(D()))) Mask
Definition: generic_ops-inl.h:46
N
Definition: rvv-inl.h:1998
HWY_INLINE Mask< D > MaskTrue(const D d)
Definition: test_util-inl.h:127
HWY_API void Store(Vec128< T, N > v, Simd< T, N, 0 > d, T *HWY_RESTRICT aligned)
Definition: arm_neon-inl.h:2934
void ForUnsignedTypes(const Func &func)
Definition: test_util-inl.h:531
decltype(Zero(D())) Vec
Definition: generic_ops-inl.h:40
HWY_TEST_DLLEXPORT void AssertArrayEqual(const TypeInfo &info, const void *expected_void, const void *actual_void, size_t N, const char *target_name, const char *filename, int line)
Definition: aligned_allocator.h:27
FuncOutput(*)(const void *, FuncInput) Func
Definition: nanobenchmark.h:105
static HWY_MAYBE_UNUSED const char * TargetName(int64_t target)
Definition: targets.h:85
constexpr size_t CeilLog2(TI x)
Definition: base.h:899
HWY_INLINE void AssertEqual(const T expected, const T actual, const char *target_name, const char *filename, int line, size_t lane=0)
Definition: test_util.h:152
HWY_DLLEXPORT HWY_NORETURN void int line
Definition: base.h:992
#define HWY_LANES(T)
Definition: set_macros-inl.h:85
#define HWY_NAMESPACE
Definition: set_macros-inl.h:82
static void Do(size_t, size_t)
Definition: test_util-inl.h:186
Definition: test_util-inl.h:168
static void Do(size_t min_lanes, size_t max_lanes)
Definition: test_util-inl.h:169