KleidiCV Coverage Report

Directory: ./
File: kleidicv/include/kleidicv/operations.h
Date: 2026-01-20 20:58:59
Exec Total Coverage
Lines: 314 314 100.0%
Functions: 7125 7179 99.2%
Branches: 10 10 100.0%
Line Branch Exec Source
1 // SPDX-FileCopyrightText: 2023 - 2025 Arm Limited and/or its affiliates <open-source-office@arm.com>
2 //
3 // SPDX-License-Identifier: Apache-2.0
4
5 #ifndef KLEIDICV_OPERATIONS_H
6 #define KLEIDICV_OPERATIONS_H
7
8 #include <algorithm>
9 #include <tuple>
10 #include <utility>
11
12 #include "kleidicv/traits.h"
13 #include "kleidicv/types.h"
14
15 namespace KLEIDICV_TARGET_NAMESPACE {
16
17 // Base for classes that change the behaviour or extend the interface of an
18 // operation.
19 //
20 // This class conveniently forwards common types and methods without having to
21 // explicitly define them in all derived classes.
22 template <typename T>
23 class OperationBase {
24 public:
25 // Type of the inner operation.
26 using OperationType = T;
27
28 // The vector traits used by the inner operation.
29 using VecTraits = typename OperationType::VecTraits;
30
31 // The context type used by the inner operation, if any.
32 using ContextType = context_type_t<OperationType>;
33
34 // Returns a reference to the inner operation.
35 4771702 OperationType &operation() KLEIDICV_STREAMING { return operation_; }
36
37 // Forwards num_lanes() calls to the inner operation.
38 99803 static size_t num_lanes() KLEIDICV_STREAMING {
39 99803 return VecTraits::num_lanes();
40 }
41
42 // Forwards vector_path_2x() calls to the inner operation.
43 template <typename... ArgTypes>
44 514258 KLEIDICV_FORCE_INLINE decltype(auto) vector_path_2x(ArgTypes &&...args)
45 KLEIDICV_STREAMING {
46 514258 return operation().vector_path_2x(std::forward<ArgTypes>(args)...);
47 }
48
49 // Forwards vector_path() calls to the inner operation.
50 template <typename... ArgTypes>
51 155388 KLEIDICV_FORCE_INLINE decltype(auto) vector_path(ArgTypes &&...args)
52 KLEIDICV_STREAMING {
53 155388 return operation().vector_path(std::forward<ArgTypes>(args)...);
54 }
55
56 // Forwards remaining_path() calls to the inner operation.
57 template <typename... ArgTypes>
58 KLEIDICV_FORCE_INLINE decltype(auto) remaining_path(ArgTypes &&...args)
59 KLEIDICV_STREAMING {
60 return operation().remaining_path(std::forward<ArgTypes>(args)...);
61 }
62
63 // Forwards tail_path() calls to the inner operation.
64 template <typename... ArgTypes>
65 1044 KLEIDICV_FORCE_INLINE decltype(auto) tail_path(ArgTypes &&...args)
66 KLEIDICV_STREAMING {
67 1044 return operation().tail_path(std::forward<ArgTypes>(args)...);
68 }
69
70 // Forwards scalar_path() calls to the inner operation.
71 template <typename... ArgTypes>
72 KLEIDICV_FORCE_INLINE decltype(auto) scalar_path(ArgTypes &&...args)
73 KLEIDICV_STREAMING {
74 return operation().scalar_path(std::forward<ArgTypes>(args)...);
75 }
76
77 template <typename... ArgTypes>
78 39781 KLEIDICV_FORCE_INLINE decltype(auto) load(ArgTypes &&...args)
79 KLEIDICV_STREAMING {
80 39781 return VecTraits::load(std::forward<ArgTypes>(args)...);
81 }
82
83 template <typename... ArgTypes>
84 575720 KLEIDICV_FORCE_INLINE decltype(auto) load_consecutive(ArgTypes &&...args)
85 KLEIDICV_STREAMING {
86 575720 return VecTraits::load_consecutive(std::forward<ArgTypes>(args)...);
87 }
88
89 template <typename... ArgTypes>
90 35041 KLEIDICV_FORCE_INLINE decltype(auto) store(ArgTypes &&...args)
91 KLEIDICV_STREAMING {
92 35041 return VecTraits::store(std::forward<ArgTypes>(args)...);
93 }
94
95 template <typename... ArgTypes>
96 72966 KLEIDICV_FORCE_INLINE decltype(auto) store_consecutive(ArgTypes &&...args)
97 KLEIDICV_STREAMING {
98 72966 return VecTraits::store_consecutive(std::forward<ArgTypes>(args)...);
99 }
100
101 // Forwards max_vectors_per_block() calls to the inner operation.
102 KLEIDICV_FORCE_INLINE
103 6940 size_t max_vectors_per_block() KLEIDICV_STREAMING {
104 6940 return operation_.max_vectors_per_block();
105 }
106
107 // Forwards on_block_finished() calls to the inner operation.
108 KLEIDICV_FORCE_INLINE
109 22590 void on_block_finished(size_t vectors_in_block) KLEIDICV_STREAMING {
110 22590 return operation_.on_block_finished(vectors_in_block);
111 }
112
113 // Returns true if the innermost operation is unrolled twice, otherwise false.
114 static constexpr bool is_unrolled_twice() KLEIDICV_STREAMING {
115 return ::KLEIDICV_TARGET_NAMESPACE::is_unrolled_twice<
116 concrete_operation_type_t<OperationType>>;
117 }
118
119 // Returns true if the innermost operation is unrolled once, otherwise false.
120 static constexpr bool is_unrolled_once() KLEIDICV_STREAMING {
121 return ::KLEIDICV_TARGET_NAMESPACE::is_unrolled_once<
122 concrete_operation_type_t<OperationType>>;
123 }
124
125 // Returns true if the innermost operation uses tail path, otherwise false.
126 static constexpr bool uses_tail_path() KLEIDICV_STREAMING {
127 return ::KLEIDICV_TARGET_NAMESPACE::uses_tail_path<
128 concrete_operation_type_t<OperationType>>;
129 }
130
131 // Returns true if the innermost operation tries to avoid tail loop, otherwise
132 // false.
133 static constexpr bool try_to_avoid_tail_loop() KLEIDICV_STREAMING {
134 return ::KLEIDICV_TARGET_NAMESPACE::try_to_avoid_tail_loop<
135 concrete_operation_type_t<OperationType>>;
136 }
137
138 protected:
139 // Constructor is protected so that only derived classes can instantiate.
140 170860 explicit OperationBase(OperationType &operation) KLEIDICV_STREAMING
141 170860 : operation_{operation} {}
142
143 private:
144 // Reference to the inner operation.
145 OperationType &operation_;
146 }; // end of class OperationBase<OperationType>
147
148 // Mixin which simply forwards all operations as-is.
149 template <typename OperationType>
150 class ForwardingOperation : public OperationBase<OperationType> {
151 public:
152 25449 explicit ForwardingOperation(OperationType &operation) KLEIDICV_STREAMING
153 25449 : OperationBase<OperationType>(operation) {}
154 }; // end of class ForwardingOperation<OperationType>
155
156 // Facade to offer a simplified row-based operation interface.
157 //
158 // A row-based operation is executed row-by-row and is unrolled within the row.
159 //
160 // Facade layout:
161 // |- operation.vector_path_<N>x()
162 // process_row() -|- operation.vector_path()
163 // |- operation.remaining_path()
164 // |- operation.num_lanes()
165 template <typename OperationType>
166 class RowBasedOperation : public OperationBase<OperationType> {
167 public:
168 33100 explicit RowBasedOperation(OperationType &operation) KLEIDICV_STREAMING
169 33100 : OperationBase<OperationType>(operation) {}
170
171 template <typename... ColumnTypes>
172 41987 KLEIDICV_FORCE_INLINE void process_row(size_t length, ColumnTypes... columns)
173 KLEIDICV_STREAMING {
174 using Self = std::remove_reference_t<decltype(*this)>;
175
176 41987 LoopUnroll loop{length, this->num_lanes()};
177
178 if constexpr (OperationType::is_unrolled_twice()) {
179 struct UnrollTwiceFunctor {
180 Self &self;
181 std::tuple<ColumnTypes &...> cols;
182
183 91794 KLEIDICV_FORCE_INLINE void operator()(size_t step) const
184 KLEIDICV_STREAMING {
185 91794 std::apply(
186 183588 [&](auto &...columns) {
187 91794 self.operation().vector_path_2x(columns...);
188 91794 ((columns += step), ...);
189 91794 },
190 91794 cols);
191 91794 }
192 };
193 35226 loop.unroll_twice_if<OperationType::is_unrolled_twice()>(
194 35226 UnrollTwiceFunctor{*this, std::tie(columns...)});
195 }
196
197 // NOLINTBEGIN(cppcoreguidelines-avoid-goto, hicpp-avoid-goto)
198 4068 if constexpr (OperationType::is_unrolled_once()) {
199 avoid_tail_loop:
200 struct UnrollOnceFunctor {
201 Self &self;
202 std::tuple<ColumnTypes &...> cols;
203
204 13500 KLEIDICV_FORCE_INLINE void operator()(size_t step) const
205 KLEIDICV_STREAMING {
206 13500 std::apply(
207 27000 [&](auto &...columns) {
208 13500 self.operation().vector_path(columns...);
209 13500 ((columns += step), ...);
210 13500 },
211 13500 cols);
212 13500 }
213 };
214
215 4328 loop.unroll_once_if<OperationType::is_unrolled_once()>(
216 4328 UnrollOnceFunctor{*this, std::tie(columns...)});
217
218 // Process leftover bytes on the unrolled-once vector path, if requested
219 // and possible.
220 if constexpr (OperationType::try_to_avoid_tail_loop()) {
221 struct BackwardAdjustFunctor {
222 std::tuple<ColumnTypes &...> cols;
223
224 260 KLEIDICV_FORCE_INLINE void operator()(size_t backward_step) const
225 KLEIDICV_STREAMING {
226 260 std::apply(
227 520 [&](auto &...columns) { ((columns -= backward_step), ...); },
228 260 cols);
229 260 }
230 };
231
8/8
✓ Branch 0 taken 92 times.
✓ Branch 1 taken 440 times.
✓ Branch 2 taken 56 times.
✓ Branch 3 taken 322 times.
✓ Branch 4 taken 56 times.
✓ Branch 5 taken 322 times.
✓ Branch 6 taken 56 times.
✓ Branch 7 taken 322 times.
 1666 if (loop.try_avoid_tail_loop(
232 1666 BackwardAdjustFunctor{std::tie(columns...)})) {
233 260 goto avoid_tail_loop;
234 }
235 }
236 }
237 // NOLINTEND(cppcoreguidelines-avoid-goto, hicpp-avoid-goto)
238
239 struct RemainingFunctor {
240 Self &self;
241 std::tuple<ColumnTypes &...> cols;
242
243 37412 KLEIDICV_FORCE_INLINE void operator()(size_t rem_length,
244 size_t) const KLEIDICV_STREAMING {
245 37412 std::apply(
246 74824 [&](auto &...columns) {
247 37412 self.operation().remaining_path(rem_length, columns...);
248 37412 },
249 37412 cols);
250 37412 }
251 };
252 41987 loop.remaining(RemainingFunctor{*this, std::tie(columns...)});
253 41987 }
254 }; // end of class RowBasedOperation<OperationType>
255
256 // Facade to offer a simplified row-based operation interface.
257 //
258 // A row-based operation is executed row-by-row, is unrolled within the row and
259 // split into blocks.
260 //
261 // Facade layout:
262 // |- operation.vector_path_<N>x()
263 // process_row() -|- operation.vector_path()
264 // |- operation.remaining_path()
265 // |- operation.num_lanes()
266 // |- operation.max_vectors_per_block()
267 // |- operation.on_block_finished()
268 template <typename OperationType>
269 class RowBasedBlockOperation : public OperationBase<OperationType> {
270 public:
271 680 explicit RowBasedBlockOperation(OperationType &operation) KLEIDICV_STREAMING
272 680 : OperationBase<OperationType>(operation) {}
273
274 template <typename... ColumnTypes>
275 1388 KLEIDICV_FORCE_INLINE void process_row(size_t length, ColumnTypes... columns)
276 KLEIDICV_STREAMING {
277 using Self = std::remove_reference_t<decltype(*this)>;
278
279 if constexpr (OperationType::is_unrolled_twice()) {
280 struct UnrollTwiceFunctor {
281 Self &self;
282 std::tuple<ColumnTypes &...> cols;
283
284 422464 KLEIDICV_FORCE_INLINE void operator()(size_t step) const
285 KLEIDICV_STREAMING {
286 422464 std::apply(
287 844928 [&](auto &...columns) {
288 422464 self.operation().vector_path_2x(columns...);
289 422464 ((columns += step), ...);
290 422464 },
291 422464 cols);
292 422464 }
293 };
294 1388 process_blocks<2>(length,
295 1388 UnrollTwiceFunctor{*this, std::tie(columns...)});
296 }
297
298 if constexpr (OperationType::is_unrolled_once()) {
299 struct UnrollOnceFunctor {
300 Self &self;
301 std::tuple<ColumnTypes &...> cols;
302
303 KLEIDICV_FORCE_INLINE void operator()(size_t step) const
304 KLEIDICV_STREAMING {
305 std::apply(
306 [&](auto &...columns) {
307 self.operation().vector_path(columns...);
308 ((columns += step), ...);
309 },
310 cols);
311 }
312 };
313 process_blocks<1>(length, UnrollOnceFunctor{*this, std::tie(columns...)});
314 }
315
316 1388 LoopUnroll loop{length, this->num_lanes()};
317 struct RemainingFunctor {
318 Self &self;
319 std::tuple<ColumnTypes &...> cols;
320
321 984 KLEIDICV_FORCE_INLINE void operator()(size_t length,
322 size_t) const KLEIDICV_STREAMING {
323 984 std::apply(
324 1968 [&](auto &...columns) {
325 984 self.operation().remaining_path(length, columns...);
326 984 },
327 984 cols);
328 984 }
329 };
330 1388 loop.remaining(RemainingFunctor{*this, std::tie(columns...)});
331 1388 }
332
333 private:
334 template <size_t UnrollFactor, typename CallbackType>
335 1388 KLEIDICV_FORCE_INLINE void process_blocks(
336 size_t &length, CallbackType callback) KLEIDICV_STREAMING {
337 // The number of elements a single iteration would process.
338 1388 const size_t elements_per_iteration = UnrollFactor * this->num_lanes();
339 // The number of elements which will be processed when this method
340 // returns.
341 2776 const size_t processed_length =
342 1388 (length / elements_per_iteration) * elements_per_iteration;
343 // The number of vectors which can be processed safely with the current
344 // unroll factor.
345 2776 const size_t max_vectors_per_block =
346 1388 (this->max_vectors_per_block() / UnrollFactor) * UnrollFactor;
347
348 // Process data up to allowed number of blocks.
349
2/2
✓ Branch 0 taken 1388 times.
✓ Branch 1 taken 4518 times.
 5906 for (size_t index = 0, block_length = 0; index < processed_length;
350 4518 index += block_length) {
351 // Process max_vectors_per_block vectors, or less in the last iteration
352 // (remaining size).
353 4518 size_t remaining_vectors = (processed_length - index) / this->num_lanes();
354 9036 size_t vectors_in_block =
355 4518 std::min(max_vectors_per_block, remaining_vectors);
356 4518 block_length = vectors_in_block * this->num_lanes();
357
358 // clang-format off
359 // Process data with the appropriate unroll factor.
360 4518 LoopUnroll loop{block_length, this->num_lanes()};
361 4518 loop.unroll_n_times<UnrollFactor>(
362 426982 [&](size_t step) KLEIDICV_STREAMING {
363 422464 callback(step);
364 // Adjust remaining length here.
365 // This improves generated code.
366 422464 length -= step;
367 422464 });
368 // clang-format on
369
370 // Notify the operation that a block was processed.
371 4518 this->on_block_finished(vectors_in_block);
372 4518 }
373 1388 }
374 }; // end of class RowBasedBlockOperation<OperationType>
375
376 // Adapter to offer an interface which allows users to use ParallelRows and to
377 // unpack them.
378 template <typename OperationType>
379 class ParallelRowsAdapter : public OperationBase<OperationType> {
380 public:
381 using VecTraits = typename OperationBase<OperationType>::VecTraits;
382 using ScalarType = typename VecTraits::ScalarType;
383 using ConstParallelColumnType = ParallelColumns<const ScalarType>;
384 using ParallelColumnType = ParallelColumns<ScalarType>;
385 using ConstColumnType = Columns<const ScalarType>;
386
387 1960 explicit ParallelRowsAdapter(OperationType &operation) KLEIDICV_STREAMING
388 1960 : OperationBase<OperationType>(operation) {}
389
390 // Forwards vector_path_2x() calls to the inner operation with one source
391 // and destination parallel columns.
392 void vector_path_2x(ConstParallelColumnType src_a, ConstColumnType src_b,
393 ParallelColumnType dst) KLEIDICV_STREAMING {
394 this->operation().vector_path_2x(src_a.first(), src_a.second(), src_b,
395 dst.first(), dst.second());
396 }
397
398 // Forwards vector_path() calls to the inner operation with one source and
399 // destination parallel columns.
400 96 void vector_path(ConstParallelColumnType src_a, ConstColumnType src_b,
401 ParallelColumnType dst) KLEIDICV_STREAMING {
402 192 this->operation().vector_path(src_a.first(), src_a.second(), src_b,
403 96 dst.first(), dst.second());
404 96 }
405
406 // Forwards remaining_path() calls to the inner operation with one source
407 // and destination parallel columns.
408 3472 void remaining_path(size_t length, ConstParallelColumnType src_a,
409 ConstColumnType src_b,
410 ParallelColumnType dst) KLEIDICV_STREAMING {
411 6944 this->operation().remaining_path(length, src_a.first(), src_a.second(),
412 3472 src_b, dst.first(), dst.second());
413 3472 }
414 }; // end of class ParallelRowsAdapter<OperationType>
415
416 // Adapter to offer an interface which allows users to apply unrolling to the
417 // adaptee operation. This adapter binds to the interface of the innermost
418 // operation in the chain.
419 //
420 // Provides specialized implementations using SFINAE principle for
421 // - vector_path_2x(),
422 // - vector_path(), and
423 // - tail_path(), and
424 // - scalar_path()
425 // methods.
426 template <typename OperationType>
427 class OperationAdapter : public OperationBase<OperationType> {
428 // Shorten rows: no need to write 'this->'.
429 using OperationBase<OperationType>::operation;
430 using OperationBase<OperationType>::num_lanes;
431 using OperationBase<OperationType>::load;
432 using OperationBase<OperationType>::load_consecutive;
433 using OperationBase<OperationType>::store;
434 using OperationBase<OperationType>::store_consecutive;
435 // The innermost operation in the chain.
436 using ConcreteOperationType = concrete_operation_type_t<OperationType>;
437
438 public:
439 using VecTraits = typename OperationBase<OperationType>::VecTraits;
440 using ContextType = typename OperationBase<OperationType>::ContextType;
441 using ScalarType = typename VecTraits::ScalarType;
442 using VectorType = typename VecTraits::VectorType;
443 using Vector2Type = typename VecTraits::Vector2Type;
444 using Vector3Type = typename VecTraits::Vector3Type;
445 using Vector4Type = typename VecTraits::Vector4Type;
446 using ConstColumnType = Columns<const ScalarType>;
447 using ColumnType = Columns<ScalarType>;
448
449 33780 explicit OperationAdapter(OperationType &operation) KLEIDICV_STREAMING
450 33780 : OperationBase<OperationType>(operation) {}
451
452 // ---------------------------------------------------------------------------
453 // Forwarding implementations for vector_path_2x().
454 // ---------------------------------------------------------------------------
455
456 // Required method:
457 // void T::vector_path([ContextType,] VectorType);
458 template <typename T = ConcreteOperationType>
459 429496 enable_if_has_vector_path_t<void, T, ContextType, VectorType> vector_path_2x(
460 ContextType ctx, ConstColumnType src) KLEIDICV_STREAMING {
461 429496 VectorType src_0, src_1;
462 429496 operation().load_consecutive(ctx, &src[0], src_0, src_1);
463 429496 operation().vector_path(ctx, src_0);
464 429496 operation().vector_path(ctx, src_1);
465 429496 }
466
467 // Required method:
468 // VectorType T::vector_path([ContextType,] VectorType);
469 template <typename T = ConcreteOperationType>
470 enable_if_has_vector_path_t<VectorType, T, ContextType, VectorType>
471 20810 vector_path_2x(ContextType ctx, ConstColumnType src,
472 ColumnType dst) KLEIDICV_STREAMING {
473 20810 VectorType src_0, src_1;
474 20810 operation().load_consecutive(ctx, &src[0], src_0, src_1);
475 20810 VectorType res_0 = operation().vector_path(ctx, src_0);
476 20810 VectorType res_1 = operation().vector_path(ctx, src_1);
477 20810 operation().store_consecutive(ctx, res_0, res_1, &dst[0]);
478 20810 }
479
480 // Required method:
481 // VectorType T::vector_path([ContextType,] VectorType, VectorType);
482 template <typename T = ConcreteOperationType>
483 enable_if_has_vector_path_t<VectorType, T, ContextType, VectorType,
484 VectorType>
485 52156 vector_path_2x(ContextType ctx, ConstColumnType src_a, ConstColumnType src_b,
486 ColumnType dst) KLEIDICV_STREAMING {
487 52156 VectorType src_a_0, src_a_1, src_b_0, src_b_1;
488 52156 operation().load_consecutive(ctx, &src_a[0], src_a_0, src_a_1);
489 52156 operation().load_consecutive(ctx, &src_b[0], src_b_0, src_b_1);
490 52156 VectorType res_0 = operation().vector_path(ctx, src_a_0, src_b_0);
491 52156 VectorType res_1 = operation().vector_path(ctx, src_a_1, src_b_1);
492 52156 operation().store_consecutive(ctx, res_0, res_1, &dst[0]);
493 52156 }
494
495 // Required method:
496 // void T::vector_path([ContextType,] VectorType, ScalarType *);
497 template <typename T = ConcreteOperationType>
498 enable_if_has_vector_path_t<void, T, ContextType, VectorType, ScalarType *>
499 622 vector_path_2x(ContextType ctx, ConstColumnType src,
500 ColumnType dst) KLEIDICV_STREAMING {
501 622 VectorType src_0, src_1;
502 622 operation().load_consecutive(ctx, &src[0], src_0, src_1);
503 622 operation().vector_path(ctx, src_0, &dst[0]);
504 622 operation().vector_path(ctx, src_1, &dst.at(num_lanes())[0]);
505 622 }
506
507 // Required method:
508 // void T::vector_path([ContextType,] VectorType, VectorType, ScalarType *);
509 template <typename T = ConcreteOperationType>
510 enable_if_has_vector_path_t<void, T, ContextType, VectorType, VectorType,
511 ScalarType *>
512 800 vector_path_2x(ContextType ctx, ConstColumnType src_a, ConstColumnType src_b,
513 ColumnType dst) KLEIDICV_STREAMING {
514 800 VectorType src_a_0, src_a_1, src_b_0, src_b_1;
515 800 operation().load_consecutive(ctx, &src_a[0], src_a_0, src_a_1);
516 800 operation().load_consecutive(ctx, &src_b[0], src_b_0, src_b_1);
517 800 operation().vector_path(ctx, src_a_0, src_b_0, &dst[0]);
518 800 operation().vector_path(ctx, src_a_1, src_b_1, &dst.at(num_lanes())[0]);
519 800 }
520
521 // Required method:
522 // void T::vector_path([ContextType,] VectorType, VectorType, VectorType,
523 // ScalarType *);
524 template <typename T = ConcreteOperationType>
525 enable_if_has_vector_path_t<void, T, ContextType, VectorType, VectorType,
526 VectorType, ScalarType *>
527 1440 vector_path_2x(ContextType ctx, ConstColumnType src_a, ConstColumnType src_b,
528 ConstColumnType src_c, ColumnType dst) KLEIDICV_STREAMING {
529 1440 VectorType src_a_0, src_a_1, src_b_0, src_b_1, src_c_0, src_c_1;
530 1440 operation().load_consecutive(ctx, &src_a[0], src_a_0, src_a_1);
531 1440 operation().load_consecutive(ctx, &src_b[0], src_b_0, src_b_1);
532 1440 operation().load_consecutive(ctx, &src_c[0], src_c_0, src_c_1);
533 1440 operation().vector_path(ctx, src_a_0, src_b_0, src_c_0, &dst[0]);
534 2880 operation().vector_path(ctx, src_a_1, src_b_1, src_c_1,
535 1440 &dst.at(num_lanes())[0]);
536 1440 }
537
538 // Required method:
539 // void T::vector_path([ContextType,] VectorType, VectorType, VectorType,
540 // ScalarType *,
541 // ScalarType *);
542 template <typename T = ConcreteOperationType>
543 enable_if_has_vector_path_t<void, T, ContextType, VectorType, VectorType,
544 VectorType, ScalarType *, ScalarType *>
545 vector_path_2x(ContextType ctx, ConstColumnType src_a, ConstColumnType src_b,
546 ConstColumnType src_c, ColumnType dst_a,
547 ColumnType dst_b) KLEIDICV_STREAMING {
548 VectorType src_a_0, src_a_1, src_b_0, src_b_1, src_c_0, src_c_1;
549 operation().load_consecutive(ctx, &src_a[0], src_a_0, src_a_1);
550 operation().load_consecutive(ctx, &src_b[0], src_b_0, src_b_1);
551 operation().load_consecutive(ctx, &src_c[0], src_c_0, src_c_1);
552 operation().vector_path(ctx, src_a_0, src_b_0, src_c_0, &dst_a[0],
553 &dst_b[0]);
554 operation().vector_path(ctx, src_a_1, src_b_1, src_c_1,
555 &dst_a.at(num_lanes())[0],
556 &dst_b.at(num_lanes())[0]);
557 }
558
559 // Required method:
560 // void T::vector_path([ContextType,] VectorType, VectorType, VectorType,
561 // VectorType,
562 // ScalarType *);
563 template <typename T = ConcreteOperationType>
564 enable_if_has_vector_path_t<void, T, ContextType, VectorType, VectorType,
565 VectorType, VectorType, ScalarType *>
566 2720 vector_path_2x(ContextType ctx, ConstColumnType src_a, ConstColumnType src_b,
567 ConstColumnType src_c, ConstColumnType src_d,
568 ColumnType dst) KLEIDICV_STREAMING {
569 2720 VectorType src_a_0, src_a_1, src_b_0, src_b_1;
570 2720 VectorType src_c_0, src_c_1, src_d_0, src_d_1;
571 2720 operation().load_consecutive(ctx, &src_a[0], src_a_0, src_a_1);
572 2720 operation().load_consecutive(ctx, &src_b[0], src_b_0, src_b_1);
573 2720 operation().load_consecutive(ctx, &src_c[0], src_c_0, src_c_1);
574 2720 operation().load_consecutive(ctx, &src_d[0], src_d_0, src_d_1);
575 2720 operation().vector_path(ctx, src_a_0, src_b_0, src_c_0, src_d_0, &dst[0]);
576 5440 operation().vector_path(ctx, src_a_1, src_b_1, src_c_1, src_d_1,
577 2720 &dst.at(num_lanes())[0]);
578 2720 }
579
580 // Required method:
581 // void T::vector_path([ContextType,] const ScalarType *, ScalarType *);
582 template <typename T = ConcreteOperationType>
583 enable_if_has_vector_path_t<void, T, ContextType, const ScalarType *,
584 ScalarType *>
585 2534 vector_path_2x(ContextType ctx, ConstColumnType src,
586 ColumnType dst) KLEIDICV_STREAMING {
587 2534 operation().vector_path(ctx, &src[0], &dst[0]);
588 5068 operation().vector_path(ctx, &src.at(num_lanes())[0],
589 2534 &dst.at(num_lanes())[0]);
590 2534 }
591
592 // Required method:
593 // void T::vector_path([ContextType,] const ScalarType *, ScalarType *);
594 template <typename T = ConcreteOperationType>
595 enable_if_has_vector_path_t<void, T, ContextType, const ScalarType *,
596 const ScalarType *, ScalarType *>
597 vector_path_2x(ContextType ctx, ConstColumnType src_a, ConstColumnType src_b,
598 ColumnType dst) KLEIDICV_STREAMING {
599 operation().vector_path(ctx, &src_a[0], &src_b[0], &dst[0]);
600 operation().vector_path(ctx, &src_a.at(num_lanes())[0],
601 &src_b.at(num_lanes())[0], &dst.at(num_lanes())[0]);
602 }
603
604 // Required method:
605 // void T::vector_path([ContextType,] const ScalarType *, const ScalarType *,
606 // const ScalarType *, ScalarType *, ScalarType *);
607 template <typename T = ConcreteOperationType>
608 enable_if_has_vector_path_t<void, T, ContextType, const ScalarType *,
609 const ScalarType *, const ScalarType *,
610 ScalarType *, ScalarType *>
611 vector_path_2x(ContextType ctx, ConstColumnType src_a, ConstColumnType src_b,
612 ConstColumnType src_c, ColumnType dst_a,
613 ColumnType dst_b) KLEIDICV_STREAMING {
614 operation().vector_path(ctx, &src_a[0], &src_b[0], &src_c[0], &dst_a[0],
615 &dst_b[0]);
616 operation().vector_path(
617 ctx, &src_a.at(num_lanes())[0], &src_b.at(num_lanes())[0],
618 &src_c.at(num_lanes())[0], &dst_a.at(num_lanes())[0],
619 &dst_b.at(num_lanes())[0]);
620 }
621
622 // Required method:
623 // void T::vector_path([ContextType,] const ScalarType *, ScalarType *,
624 // ScalarType *);
625 template <typename T = ConcreteOperationType>
626 enable_if_has_vector_path_t<void, T, ContextType, const ScalarType *,
627 ScalarType *, ScalarType *>
628 vector_path_2x(ContextType ctx, ConstColumnType src, ColumnType dst_a,
629 ColumnType dst_b) KLEIDICV_STREAMING {
630 operation().vector_path(ctx, &src[0], &dst_a[0], &dst_b[0]);
631 operation().vector_path(ctx, &src.at(num_lanes())[0],
632 &dst_a.at(num_lanes())[0],
633 &dst_b.at(num_lanes())[0]);
634 }
635
636 // Required method:
637 // void T::vector_path([ContextType,] const ScalarType *, VectorType &,
638 // VectorType &);
639 template <typename T = ConcreteOperationType>
640 enable_if_has_vector_path_t<void, T, ContextType, const ScalarType *,
641 VectorType &, VectorType &>
642 vector_path_2x(ContextType ctx, ConstColumnType src, ColumnType dst_a,
643 ColumnType dst_b) KLEIDICV_STREAMING {
644 Vector2Type vdst_a, vdst_b;
645
646 operation().vector_path(ctx, &src[0], vdst_a.val[0], vdst_b.val[0]);
647 operation().vector_path(ctx, &src.at(num_lanes() * 2)[0], vdst_a.val[1],
648 vdst_b.val[1]);
649
650 store(vdst_a, &dst_a[0]);
651 store(vdst_b, &dst_b[0]);
652 }
653
654 // Required method:
655 // void T::vector_path([ContextType,] Vector2Type,
656 // VectorType &, VectorType &);
657 template <typename T = ConcreteOperationType>
658 enable_if_has_vector_path_t<void, T, ContextType, Vector2Type, VectorType &,
659 VectorType &>
660 800 vector_path_2x(ContextType ctx, ConstColumnType src, ColumnType dst_a,
661 ColumnType dst_b) KLEIDICV_STREAMING {
662 800 Vector2Type vsrc_0, vsrc_1;
663 800 Vector2Type vdst_a, vdst_b;
664
665 800 load_consecutive(&src[0], vsrc_0, vsrc_1);
666
667 800 operation().vector_path(ctx, vsrc_0, vdst_a.val[0], vdst_b.val[0]);
668 800 operation().vector_path(ctx, vsrc_1, vdst_a.val[1], vdst_b.val[1]);
669
670 800 store(vdst_a, &dst_a[0]);
671 800 store(vdst_b, &dst_b[0]);
672 800 }
673
674 // Required method:
675 // void T::vector_path([ContextType,] const ScalarType *, ScalarType *,
676 // ScalarType *, ScalarType *);
677 template <typename T = ConcreteOperationType>
678 enable_if_has_vector_path_t<void, T, ContextType, const ScalarType *,
679 ScalarType *, ScalarType *, ScalarType *>
680 vector_path_2x(ContextType ctx, ConstColumnType src, ColumnType dst_a,
681 ColumnType dst_b, ColumnType dst_c) KLEIDICV_STREAMING {
682 operation().vector_path(ctx, &src[0], &dst_a[0], &dst_b[0], &dst_c[0]);
683 operation().vector_path(
684 ctx, &src.at(num_lanes() * 3)[0], &dst_a.at(num_lanes())[0],
685 &dst_b.at(num_lanes())[0], &dst_c.at(num_lanes())[0]);
686 }
687
688 // Required method:
689 // void T::vector_path([ContextType,] const ScalarType *, VectorType &,
690 // VectorType &, VectorType &);
691 template <typename T = ConcreteOperationType>
692 enable_if_has_vector_path_t<void, T, ContextType, const ScalarType *,
693 VectorType &, VectorType &, VectorType &>
694 vector_path_2x(ContextType ctx, ConstColumnType src, ColumnType dst_a,
695 ColumnType dst_b, ColumnType dst_c) KLEIDICV_STREAMING {
696 Vector2Type vdst_a, vdst_b, vdst_c;
697
698 operation().vector_path(ctx, &src[0], vdst_a.val[0], vdst_b.val[0],
699 vdst_c.val[0]);
700 operation().vector_path(ctx, &src.at(num_lanes() * 3)[0], vdst_a.val[1],
701 vdst_b.val[1], vdst_c.val[1]);
702
703 store(vdst_a, &dst_a[0]);
704 store(vdst_b, &dst_b[0]);
705 store(vdst_c, &dst_c[0]);
706 }
707
708 // Required method:
709 // void T::vector_path([ContextType,] , Vector3Type , VectorType &
710 // VectorType &, VectorType &);
711 template <typename T = ConcreteOperationType>
712 enable_if_has_vector_path_t<void, T, ContextType, Vector3Type, VectorType &,
713 VectorType &, VectorType &>
714 1440 vector_path_2x(ContextType ctx, ConstColumnType src, ColumnType dst_a,
715 ColumnType dst_b, ColumnType dst_c) KLEIDICV_STREAMING {
716 1440 Vector3Type vsrc_0, vsrc_1;
717 1440 Vector2Type vdst_a, vdst_b, vdst_c;
718
719 1440 load_consecutive(&src[0], vsrc_0, vsrc_1);
720
721 2880 operation().vector_path(ctx, vsrc_0, vdst_a.val[0], vdst_b.val[0],
722 1440 vdst_c.val[0]);
723 2880 operation().vector_path(ctx, vsrc_1, vdst_a.val[1], vdst_b.val[1],
724 1440 vdst_c.val[1]);
725
726 1440 store(vdst_a, &dst_a[0]);
727 1440 store(vdst_b, &dst_b[0]);
728 1440 store(vdst_c, &dst_c[0]);
729 1440 }
730
731 // Required method:
732 // void T::vector_path([ContextType,] const ScalarType *, ScalarType *,
733 // ScalarType *, ScalarType *, ScalarType *);
734 template <typename T = ConcreteOperationType>
735 enable_if_has_vector_path_t<void, T, ContextType, const ScalarType *,
736 ScalarType *, ScalarType *, ScalarType *,
737 ScalarType *>
738 vector_path_2x(ContextType ctx, ConstColumnType src, ColumnType dst_a,
739 ColumnType dst_b, ColumnType dst_c,
740 ColumnType dst_d) KLEIDICV_STREAMING {
741 operation().vector_path(ctx, &src[0], &dst_a[0], &dst_b[0], &dst_c[0],
742 &dst_d[0]);
743 operation().vector_path(
744 ctx, &src.at(num_lanes() * 4)[0], &dst_a.at(num_lanes())[0],
745 &dst_b.at(num_lanes())[0], &dst_c.at(num_lanes())[0],
746 &dst_d.at(num_lanes())[0]);
747 }
748
749 // Required method:
750 // void T::vector_path([ContextType,] const ScalarType *,
751 // VectorType &, VectorType &, VectorType &,
752 // VectorType &);
753 // Operation with multiple destination address, and with store. This
754 // gives the possibility to store the 2 result vectors in 1 step
755 template <typename T = ConcreteOperationType>
756 enable_if_has_vector_path_t<void, T, ContextType, const ScalarType *,
757 VectorType &, VectorType &, VectorType &,
758 VectorType &>
759 vector_path_2x(ContextType ctx, ConstColumnType src, ColumnType dst_a,
760 ColumnType dst_b, ColumnType dst_c,
761 ColumnType dst_d) KLEIDICV_STREAMING {
762 Vector2Type vdst_a, vdst_b, vdst_c, vdst_d;
763
764 operation().vector_path(ctx, &src[0], vdst_a.val[0], vdst_b.val[0],
765 vdst_c.val[0], vdst_d.val[0]);
766
767 operation().vector_path(ctx, &src.at(num_lanes() * 4)[0], vdst_a.val[1],
768 vdst_b.val[1], vdst_c.val[1], vdst_d.val[1]);
769
770 store(vdst_a, &dst_a[0]);
771 store(vdst_b, &dst_b[0]);
772 store(vdst_c, &dst_c[0]);
773 store(vdst_d, &dst_d[0]);
774 }
775
776 // Required method:
777 // void T::vector_path([ContextType,] const Vector4Type
778 // VectorType &, VectorType &, VectorType &,
779 // VectorType &);
780 // Operation with multiple destination address, and with load and store.
781 // This gives the possibility to load the source x4 vectors and store
782 // the 2x4 result vectors in 1 step
783 template <typename T = ConcreteOperationType>
784 enable_if_has_vector_path_t<void, T, ContextType, const Vector4Type,
785 VectorType &, VectorType &, VectorType &,
786 VectorType &>
787 1440 vector_path_2x(ContextType ctx, ConstColumnType src, ColumnType dst_a,
788 ColumnType dst_b, ColumnType dst_c,
789 ColumnType dst_d) KLEIDICV_STREAMING {
790 1440 Vector4Type vsrc_0, vsrc_1;
791 1440 Vector2Type vdst_a, vdst_b, vdst_c, vdst_d;
792
793 1440 load_consecutive(&src[0], vsrc_0, vsrc_1);
794
795 2880 operation().vector_path(ctx, vsrc_0, vdst_a.val[0], vdst_b.val[0],
796 1440 vdst_c.val[0], vdst_d.val[0]);
797
798 2880 operation().vector_path(ctx, vsrc_1, vdst_a.val[1], vdst_b.val[1],
799 1440 vdst_c.val[1], vdst_d.val[1]);
800
801 1440 store(vdst_a, &dst_a[0]);
802 1440 store(vdst_b, &dst_b[0]);
803 1440 store(vdst_c, &dst_c[0]);
804 1440 store(vdst_d, &dst_d[0]);
805 1440 }
806
807 // ---------------------------------------------------------------------------
808 // Forwarding implementations for vector_path().
809 // ---------------------------------------------------------------------------
810
811 // Required method:
812 // void T::vector_path([ContextType,] VectorType);
813 template <typename T = ConcreteOperationType>
814 5237 enable_if_has_vector_path_t<void, T, ContextType, VectorType> vector_path(
815 ContextType ctx, ConstColumnType src) KLEIDICV_STREAMING {
816 5237 VectorType src_0;
817 5237 operation().load(ctx, &src[0], src_0);
818 5237 operation().vector_path(ctx, src_0);
819 5237 }
820
821 // Required method:
822 // VectorType T::vector_path([ContextType,] VectorType);
823 template <typename T = ConcreteOperationType>
824 enable_if_has_vector_path_t<VectorType, T, ContextType, VectorType>
825 13036 vector_path(ContextType ctx, ConstColumnType src,
826 ColumnType dst) KLEIDICV_STREAMING {
827 13036 VectorType src_0;
828 13036 operation().load(ctx, &src[0], src_0);
829 13036 VectorType res_0 = operation().vector_path(ctx, src_0);
830 13036 operation().store(ctx, res_0, &dst[0]);
831 13036 }
832
833 // Required method:
834 // VectorType T::vector_path([ContextType,] VectorType, VectorType);
835 template <typename T = ConcreteOperationType>
836 enable_if_has_vector_path_t<VectorType, T, ContextType, VectorType,
837 VectorType>
838 10325 vector_path(ContextType ctx, ConstColumnType src_a, ConstColumnType src_b,
839 ColumnType dst) KLEIDICV_STREAMING {
840 10325 VectorType src_a_0, src_b_0;
841 10325 operation().load(ctx, &src_a[0], src_a_0);
842 10325 operation().load(ctx, &src_b[0], src_b_0);
843 10325 VectorType res_0 = operation().vector_path(ctx, src_a_0, src_b_0);
844 10325 operation().store(ctx, res_0, &dst[0]);
845 10325 }
846
847 // Required method:
848 // void T::vector_path([ContextType,] VectorType, ScalarType *);
849 template <typename T = ConcreteOperationType>
850 enable_if_has_vector_path_t<void, T, ContextType, VectorType, ScalarType *>
851 4 vector_path(ContextType ctx, ConstColumnType src,
852 ColumnType dst) KLEIDICV_STREAMING {
853 4 VectorType src_0;
854 4 operation().load(ctx, &src[0], src_0);
855 4 operation().vector_path(ctx, src_0, &dst[0]);
856 4 }
857
858 // Required method:
859 // void T::vector_path([ContextType,] VectorType, VectorType, ScalarType *);
860 template <typename T = ConcreteOperationType>
861 enable_if_has_vector_path_t<void, T, ContextType, VectorType, VectorType,
862 ScalarType *>
863 vector_path(ContextType ctx, ConstColumnType src_a, ConstColumnType src_b,
864 ColumnType dst) KLEIDICV_STREAMING {
865 VectorType src_a_0, src_b_0;
866 operation().load(ctx, &src_a[0], src_a_0);
867 operation().load(ctx, &src_b[0], src_b_0);
868 operation().vector_path(ctx, src_a_0, src_b_0, &dst[0]);
869 }
870
871 // Required method:
872 // void T::vector_path([ContextType,] VectorType, VectorType, VectorType,
873 // ScalarType *);
874 template <typename T = ConcreteOperationType>
875 enable_if_has_vector_path_t<void, T, ContextType, VectorType, VectorType,
876 VectorType, ScalarType *>
877 vector_path(ContextType ctx, ConstColumnType src_a, ConstColumnType src_b,
878 ConstColumnType src_c, ColumnType dst) KLEIDICV_STREAMING {
879 VectorType src_a_0, src_b_0, src_c_0;
880 operation().load(ctx, &src_a[0], src_a_0);
881 operation().load(ctx, &src_b[0], src_b_0);
882 operation().load(ctx, &src_c[0], src_c_0);
883 operation().vector_path(ctx, src_a_0, src_b_0, src_c_0, &dst[0]);
884 }
885
886 // Required method:
887 // void T::vector_path([ContextType,] VectorType, VectorType, VectorType,
888 // ScalarType *, ScalarType *);
889 template <typename T = ConcreteOperationType>
890 enable_if_has_vector_path_t<void, T, ContextType, VectorType, VectorType,
891 VectorType, ScalarType *, ScalarType *>
892 96 vector_path(ContextType ctx, ConstColumnType src_a, ConstColumnType src_b,
893 ConstColumnType src_c, ColumnType dst_a,
894 ColumnType dst_b) KLEIDICV_STREAMING {
895 96 VectorType src_a_0, src_b_0, src_c_0;
896 96 operation().load(ctx, &src_a[0], src_a_0);
897 96 operation().load(ctx, &src_b[0], src_b_0);
898 96 operation().load(ctx, &src_c[0], src_c_0);
899 192 operation().vector_path(ctx, src_a_0, src_b_0, src_c_0, &dst_a[0],
900 96 &dst_b[0]);
901 96 }
902
903 // Required method:
904 // void T::vector_path([ContextType,] VectorType, VectorType, VectorType,
905 // VectorType, ScalarType *);
906 template <typename T = ConcreteOperationType>
907 enable_if_has_vector_path_t<void, T, ContextType, VectorType, VectorType,
908 VectorType, VectorType, ScalarType *>
909 vector_path(ContextType ctx, ConstColumnType src_a, ConstColumnType src_b,
910 ConstColumnType src_c, ConstColumnType src_d,
911 ColumnType dst) KLEIDICV_STREAMING {
912 VectorType src_a_0, src_b_0, src_c_0, src_d_0;
913 operation().load(ctx, &src_a[0], src_a_0);
914 operation().load(ctx, &src_b[0], src_b_0);
915 operation().load(ctx, &src_c[0], src_c_0);
916 operation().load(ctx, &src_d[0], src_d_0);
917 operation().vector_path(ctx, src_a_0, src_b_0, src_c_0, src_d_0, &dst[0]);
918 }
919
920 // Required method:
921 // void T::vector_path([ContextType,] const ScalarType *, ScalarType *);
922 template <typename T = ConcreteOperationType>
923 enable_if_has_vector_path_t<void, T, ContextType, const ScalarType *,
924 ScalarType *>
925 13498 vector_path(ContextType ctx, ConstColumnType src,
926 ColumnType dst) KLEIDICV_STREAMING {
927 13498 operation().vector_path(ctx, &src[0], &dst[0]);
928 13498 }
929
930 // Required method:
931 // void T::vector_path([ContextType,] const ScalarType *, const ScalarType *,
932 // ScalarType *);
933 template <typename T = ConcreteOperationType>
934 enable_if_has_vector_path_t<void, T, ContextType, const ScalarType *,
935 const ScalarType *, ScalarType *>
936 vector_path(ContextType ctx, ConstColumnType src_a, ConstColumnType src_b,
937 ColumnType dst) KLEIDICV_STREAMING {
938 operation().vector_path(ctx, &src_a[0], &src_b[0], &dst[0]);
939 }
940
941 // Required method:
942 // void T::vector_path([ContextType,] const ScalarType *, const ScalarType *,
943 // const ScalarType *, ScalarType *, ScalarType *);
944 template <typename T = ConcreteOperationType>
945 enable_if_has_vector_path_t<void, T, ContextType, const ScalarType *,
946 const ScalarType *, const ScalarType *,
947 ScalarType *, ScalarType *>
948 3008 vector_path(ContextType ctx, ConstColumnType src_a, ConstColumnType src_b,
949 ConstColumnType src_c, ColumnType dst_a,
950 ColumnType dst_b) KLEIDICV_STREAMING {
951 6016 operation().vector_path(ctx, &src_a[0], &src_b[0], &src_c[0], &dst_a[0],
952 3008 &dst_b[0]);
953 3008 }
954
955 // Required method:
956 // void T::vector_path([ContextType,] const ScalarType *, ScalarType *,
957 // ScalarType *);
958 template <typename T = ConcreteOperationType>
959 enable_if_has_vector_path_t<void, T, ContextType, const ScalarType *,
960 ScalarType *, ScalarType *>
961 vector_path(ContextType ctx, ConstColumnType src, ColumnType dst_a,
962 ColumnType dst_b) KLEIDICV_STREAMING {
963 operation().vector_path(ctx, &src[0], &dst_a[0], &dst_b[0]);
964 }
965
966 // Required method:
967 // void T::vector_path([ContextType,] const ScalarType *, VectorType &,
968 // VectorType &);
969 template <typename T = ConcreteOperationType>
970 enable_if_has_vector_path_t<void, T, ContextType, const ScalarType *,
971 VectorType &, VectorType &>
972 vector_path(ContextType ctx, ConstColumnType src, ColumnType dst_a,
973 ColumnType dst_b) KLEIDICV_STREAMING {
974 VectorType vdst_a, vdst_b;
975
976 operation().vector_path(ctx, &src[0], vdst_a, vdst_b);
977
978 store(vdst_a, &dst_a[0]);
979 store(vdst_b, &dst_b[0]);
980 }
981
982 // Required method:
983 // void T::vector_path([ContextType,] Vector2Type, VectorType &,
984 // VectorType &);
985 template <typename T = ConcreteOperationType>
986 enable_if_has_vector_path_t<void, T, ContextType, Vector2Type, VectorType &,
987 VectorType &>
988 vector_path(ContextType ctx, ConstColumnType src, ColumnType dst_a,
989 ColumnType dst_b) KLEIDICV_STREAMING {
990 VectorType vdst_a, vdst_b;
991 Vector2Type vsrc;
992
993 load(&src[0], vsrc);
994
995 operation().vector_path(ctx, vsrc, vdst_a, vdst_b);
996
997 store(vdst_a, &dst_a[0]);
998 store(vdst_b, &dst_b[0]);
999 }
1000
1001 // Required method:
1002 // void T::vector_path([ContextType,] const ScalarType *, ScalarType *,
1003 // ScalarType *, ScalarType *);
1004 template <typename T = ConcreteOperationType>
1005 enable_if_has_vector_path_t<void, T, ContextType, const ScalarType *,
1006 ScalarType *, ScalarType *, ScalarType *>
1007 vector_path(ContextType ctx, ConstColumnType src, ColumnType dst_a,
1008 ColumnType dst_b, ColumnType dst_c) KLEIDICV_STREAMING {
1009 operation().vector_path(ctx, &src[0], &dst_a[0], &dst_b[0], &dst_c[0]);
1010 }
1011
1012 // Required method:
1013 // void T::vector_path([ContextType,] const ScalarType *, VectorType &,
1014 // VectorType &, VectorType &);
1015 template <typename T = ConcreteOperationType>
1016 enable_if_has_vector_path_t<void, T, ContextType, const ScalarType *,
1017 VectorType &, VectorType &, VectorType &>
1018 vector_path(ContextType ctx, ConstColumnType src, ColumnType dst_a,
1019 ColumnType dst_b, ColumnType dst_c) KLEIDICV_STREAMING {
1020 VectorType vdst_a, vdst_b, vdst_c;
1021
1022 operation().vector_path(ctx, &src[0], vdst_a, vdst_b, vdst_c);
1023
1024 store(vdst_a, &dst_a[0]);
1025 store(vdst_b, &dst_b[0]);
1026 store(vdst_c, &dst_c[0]);
1027 }
1028
1029 // Required method:
1030 // void T::vector_path([ContextType,] Vector3Type, VectorType &,
1031 // VectorType &, VectorType &);
1032 template <typename T = ConcreteOperationType>
1033 enable_if_has_vector_path_t<void, T, ContextType, Vector3Type, VectorType &,
1034 VectorType &, VectorType &>
1035 vector_path(ContextType ctx, ConstColumnType src, ColumnType dst_a,
1036 ColumnType dst_b, ColumnType dst_c) KLEIDICV_STREAMING {
1037 VectorType vdst_a, vdst_b, vdst_c;
1038 Vector3Type vsrc;
1039
1040 load(&src[0], vsrc);
1041
1042 operation().vector_path(ctx, vsrc, vdst_a, vdst_b, vdst_c);
1043
1044 store(vdst_a, &dst_a[0]);
1045 store(vdst_b, &dst_b[0]);
1046 store(vdst_c, &dst_c[0]);
1047 }
1048
1049 // Required method:
1050 // void T::vector_path([ContextType,] const ScalarType *, ScalarType *,
1051 // ScalarType *, ScalarType *, ScalarType *);
1052 // Operation type with multiple destination, store in the vectorpath
1053 template <typename T = ConcreteOperationType>
1054 enable_if_has_vector_path_t<void, T, ContextType, const ScalarType *,
1055 ScalarType *, ScalarType *, ScalarType *,
1056 ScalarType *>
1057 vector_path(ContextType ctx, ConstColumnType src, ColumnType dst_a,
1058 ColumnType dst_b, ColumnType dst_c,
1059 ColumnType dst_d) KLEIDICV_STREAMING {
1060 operation().vector_path(ctx, &src[0], &dst_a[0], &dst_b[0], &dst_c[0],
1061 &dst_d[0]);
1062 }
1063
1064 // Required method:
1065 // void T::vector_path([ContextType,] const ScalarType *,
1066 // VectorType &, VectorType &, VectorType &,
1067 // VectorType &);
1068 // Operation type with multiple destination, and with store
1069 template <typename T = ConcreteOperationType>
1070 enable_if_has_vector_path_t<void, T, ContextType, const ScalarType *,
1071 VectorType &, VectorType &, VectorType &,
1072 VectorType &>
1073 vector_path(ContextType ctx, ConstColumnType src, ColumnType dst_a,
1074 ColumnType dst_b, ColumnType dst_c,
1075 ColumnType dst_d) KLEIDICV_STREAMING {
1076 VectorType vdst_a, vdst_b, vdst_c, vdst_d;
1077
1078 operation().vector_path(ctx, &src[0], vdst_a, vdst_b, vdst_c, vdst_d);
1079
1080 store(vdst_a, &dst_a[0]);
1081 store(vdst_b, &dst_b[0]);
1082 store(vdst_c, &dst_c[0]);
1083 store(vdst_d, &dst_d[0]);
1084 }
1085
1086 // Required method:
1087 // void T::vector_path([ContextType,] const Vector4Type,
1088 // VectorType &, VectorType &, VectorType &,
1089 // VectorType &);
1090 // Operation type with multiple destination, and with load and store
1091 template <typename T = ConcreteOperationType>
1092 enable_if_has_vector_path_t<void, T, ContextType, const Vector4Type,
1093 VectorType &, VectorType &, VectorType &,
1094 VectorType &>
1095 vector_path(ContextType ctx, ConstColumnType src, ColumnType dst_a,
1096 ColumnType dst_b, ColumnType dst_c,
1097 ColumnType dst_d) KLEIDICV_STREAMING {
1098 VectorType vdst_a, vdst_b, vdst_c, vdst_d;
1099 Vector4Type vsrc;
1100
1101 load(&src[0], vsrc);
1102
1103 operation().vector_path(ctx, vsrc, vdst_a, vdst_b, vdst_c, vdst_d);
1104
1105 store(vdst_a, &dst_a[0]);
1106 store(vdst_b, &dst_b[0]);
1107 store(vdst_c, &dst_c[0]);
1108 store(vdst_d, &dst_d[0]);
1109 }
1110
1111 // ---------------------------------------------------------------------------
1112 // Forwarding implementations for tail_path().
1113 // ---------------------------------------------------------------------------
1114
1115 // Required method:
1116 // void T::tail_path([ContextType,] VectorType, ScalarType *);
1117 template <typename T = ConcreteOperationType>
1118 enable_if_has_tail_path_t<void, T, ContextType, VectorType, ScalarType *>
1119 566 tail_path(ContextType ctx, ConstColumnType src,
1120 ColumnType dst) KLEIDICV_STREAMING {
1121 566 VectorType src_0;
1122 566 operation().load(ctx, &src[0], src_0);
1123 566 operation().tail_path(ctx, src_0, &dst[0]);
1124 566 }
1125
1126 // Required method:
1127 // void T::tail_path([ContextType,] const ScalarType *, ScalarType *);
1128 template <typename T = ConcreteOperationType>
1129 enable_if_has_tail_path_t<void, T, ContextType, const ScalarType *,
1130 ScalarType *>
1131 478 tail_path(ContextType ctx, ConstColumnType src,
1132 ColumnType dst) KLEIDICV_STREAMING {
1133 478 operation().tail_path(ctx, &src[0], &dst[0]);
1134 478 }
1135
1136 // ---------------------------------------------------------------------------
1137 // Forwarding implementations for scalar_path().
1138 // ---------------------------------------------------------------------------
1139
1140 // Required method:
1141 // void T::scalar_path([ContextType,] ScalarType);
1142 template <typename T = ConcreteOperationType>
1143 34073 enable_if_has_scalar_path_t<void, T, ContextType, ScalarType> scalar_path(
1144 ContextType ctx, ConstColumnType src) KLEIDICV_STREAMING {
1145 34073 operation().scalar_path(ctx, src[0]);
1146 34073 }
1147
1148 // Required method:
1149 // ScalarType T::scalar_path([ContextType,] ScalarType);
1150 template <typename T = ConcreteOperationType>
1151 enable_if_has_scalar_path_t<ScalarType, T, ContextType, ScalarType>
1152 8012 scalar_path(ContextType ctx, ConstColumnType src,
1153 ColumnType dst) KLEIDICV_STREAMING {
1154 8012 dst[0] = operation().scalar_path(ctx, src[0]);
1155 8012 }
1156
1157 // Required method:
1158 // ScalarType T::scalar_path([ContextType,] ScalarType, ScalarType);
1159 template <typename T = ConcreteOperationType>
1160 enable_if_has_scalar_path_t<ScalarType, T, ContextType, ScalarType,
1161 ScalarType>
1162 20434 scalar_path(ContextType ctx, ConstColumnType src_a, ConstColumnType src_b,
1163 ColumnType dst) KLEIDICV_STREAMING {
1164 20434 dst[0] = operation().scalar_path(ctx, src_a[0], src_b[0]);
1165 20434 }
1166
1167 // Required method:
1168 // void T::scalar_path([ContextType,] const ScalarType *, ScalarType *);
1169 template <typename T = ConcreteOperationType>
1170 enable_if_has_scalar_path_t<void, T, ContextType, const ScalarType *,
1171 ScalarType *>
1172 4887 scalar_path(ContextType ctx, ConstColumnType src,
1173 ColumnType dst) KLEIDICV_STREAMING {
1174 4887 operation().scalar_path(ctx, &src[0], &dst[0]);
1175 4887 }
1176
1177 // Required method:
1178 // void T::scalar_path([ContextType,] const ScalarType *, const ScalarType *,
1179 // ScalarType *);
1180 template <typename T = ConcreteOperationType>
1181 enable_if_has_scalar_path_t<void, T, ContextType, const ScalarType *,
1182 const ScalarType *, ScalarType *>
1183 256 scalar_path(ContextType ctx, ConstColumnType src_a, ConstColumnType src_b,
1184 ColumnType dst) KLEIDICV_STREAMING {
1185 256 operation().scalar_path(ctx, &src_a[0], &src_b[0], &dst[0]);
1186 256 }
1187
1188 // Required method:
1189 // void T::scalar_path([ContextType,] const ScalarType *, const ScalarType *,
1190 // const ScalarType *, ScalarType *);
1191 template <typename T = ConcreteOperationType>
1192 enable_if_has_scalar_path_t<void, T, ContextType, const ScalarType *,
1193 const ScalarType *, const ScalarType *,
1194 ScalarType *>
1195 512 scalar_path(ContextType ctx, ConstColumnType src_a, ConstColumnType src_b,
1196 ConstColumnType src_c, ColumnType dst) KLEIDICV_STREAMING {
1197 512 operation().scalar_path(ctx, &src_a[0], &src_b[0], &src_c[0], &dst[0]);
1198 512 }
1199
1200 // Required method:
1201 // void T::scalar_path([ContextType,] const ScalarType *, const ScalarType *,
1202 // const ScalarType *, const ScalarType *, ScalarType *);
1203 template <typename T = ConcreteOperationType>
1204 enable_if_has_scalar_path_t<void, T, ContextType, const ScalarType *,
1205 const ScalarType *, const ScalarType *,
1206 const ScalarType *, ScalarType *>
1207 1024 scalar_path(ContextType ctx, ConstColumnType src_a, ConstColumnType src_b,
1208 ConstColumnType src_c, ConstColumnType src_d,
1209 ColumnType dst) KLEIDICV_STREAMING {
1210 2048 operation().scalar_path(ctx, &src_a[0], &src_b[0], &src_c[0], &src_d[0],
1211 1024 &dst[0]);
1212 1024 }
1213
1214 // Required method:
1215 // void T::scalar_path([ContextType,] size_t, const ScalarType *,
1216 // const ScalarType *, const ScalarType *, ScalarType *,
1217 // ScalarType *);
1218 template <typename T = ConcreteOperationType>
1219 enable_if_has_scalar_path_t<void, T, ContextType, size_t, const ScalarType *,
1220 const ScalarType *, const ScalarType *,
1221 ScalarType *, ScalarType *>
1222 656 scalar_path(ContextType ctx, size_t length, ConstColumnType src_a,
1223 ConstColumnType src_b, ConstColumnType src_c, ColumnType dst_a,
1224 ColumnType dst_b) KLEIDICV_STREAMING {
1225 1312 operation().scalar_path(ctx, length, &src_a[0], &src_b[0], &src_c[0],
1226 656 &dst_a[0], &dst_b[0]);
1227 656 }
1228
1229 // Required method:
1230 // void T::scalar_path([ContextType,] const ScalarType *, ScalarType *,
1231 // ScalarType *);
1232 template <typename T = ConcreteOperationType>
1233 enable_if_has_scalar_path_t<void, T, ContextType, const ScalarType *,
1234 ScalarType *, ScalarType *>
1235 256 scalar_path(ContextType ctx, ConstColumnType src, ColumnType dst_a,
1236 ColumnType dst_b) KLEIDICV_STREAMING {
1237 256 operation().scalar_path(ctx, &src[0], &dst_a[0], &dst_b[0]);
1238 256 }
1239
1240 // Required method:
1241 // void T::scalar_path([ContextType,] const ScalarType *, ScalarType *,
1242 // ScalarType *, ScalarType *);
1243 template <typename T = ConcreteOperationType>
1244 enable_if_has_scalar_path_t<void, T, ContextType, const ScalarType *,
1245 ScalarType *, ScalarType *, ScalarType *>
1246 512 scalar_path(ContextType ctx, ConstColumnType src, ColumnType dst_a,
1247 ColumnType dst_b, ColumnType dst_c) KLEIDICV_STREAMING {
1248 512 operation().scalar_path(ctx, &src[0], &dst_a[0], &dst_b[0], &dst_c[0]);
1249 512 }
1250
1251 // Required method:
1252 // void T::scalar_path([ContextType,] const ScalarType *, ScalarType *,
1253 // ScalarType *, ScalarType *, ScalarType *);
1254 template <typename T = ConcreteOperationType>
1255 enable_if_has_scalar_path_t<void, T, ContextType, const ScalarType *,
1256 ScalarType *, ScalarType *, ScalarType *,
1257 ScalarType *>
1258 512 scalar_path(ContextType ctx, ConstColumnType src, ColumnType dst_a,
1259 ColumnType dst_b, ColumnType dst_c,
1260 ColumnType dst_d) KLEIDICV_STREAMING {
1261 1024 operation().scalar_path(ctx, &src[0], &dst_a[0], &dst_b[0], &dst_c[0],
1262 512 &dst_d[0]);
1263 512 }
1264 }; // end of class OperationAdapter<OperationType>
1265
1266 // Removes context before forwarding calls.
1267 template <typename OperationType>
1268 class RemoveContextAdapter : public OperationBase<OperationType> {
1269 public:
1270 using ContextType = typename OperationBase<OperationType>::ContextType;
1271
1272 8331 explicit RemoveContextAdapter(OperationType &operation) KLEIDICV_STREAMING
1273 8331 : OperationBase<OperationType>(operation) {}
1274
1275 template <typename... ArgTypes>
1276 3322 decltype(auto) load(ContextType, ArgTypes &&...args) KLEIDICV_STREAMING {
1277 3322 return OperationBase<OperationType>::load(std::forward<ArgTypes>(args)...);
1278 }
1279
1280 template <typename... ArgTypes>
1281 489603 decltype(auto) load_consecutive(ContextType,
1282 ArgTypes &&...args) KLEIDICV_STREAMING {
1283 489603 return OperationBase<OperationType>::load_consecutive(
1284 489603 std::forward<ArgTypes>(args)...);
1285 }
1286
1287 template <typename... ArgTypes>
1288 2978 decltype(auto) store(ContextType, ArgTypes &&...args) KLEIDICV_STREAMING {
1289 2978 return OperationBase<OperationType>::store(std::forward<ArgTypes>(args)...);
1290 }
1291
1292 template <typename... ArgTypes>
1293 28164 decltype(auto) store_consecutive(ContextType,
1294 ArgTypes &&...args) KLEIDICV_STREAMING {
1295 28164 return OperationBase<OperationType>::store_consecutive(
1296 28164 std::forward<ArgTypes>(args)...);
1297 }
1298
1299 template <typename... ArgTypes>
1300 931832 decltype(auto) vector_path(ContextType,
1301 ArgTypes &&...args) KLEIDICV_STREAMING {
1302 931832 return this->operation().vector_path(std::forward<ArgTypes>(args)...);
1303 }
1304
1305 // Forwards remaining_path() calls to the inner operation.
1306 template <typename... ArgTypes>
1307 decltype(auto) remaining_path(ContextType,
1308 ArgTypes &&...args) KLEIDICV_STREAMING {
1309 return this->operation().remaining_path(std::forward<ArgTypes>(args)...);
1310 }
1311
1312 template <typename... ArgTypes>
1313 decltype(auto) tail_path(ContextType, ArgTypes &&...args) KLEIDICV_STREAMING {
1314 return this->operation().tail_path(std::forward<ArgTypes>(args)...);
1315 }
1316
1317 template <typename... ArgTypes>
1318 71134 decltype(auto) scalar_path(ContextType,
1319 ArgTypes &&...args) KLEIDICV_STREAMING {
1320 71134 return this->operation().scalar_path(std::forward<ArgTypes>(args)...);
1321 }
1322 }; // end of class RemoveContextAdapter<OperationType>
1323
1324 } // namespace KLEIDICV_TARGET_NAMESPACE
1325
1326 #endif // KLEIDICV_OPERATIONS_H
1327