KleidiCV Coverage Report

Directory: ./
File: kleidicv/include/kleidicv/operations.h
Date: 2025-11-25 17:23:32
Exec Total Coverage
Lines: 286 286 100.0%
Functions: 6551 6601 99.2%
Branches: 10 10 100.0%
Line Branch Exec Source
1 // SPDX-FileCopyrightText: 2023 - 2025 Arm Limited and/or its affiliates <open-source-office@arm.com>
2 //
3 // SPDX-License-Identifier: Apache-2.0
4
5 #ifndef KLEIDICV_OPERATIONS_H
6 #define KLEIDICV_OPERATIONS_H
7
8 #include <algorithm>
9 #include <utility>
10
11 #include "kleidicv/traits.h"
12 #include "kleidicv/types.h"
13
14 namespace KLEIDICV_TARGET_NAMESPACE {
15
16 // Base for classes that change the behaviour or extend the interface of an
17 // operation.
18 //
19 // This class conveniently forwards common types and methods without having to
20 // explicitly define them in all derived classes.
21 template <typename T>
22 class OperationBase {
23 public:
24 // Type of the inner operation.
25 using OperationType = T;
26
27 // The vector traits used by the inner operation.
28 using VecTraits = typename OperationType::VecTraits;
29
30 // The context type used by the inner operation, if any.
31 using ContextType = context_type_t<OperationType>;
32
33 // Returns a reference to the inner operation.
34 4716345 OperationType &operation() KLEIDICV_STREAMING { return operation_; }
35
36 // Forwards num_lanes() calls to the inner operation.
37 91690 static size_t num_lanes() KLEIDICV_STREAMING {
38 91690 return VecTraits::num_lanes();
39 }
40
41 // Forwards vector_path_2x() calls to the inner operation.
42 template <typename... ArgTypes>
43 515694 decltype(auto) vector_path_2x(ArgTypes &&...args) KLEIDICV_STREAMING {
44 515694 return operation().vector_path_2x(std::forward<ArgTypes>(args)...);
45 }
46
47 // Forwards vector_path() calls to the inner operation.
48 template <typename... ArgTypes>
49 152715 decltype(auto) vector_path(ArgTypes &&...args) KLEIDICV_STREAMING {
50 152715 return operation().vector_path(std::forward<ArgTypes>(args)...);
51 }
52
53 // Forwards remaining_path() calls to the inner operation.
54 template <typename... ArgTypes>
55 decltype(auto) remaining_path(ArgTypes &&...args) KLEIDICV_STREAMING {
56 return operation().remaining_path(std::forward<ArgTypes>(args)...);
57 }
58
59 // Forwards tail_path() calls to the inner operation.
60 template <typename... ArgTypes>
61 708 decltype(auto) tail_path(ArgTypes &&...args) KLEIDICV_STREAMING {
62 708 return operation().tail_path(std::forward<ArgTypes>(args)...);
63 }
64
65 // Forwards scalar_path() calls to the inner operation.
66 template <typename... ArgTypes>
67 decltype(auto) scalar_path(ArgTypes &&...args) KLEIDICV_STREAMING {
68 return operation().scalar_path(std::forward<ArgTypes>(args)...);
69 }
70
71 template <typename... ArgTypes>
72 33295 decltype(auto) load(ArgTypes &&...args) KLEIDICV_STREAMING {
73 33295 return VecTraits::load(std::forward<ArgTypes>(args)...);
74 }
75
76 template <typename... ArgTypes>
77 577910 decltype(auto) load_consecutive(ArgTypes &&...args) KLEIDICV_STREAMING {
78 577910 return VecTraits::load_consecutive(std::forward<ArgTypes>(args)...);
79 }
80
81 template <typename... ArgTypes>
82 31704 decltype(auto) store(ArgTypes &&...args) KLEIDICV_STREAMING {
83 31704 return VecTraits::store(std::forward<ArgTypes>(args)...);
84 }
85
86 template <typename... ArgTypes>
87 74110 decltype(auto) store_consecutive(ArgTypes &&...args) KLEIDICV_STREAMING {
88 74110 return VecTraits::store_consecutive(std::forward<ArgTypes>(args)...);
89 }
90
91 // Forwards max_vectors_per_block() calls to the inner operation.
92 6660 size_t max_vectors_per_block() KLEIDICV_STREAMING {
93 6660 return operation_.max_vectors_per_block();
94 }
95
96 // Forwards on_block_finished() calls to the inner operation.
97 22530 void on_block_finished(size_t vectors_in_block) KLEIDICV_STREAMING {
98 22530 return operation_.on_block_finished(vectors_in_block);
99 }
100
101 // Returns true if the innermost operation is unrolled twice, otherwise false.
102 static constexpr bool is_unrolled_twice() KLEIDICV_STREAMING {
103 return ::KLEIDICV_TARGET_NAMESPACE::is_unrolled_twice<
104 concrete_operation_type_t<OperationType>>;
105 }
106
107 // Returns true if the innermost operation is unrolled once, otherwise false.
108 static constexpr bool is_unrolled_once() KLEIDICV_STREAMING {
109 return ::KLEIDICV_TARGET_NAMESPACE::is_unrolled_once<
110 concrete_operation_type_t<OperationType>>;
111 }
112
113 // Returns true if the innermost operation uses tail path, otherwise false.
114 static constexpr bool uses_tail_path() KLEIDICV_STREAMING {
115 return ::KLEIDICV_TARGET_NAMESPACE::uses_tail_path<
116 concrete_operation_type_t<OperationType>>;
117 }
118
119 // Returns true if the innermost operation tries to avoid tail loop, otherwise
120 // false.
121 static constexpr bool try_to_avoid_tail_loop() KLEIDICV_STREAMING {
122 return ::KLEIDICV_TARGET_NAMESPACE::try_to_avoid_tail_loop<
123 concrete_operation_type_t<OperationType>>;
124 }
125
126 protected:
127 // Constructor is protected so that only derived classes can instantiate.
128 147900 explicit OperationBase(OperationType &operation) KLEIDICV_STREAMING
129 147900 : operation_{operation} {}
130
131 private:
132 // Reference to the inner operation.
133 OperationType &operation_;
134 }; // end of class OperationBase<OperationType>
135
136 // Mixin which simply forwards all operations as-is.
137 template <typename OperationType>
138 class ForwardingOperation : public OperationBase<OperationType> {
139 public:
140 22021 explicit ForwardingOperation(OperationType &operation) KLEIDICV_STREAMING
141 22021 : OperationBase<OperationType>(operation) {}
142 }; // end of class ForwardingOperation<OperationType>
143
144 // Facade to offer a simplified row-based operation interface.
145 //
146 // A row-based operation is executed row-by-row and is unrolled within the row.
147 //
148 // Facade layout:
149 // |- operation.vector_path_<N>x()
150 // process_row() -|- operation.vector_path()
151 // |- operation.remaining_path()
152 // |- operation.num_lanes()
153 template <typename OperationType>
154 class RowBasedOperation : public OperationBase<OperationType> {
155 public:
156 28652 explicit RowBasedOperation(OperationType &operation) KLEIDICV_STREAMING
157 28652 : OperationBase<OperationType>(operation) {}
158
159 // NOLINTBEGIN(cppcoreguidelines-avoid-goto, hicpp-avoid-goto)
160 template <typename... ColumnTypes>
161 38023 void process_row(size_t length, ColumnTypes... columns) KLEIDICV_STREAMING {
162 38023 LoopUnroll loop{length, this->num_lanes()};
163
164 // clang-format off
165
166 38023 loop.unroll_twice_if<OperationType::is_unrolled_twice()>(
167 131245 [&](size_t step) KLEIDICV_STREAMING {
168 93222 this->operation().vector_path_2x(columns...);
169 93222 ((columns += step), ...);
170 93222 });
171
172 avoid_tail_loop:
173
174 38216 loop.unroll_once_if<OperationType::is_unrolled_once()>(
175 51698 [&](size_t step) KLEIDICV_STREAMING {
176 13482 this->operation().vector_path(columns...);
177 13482 ((columns += step), ...);
178 13482 });
179
180 // Process leftover bytes on the unrolled-once vector path, if requested and
181 // possible.
182 if constexpr (OperationType::is_unrolled_once() && OperationType::try_to_avoid_tail_loop()) {
183
8/8
✓ Branch 0 taken 67 times.
✓ Branch 1 taken 440 times.
✓ Branch 2 taken 42 times.
✓ Branch 3 taken 340 times.
✓ Branch 4 taken 42 times.
✓ Branch 5 taken 340 times.
✓ Branch 6 taken 42 times.
✓ Branch 7 taken 340 times.
 1653 if (loop.try_avoid_tail_loop(
184 1846 [&](size_t backward_step) KLEIDICV_STREAMING {
185 // Adjust pointers backwards to include
186 // the leftover bytes.
187 193 ((columns -= backward_step), ...);
188 193 })) {
189 193 goto avoid_tail_loop;
190 }
191 }
192
193 38023 loop.remaining(
194 70215 [&](size_t length, size_t /* step */) KLEIDICV_STREAMING {
195 32192 this->operation().remaining_path(length, columns...);
196 32192 });
197
198 // clang-format on
199 38023 }
200 // NOLINTEND(cppcoreguidelines-avoid-goto, hicpp-avoid-goto)
201 }; // end of class RowBasedOperation<OperationType>
202
203 // Facade to offer a simplified row-based operation interface.
204 //
205 // A row-based operation is executed row-by-row, is unrolled within the row and
206 // split into blocks.
207 //
208 // Facade layout:
209 // |- operation.vector_path_<N>x()
210 // process_row() -|- operation.vector_path()
211 // |- operation.remaining_path()
212 // |- operation.num_lanes()
213 // |- operation.max_vectors_per_block()
214 // |- operation.on_block_finished()
215 template <typename OperationType>
216 class RowBasedBlockOperation : public OperationBase<OperationType> {
217 public:
218 624 explicit RowBasedBlockOperation(OperationType &operation) KLEIDICV_STREAMING
219 624 : OperationBase<OperationType>(operation) {}
220
221 template <typename... ColumnTypes>
222 1332 void process_row(size_t length, ColumnTypes... columns) KLEIDICV_STREAMING {
223 if constexpr (OperationType::is_unrolled_twice()) {
224 423804 process_blocks<2>(length, [&](size_t step) KLEIDICV_STREAMING {
225 422472 this->operation().vector_path_2x(columns...);
226 422472 ((columns += step), ...);
227 422472 });
228 }
229
230 if constexpr (OperationType::is_unrolled_once()) {
231 process_blocks<1>(length, [&](size_t step) KLEIDICV_STREAMING {
232 this->operation().vector_path(columns...);
233 ((columns += step), ...);
234 });
235 }
236
237 // clang-format off
238 1332 LoopUnroll loop{length, this->num_lanes()};
239 1332 loop.remaining(
240 2256 [&](size_t length, size_t /* step */) KLEIDICV_STREAMING {
241 924 this->operation().remaining_path(length, columns...);
242 924 });
243 // clang-format on
244 1332 }
245
246 private:
247 template <size_t UnrollFactor, typename CallbackType>
248 1332 void process_blocks(size_t &length,
249 CallbackType callback) KLEIDICV_STREAMING {
250 // The number of elements a single iteration would process.
251 1332 const size_t elements_per_iteration = UnrollFactor * this->num_lanes();
252 // The number of elements which will be processed when this method returns.
253 2664 const size_t processed_length =
254 1332 (length / elements_per_iteration) * elements_per_iteration;
255 // The number of vectors which can be processed safely with the current
256 // unroll factor.
257 2664 const size_t max_vectors_per_block =
258 1332 (this->max_vectors_per_block() / UnrollFactor) * UnrollFactor;
259
260 // Process data up to allowed number of blocks.
261
2/2
✓ Branch 0 taken 1332 times.
✓ Branch 1 taken 4506 times.
 5838 for (size_t index = 0, block_length = 0; index < processed_length;
262 4506 index += block_length) {
263 // Process max_vectors_per_block vectors, or less in the last iteration
264 // (remaining size).
265 4506 size_t remaining_vectors = (processed_length - index) / this->num_lanes();
266 9012 size_t vectors_in_block =
267 4506 std::min(max_vectors_per_block, remaining_vectors);
268 4506 block_length = vectors_in_block * this->num_lanes();
269
270 // clang-format off
271 // Process data with the appropriate unroll factor.
272 4506 LoopUnroll loop{block_length, this->num_lanes()};
273 4506 loop.unroll_n_times<UnrollFactor>(
274 426978 [&](size_t step) KLEIDICV_STREAMING {
275 422472 callback(step);
276 // Adjust remaining length here.
277 // This improves generated code.
278 422472 length -= step;
279 422472 });
280 // clang-format on
281
282 // Notify the operation that a block was processed.
283 4506 this->on_block_finished(vectors_in_block);
284 4506 }
285 1332 }
286 }; // end of class RowBasedBlockOperation<OperationType>
287
288 // Adapter to offer an interface which allows users to use ParallelRows and to
289 // unpack them.
290 template <typename OperationType>
291 class ParallelRowsAdapter : public OperationBase<OperationType> {
292 public:
293 using VecTraits = typename OperationBase<OperationType>::VecTraits;
294 using ScalarType = typename VecTraits::ScalarType;
295 using ConstParallelColumnType = ParallelColumns<const ScalarType>;
296 using ParallelColumnType = ParallelColumns<ScalarType>;
297 using ConstColumnType = Columns<const ScalarType>;
298
299 1520 explicit ParallelRowsAdapter(OperationType &operation) KLEIDICV_STREAMING
300 1520 : OperationBase<OperationType>(operation) {}
301
302 // Forwards vector_path_2x() calls to the inner operation with one source and
303 // destination parallel columns.
304 void vector_path_2x(ConstParallelColumnType src_a, ConstColumnType src_b,
305 ParallelColumnType dst) KLEIDICV_STREAMING {
306 this->operation().vector_path_2x(src_a.first(), src_a.second(), src_b,
307 dst.first(), dst.second());
308 }
309
310 // Forwards vector_path() calls to the inner operation with one source and
311 // destination parallel columns.
312 96 void vector_path(ConstParallelColumnType src_a, ConstColumnType src_b,
313 ParallelColumnType dst) KLEIDICV_STREAMING {
314 192 this->operation().vector_path(src_a.first(), src_a.second(), src_b,
315 96 dst.first(), dst.second());
316 96 }
317
318 // Forwards remaining_path() calls to the inner operation with one source and
319 // destination parallel columns.
320 3472 void remaining_path(size_t length, ConstParallelColumnType src_a,
321 ConstColumnType src_b,
322 ParallelColumnType dst) KLEIDICV_STREAMING {
323 6944 this->operation().remaining_path(length, src_a.first(), src_a.second(),
324 3472 src_b, dst.first(), dst.second());
325 3472 }
326 }; // end of class ParallelRowsAdapter<OperationType>
327
328 // Adapter to offer an interface which allows users to apply unrolling to the
329 // adaptee operation. This adapter binds to the interface of the innermost
330 // operation in the chain.
331 //
332 // Provides specialized implementations using SFINAE principle for
333 // - vector_path_2x(),
334 // - vector_path(), and
335 // - tail_path(), and
336 // - scalar_path()
337 // methods.
338 template <typename OperationType>
339 class OperationAdapter : public OperationBase<OperationType> {
340 // Shorten rows: no need to write 'this->'.
341 using OperationBase<OperationType>::operation;
342 using OperationBase<OperationType>::num_lanes;
343 using OperationBase<OperationType>::load;
344 using OperationBase<OperationType>::load_consecutive;
345 using OperationBase<OperationType>::store;
346 using OperationBase<OperationType>::store_consecutive;
347 // The innermost operation in the chain.
348 using ConcreteOperationType = concrete_operation_type_t<OperationType>;
349
350 public:
351 using VecTraits = typename OperationBase<OperationType>::VecTraits;
352 using ContextType = typename OperationBase<OperationType>::ContextType;
353 using ScalarType = typename VecTraits::ScalarType;
354 using VectorType = typename VecTraits::VectorType;
355 using Vector2Type = typename VecTraits::Vector2Type;
356 using Vector3Type = typename VecTraits::Vector3Type;
357 using Vector4Type = typename VecTraits::Vector4Type;
358 using ConstColumnType = Columns<const ScalarType>;
359 using ColumnType = Columns<ScalarType>;
360
361 29276 explicit OperationAdapter(OperationType &operation) KLEIDICV_STREAMING
362 29276 : OperationBase<OperationType>(operation) {}
363
364 // ---------------------------------------------------------------------------
365 // Forwarding implementations for vector_path_2x().
366 // ---------------------------------------------------------------------------
367
368 // Required method:
369 // void T::vector_path([ContextType,] VectorType);
370 template <typename T = ConcreteOperationType>
371 429564 enable_if_has_vector_path_t<void, T, ContextType, VectorType> vector_path_2x(
372 ContextType ctx, ConstColumnType src) KLEIDICV_STREAMING {
373 429564 VectorType src_0, src_1;
374 429564 operation().load_consecutive(ctx, &src[0], src_0, src_1);
375 429564 operation().vector_path(ctx, src_0);
376 429564 operation().vector_path(ctx, src_1);
377 429564 }
378
379 // Required method:
380 // VectorType T::vector_path([ContextType,] VectorType);
381 template <typename T = ConcreteOperationType>
382 enable_if_has_vector_path_t<VectorType, T, ContextType, VectorType>
383 21020 vector_path_2x(ContextType ctx, ConstColumnType src,
384 ColumnType dst) KLEIDICV_STREAMING {
385 21020 VectorType src_0, src_1;
386 21020 operation().load_consecutive(ctx, &src[0], src_0, src_1);
387 21020 VectorType res_0 = operation().vector_path(ctx, src_0);
388 21020 VectorType res_1 = operation().vector_path(ctx, src_1);
389 21020 operation().store_consecutive(ctx, res_0, res_1, &dst[0]);
390 21020 }
391
392 // Required method:
393 // VectorType T::vector_path([ContextType,] VectorType, VectorType);
394 template <typename T = ConcreteOperationType>
395 enable_if_has_vector_path_t<VectorType, T, ContextType, VectorType,
396 VectorType>
397 53090 vector_path_2x(ContextType ctx, ConstColumnType src_a, ConstColumnType src_b,
398 ColumnType dst) KLEIDICV_STREAMING {
399 53090 VectorType src_a_0, src_a_1, src_b_0, src_b_1;
400 53090 operation().load_consecutive(ctx, &src_a[0], src_a_0, src_a_1);
401 53090 operation().load_consecutive(ctx, &src_b[0], src_b_0, src_b_1);
402 53090 VectorType res_0 = operation().vector_path(ctx, src_a_0, src_b_0);
403 53090 VectorType res_1 = operation().vector_path(ctx, src_a_1, src_b_1);
404 53090 operation().store_consecutive(ctx, res_0, res_1, &dst[0]);
405 53090 }
406
407 // Required method:
408 // void T::vector_path([ContextType,] VectorType, ScalarType *);
409 template <typename T = ConcreteOperationType>
410 enable_if_has_vector_path_t<void, T, ContextType, VectorType, ScalarType *>
411 666 vector_path_2x(ContextType ctx, ConstColumnType src,
412 ColumnType dst) KLEIDICV_STREAMING {
413 666 VectorType src_0, src_1;
414 666 operation().load_consecutive(ctx, &src[0], src_0, src_1);
415 666 operation().vector_path(ctx, src_0, &dst[0]);
416 666 operation().vector_path(ctx, src_1, &dst.at(num_lanes())[0]);
417 666 }
418
419 // Required method:
420 // void T::vector_path([ContextType,] VectorType, VectorType, ScalarType *);
421 template <typename T = ConcreteOperationType>
422 enable_if_has_vector_path_t<void, T, ContextType, VectorType, VectorType,
423 ScalarType *>
424 800 vector_path_2x(ContextType ctx, ConstColumnType src_a, ConstColumnType src_b,
425 ColumnType dst) KLEIDICV_STREAMING {
426 800 VectorType src_a_0, src_a_1, src_b_0, src_b_1;
427 800 operation().load_consecutive(ctx, &src_a[0], src_a_0, src_a_1);
428 800 operation().load_consecutive(ctx, &src_b[0], src_b_0, src_b_1);
429 800 operation().vector_path(ctx, src_a_0, src_b_0, &dst[0]);
430 800 operation().vector_path(ctx, src_a_1, src_b_1, &dst.at(num_lanes())[0]);
431 800 }
432
433 // Required method:
434 // void T::vector_path([ContextType,] VectorType, VectorType, VectorType,
435 // ScalarType *);
436 template <typename T = ConcreteOperationType>
437 enable_if_has_vector_path_t<void, T, ContextType, VectorType, VectorType,
438 VectorType, ScalarType *>
439 1440 vector_path_2x(ContextType ctx, ConstColumnType src_a, ConstColumnType src_b,
440 ConstColumnType src_c, ColumnType dst) KLEIDICV_STREAMING {
441 1440 VectorType src_a_0, src_a_1, src_b_0, src_b_1, src_c_0, src_c_1;
442 1440 operation().load_consecutive(ctx, &src_a[0], src_a_0, src_a_1);
443 1440 operation().load_consecutive(ctx, &src_b[0], src_b_0, src_b_1);
444 1440 operation().load_consecutive(ctx, &src_c[0], src_c_0, src_c_1);
445 1440 operation().vector_path(ctx, src_a_0, src_b_0, src_c_0, &dst[0]);
446 2880 operation().vector_path(ctx, src_a_1, src_b_1, src_c_1,
447 1440 &dst.at(num_lanes())[0]);
448 1440 }
449
450 // Required method:
451 // void T::vector_path([ContextType,] VectorType, VectorType, VectorType,
452 // ScalarType *,
453 // ScalarType *);
454 template <typename T = ConcreteOperationType>
455 enable_if_has_vector_path_t<void, T, ContextType, VectorType, VectorType,
456 VectorType, ScalarType *, ScalarType *>
457 vector_path_2x(ContextType ctx, ConstColumnType src_a, ConstColumnType src_b,
458 ConstColumnType src_c, ColumnType dst_a,
459 ColumnType dst_b) KLEIDICV_STREAMING {
460 VectorType src_a_0, src_a_1, src_b_0, src_b_1, src_c_0, src_c_1;
461 operation().load_consecutive(ctx, &src_a[0], src_a_0, src_a_1);
462 operation().load_consecutive(ctx, &src_b[0], src_b_0, src_b_1);
463 operation().load_consecutive(ctx, &src_c[0], src_c_0, src_c_1);
464 operation().vector_path(ctx, src_a_0, src_b_0, src_c_0, &dst_a[0],
465 &dst_b[0]);
466 operation().vector_path(ctx, src_a_1, src_b_1, src_c_1,
467 &dst_a.at(num_lanes())[0],
468 &dst_b.at(num_lanes())[0]);
469 }
470
471 // Required method:
472 // void T::vector_path([ContextType,] VectorType, VectorType, VectorType,
473 // VectorType,
474 // ScalarType *);
475 template <typename T = ConcreteOperationType>
476 enable_if_has_vector_path_t<void, T, ContextType, VectorType, VectorType,
477 VectorType, VectorType, ScalarType *>
478 2720 vector_path_2x(ContextType ctx, ConstColumnType src_a, ConstColumnType src_b,
479 ConstColumnType src_c, ConstColumnType src_d,
480 ColumnType dst) KLEIDICV_STREAMING {
481 2720 VectorType src_a_0, src_a_1, src_b_0, src_b_1;
482 2720 VectorType src_c_0, src_c_1, src_d_0, src_d_1;
483 2720 operation().load_consecutive(ctx, &src_a[0], src_a_0, src_a_1);
484 2720 operation().load_consecutive(ctx, &src_b[0], src_b_0, src_b_1);
485 2720 operation().load_consecutive(ctx, &src_c[0], src_c_0, src_c_1);
486 2720 operation().load_consecutive(ctx, &src_d[0], src_d_0, src_d_1);
487 2720 operation().vector_path(ctx, src_a_0, src_b_0, src_c_0, src_d_0, &dst[0]);
488 5440 operation().vector_path(ctx, src_a_1, src_b_1, src_c_1, src_d_1,
489 2720 &dst.at(num_lanes())[0]);
490 2720 }
491
492 // Required method:
493 // void T::vector_path([ContextType,] const ScalarType *, ScalarType *);
494 template <typename T = ConcreteOperationType>
495 enable_if_has_vector_path_t<void, T, ContextType, const ScalarType *,
496 ScalarType *>
497 2714 vector_path_2x(ContextType ctx, ConstColumnType src,
498 ColumnType dst) KLEIDICV_STREAMING {
499 2714 operation().vector_path(ctx, &src[0], &dst[0]);
500 5428 operation().vector_path(ctx, &src.at(num_lanes())[0],
501 2714 &dst.at(num_lanes())[0]);
502 2714 }
503
504 // Required method:
505 // void T::vector_path([ContextType,] const ScalarType *, ScalarType *);
506 template <typename T = ConcreteOperationType>
507 enable_if_has_vector_path_t<void, T, ContextType, const ScalarType *,
508 const ScalarType *, ScalarType *>
509 vector_path_2x(ContextType ctx, ConstColumnType src_a, ConstColumnType src_b,
510 ColumnType dst) KLEIDICV_STREAMING {
511 operation().vector_path(ctx, &src_a[0], &src_b[0], &dst[0]);
512 operation().vector_path(ctx, &src_a.at(num_lanes())[0],
513 &src_b.at(num_lanes())[0], &dst.at(num_lanes())[0]);
514 }
515
516 // Required method:
517 // void T::vector_path([ContextType,] const ScalarType *, const ScalarType *,
518 // const ScalarType *, ScalarType *, ScalarType *);
519 template <typename T = ConcreteOperationType>
520 enable_if_has_vector_path_t<void, T, ContextType, const ScalarType *,
521 const ScalarType *, const ScalarType *,
522 ScalarType *, ScalarType *>
523 vector_path_2x(ContextType ctx, ConstColumnType src_a, ConstColumnType src_b,
524 ConstColumnType src_c, ColumnType dst_a,
525 ColumnType dst_b) KLEIDICV_STREAMING {
526 operation().vector_path(ctx, &src_a[0], &src_b[0], &src_c[0], &dst_a[0],
527 &dst_b[0]);
528 operation().vector_path(
529 ctx, &src_a.at(num_lanes())[0], &src_b.at(num_lanes())[0],
530 &src_c.at(num_lanes())[0], &dst_a.at(num_lanes())[0],
531 &dst_b.at(num_lanes())[0]);
532 }
533
534 // Required method:
535 // void T::vector_path([ContextType,] const ScalarType *, ScalarType *,
536 // ScalarType *);
537 template <typename T = ConcreteOperationType>
538 enable_if_has_vector_path_t<void, T, ContextType, const ScalarType *,
539 ScalarType *, ScalarType *>
540 vector_path_2x(ContextType ctx, ConstColumnType src, ColumnType dst_a,
541 ColumnType dst_b) KLEIDICV_STREAMING {
542 operation().vector_path(ctx, &src[0], &dst_a[0], &dst_b[0]);
543 operation().vector_path(ctx, &src.at(num_lanes())[0],
544 &dst_a.at(num_lanes())[0],
545 &dst_b.at(num_lanes())[0]);
546 }
547
548 // Required method:
549 // void T::vector_path([ContextType,] const ScalarType *, VectorType &,
550 // VectorType &);
551 template <typename T = ConcreteOperationType>
552 enable_if_has_vector_path_t<void, T, ContextType, const ScalarType *,
553 VectorType &, VectorType &>
554 vector_path_2x(ContextType ctx, ConstColumnType src, ColumnType dst_a,
555 ColumnType dst_b) KLEIDICV_STREAMING {
556 Vector2Type vdst_a, vdst_b;
557
558 operation().vector_path(ctx, &src[0], vdst_a.val[0], vdst_b.val[0]);
559 operation().vector_path(ctx, &src.at(num_lanes() * 2)[0], vdst_a.val[1],
560 vdst_b.val[1]);
561
562 store(vdst_a, &dst_a[0]);
563 store(vdst_b, &dst_b[0]);
564 }
565
566 // Required method:
567 // void T::vector_path([ContextType,] Vector2Type,
568 // VectorType &, VectorType &);
569 template <typename T = ConcreteOperationType>
570 enable_if_has_vector_path_t<void, T, ContextType, Vector2Type, VectorType &,
571 VectorType &>
572 800 vector_path_2x(ContextType ctx, ConstColumnType src, ColumnType dst_a,
573 ColumnType dst_b) KLEIDICV_STREAMING {
574 800 Vector2Type vsrc_0, vsrc_1;
575 800 Vector2Type vdst_a, vdst_b;
576
577 800 load_consecutive(&src[0], vsrc_0, vsrc_1);
578
579 800 operation().vector_path(ctx, vsrc_0, vdst_a.val[0], vdst_b.val[0]);
580 800 operation().vector_path(ctx, vsrc_1, vdst_a.val[1], vdst_b.val[1]);
581
582 800 store(vdst_a, &dst_a[0]);
583 800 store(vdst_b, &dst_b[0]);
584 800 }
585
586 // Required method:
587 // void T::vector_path([ContextType,] const ScalarType *, ScalarType *,
588 // ScalarType *, ScalarType *);
589 template <typename T = ConcreteOperationType>
590 enable_if_has_vector_path_t<void, T, ContextType, const ScalarType *,
591 ScalarType *, ScalarType *, ScalarType *>
592 vector_path_2x(ContextType ctx, ConstColumnType src, ColumnType dst_a,
593 ColumnType dst_b, ColumnType dst_c) KLEIDICV_STREAMING {
594 operation().vector_path(ctx, &src[0], &dst_a[0], &dst_b[0], &dst_c[0]);
595 operation().vector_path(
596 ctx, &src.at(num_lanes() * 3)[0], &dst_a.at(num_lanes())[0],
597 &dst_b.at(num_lanes())[0], &dst_c.at(num_lanes())[0]);
598 }
599
600 // Required method:
601 // void T::vector_path([ContextType,] const ScalarType *, VectorType &,
602 // VectorType &, VectorType &);
603 template <typename T = ConcreteOperationType>
604 enable_if_has_vector_path_t<void, T, ContextType, const ScalarType *,
605 VectorType &, VectorType &, VectorType &>
606 vector_path_2x(ContextType ctx, ConstColumnType src, ColumnType dst_a,
607 ColumnType dst_b, ColumnType dst_c) KLEIDICV_STREAMING {
608 Vector2Type vdst_a, vdst_b, vdst_c;
609
610 operation().vector_path(ctx, &src[0], vdst_a.val[0], vdst_b.val[0],
611 vdst_c.val[0]);
612 operation().vector_path(ctx, &src.at(num_lanes() * 3)[0], vdst_a.val[1],
613 vdst_b.val[1], vdst_c.val[1]);
614
615 store(vdst_a, &dst_a[0]);
616 store(vdst_b, &dst_b[0]);
617 store(vdst_c, &dst_c[0]);
618 }
619
620 // Required method:
621 // void T::vector_path([ContextType,] , Vector3Type , VectorType &
622 // VectorType &, VectorType &);
623 template <typename T = ConcreteOperationType>
624 enable_if_has_vector_path_t<void, T, ContextType, Vector3Type, VectorType &,
625 VectorType &, VectorType &>
626 1440 vector_path_2x(ContextType ctx, ConstColumnType src, ColumnType dst_a,
627 ColumnType dst_b, ColumnType dst_c) KLEIDICV_STREAMING {
628 1440 Vector3Type vsrc_0, vsrc_1;
629 1440 Vector2Type vdst_a, vdst_b, vdst_c;
630
631 1440 load_consecutive(&src[0], vsrc_0, vsrc_1);
632
633 2880 operation().vector_path(ctx, vsrc_0, vdst_a.val[0], vdst_b.val[0],
634 1440 vdst_c.val[0]);
635 2880 operation().vector_path(ctx, vsrc_1, vdst_a.val[1], vdst_b.val[1],
636 1440 vdst_c.val[1]);
637
638 1440 store(vdst_a, &dst_a[0]);
639 1440 store(vdst_b, &dst_b[0]);
640 1440 store(vdst_c, &dst_c[0]);
641 1440 }
642
643 // Required method:
644 // void T::vector_path([ContextType,] const ScalarType *, ScalarType *,
645 // ScalarType *, ScalarType *, ScalarType *);
646 template <typename T = ConcreteOperationType>
647 enable_if_has_vector_path_t<void, T, ContextType, const ScalarType *,
648 ScalarType *, ScalarType *, ScalarType *,
649 ScalarType *>
650 vector_path_2x(ContextType ctx, ConstColumnType src, ColumnType dst_a,
651 ColumnType dst_b, ColumnType dst_c,
652 ColumnType dst_d) KLEIDICV_STREAMING {
653 operation().vector_path(ctx, &src[0], &dst_a[0], &dst_b[0], &dst_c[0],
654 &dst_d[0]);
655 operation().vector_path(
656 ctx, &src.at(num_lanes() * 4)[0], &dst_a.at(num_lanes())[0],
657 &dst_b.at(num_lanes())[0], &dst_c.at(num_lanes())[0],
658 &dst_d.at(num_lanes())[0]);
659 }
660
661 // Required method:
662 // void T::vector_path([ContextType,] const ScalarType *,
663 // VectorType &, VectorType &, VectorType &,
664 // VectorType &);
665 // Operation with multiple destination address, and with store. This
666 // gives the possibility to store the 2 result vectors in 1 step
667 template <typename T = ConcreteOperationType>
668 enable_if_has_vector_path_t<void, T, ContextType, const ScalarType *,
669 VectorType &, VectorType &, VectorType &,
670 VectorType &>
671 vector_path_2x(ContextType ctx, ConstColumnType src, ColumnType dst_a,
672 ColumnType dst_b, ColumnType dst_c,
673 ColumnType dst_d) KLEIDICV_STREAMING {
674 Vector2Type vdst_a, vdst_b, vdst_c, vdst_d;
675
676 operation().vector_path(ctx, &src[0], vdst_a.val[0], vdst_b.val[0],
677 vdst_c.val[0], vdst_d.val[0]);
678
679 operation().vector_path(ctx, &src.at(num_lanes() * 4)[0], vdst_a.val[1],
680 vdst_b.val[1], vdst_c.val[1], vdst_d.val[1]);
681
682 store(vdst_a, &dst_a[0]);
683 store(vdst_b, &dst_b[0]);
684 store(vdst_c, &dst_c[0]);
685 store(vdst_d, &dst_d[0]);
686 }
687
688 // Required method:
689 // void T::vector_path([ContextType,] const Vector4Type
690 // VectorType &, VectorType &, VectorType &,
691 // VectorType &);
692 // Operation with multiple destination address, and with load and store.
693 // This gives the possibility to load the source x4 vectors and store
694 // the 2x4 result vectors in 1 step
695 template <typename T = ConcreteOperationType>
696 enable_if_has_vector_path_t<void, T, ContextType, const Vector4Type,
697 VectorType &, VectorType &, VectorType &,
698 VectorType &>
699 1440 vector_path_2x(ContextType ctx, ConstColumnType src, ColumnType dst_a,
700 ColumnType dst_b, ColumnType dst_c,
701 ColumnType dst_d) KLEIDICV_STREAMING {
702 1440 Vector4Type vsrc_0, vsrc_1;
703 1440 Vector2Type vdst_a, vdst_b, vdst_c, vdst_d;
704
705 1440 load_consecutive(&src[0], vsrc_0, vsrc_1);
706
707 2880 operation().vector_path(ctx, vsrc_0, vdst_a.val[0], vdst_b.val[0],
708 1440 vdst_c.val[0], vdst_d.val[0]);
709
710 2880 operation().vector_path(ctx, vsrc_1, vdst_a.val[1], vdst_b.val[1],
711 1440 vdst_c.val[1], vdst_d.val[1]);
712
713 1440 store(vdst_a, &dst_a[0]);
714 1440 store(vdst_b, &dst_b[0]);
715 1440 store(vdst_c, &dst_c[0]);
716 1440 store(vdst_d, &dst_d[0]);
717 1440 }
718
719 // ---------------------------------------------------------------------------
720 // Forwarding implementations for vector_path().
721 // ---------------------------------------------------------------------------
722
723 // Required method:
724 // void T::vector_path([ContextType,] VectorType);
725 template <typename T = ConcreteOperationType>
726 4925 enable_if_has_vector_path_t<void, T, ContextType, VectorType> vector_path(
727 ContextType ctx, ConstColumnType src) KLEIDICV_STREAMING {
728 4925 VectorType src_0;
729 4925 operation().load(ctx, &src[0], src_0);
730 4925 operation().vector_path(ctx, src_0);
731 4925 }
732
733 // Required method:
734 // VectorType T::vector_path([ContextType,] VectorType);
735 template <typename T = ConcreteOperationType>
736 enable_if_has_vector_path_t<VectorType, T, ContextType, VectorType>
737 12342 vector_path(ContextType ctx, ConstColumnType src,
738 ColumnType dst) KLEIDICV_STREAMING {
739 12342 VectorType src_0;
740 12342 operation().load(ctx, &src[0], src_0);
741 12342 VectorType res_0 = operation().vector_path(ctx, src_0);
742 12342 operation().store(ctx, res_0, &dst[0]);
743 12342 }
744
745 // Required method:
746 // VectorType T::vector_path([ContextType,] VectorType, VectorType);
747 template <typename T = ConcreteOperationType>
748 enable_if_has_vector_path_t<VectorType, T, ContextType, VectorType,
749 VectorType>
750 7682 vector_path(ContextType ctx, ConstColumnType src_a, ConstColumnType src_b,
751 ColumnType dst) KLEIDICV_STREAMING {
752 7682 VectorType src_a_0, src_b_0;
753 7682 operation().load(ctx, &src_a[0], src_a_0);
754 7682 operation().load(ctx, &src_b[0], src_b_0);
755 7682 VectorType res_0 = operation().vector_path(ctx, src_a_0, src_b_0);
756 7682 operation().store(ctx, res_0, &dst[0]);
757 7682 }
758
759 // Required method:
760 // void T::vector_path([ContextType,] VectorType, ScalarType *);
761 template <typename T = ConcreteOperationType>
762 enable_if_has_vector_path_t<void, T, ContextType, VectorType, ScalarType *>
763 4 vector_path(ContextType ctx, ConstColumnType src,
764 ColumnType dst) KLEIDICV_STREAMING {
765 4 VectorType src_0;
766 4 operation().load(ctx, &src[0], src_0);
767 4 operation().vector_path(ctx, src_0, &dst[0]);
768 4 }
769
770 // Required method:
771 // void T::vector_path([ContextType,] VectorType, VectorType, ScalarType *);
772 template <typename T = ConcreteOperationType>
773 enable_if_has_vector_path_t<void, T, ContextType, VectorType, VectorType,
774 ScalarType *>
775 vector_path(ContextType ctx, ConstColumnType src_a, ConstColumnType src_b,
776 ColumnType dst) KLEIDICV_STREAMING {
777 VectorType src_a_0, src_b_0;
778 operation().load(ctx, &src_a[0], src_a_0);
779 operation().load(ctx, &src_b[0], src_b_0);
780 operation().vector_path(ctx, src_a_0, src_b_0, &dst[0]);
781 }
782
783 // Required method:
784 // void T::vector_path([ContextType,] VectorType, VectorType, VectorType,
785 // ScalarType *);
786 template <typename T = ConcreteOperationType>
787 enable_if_has_vector_path_t<void, T, ContextType, VectorType, VectorType,
788 VectorType, ScalarType *>
789 vector_path(ContextType ctx, ConstColumnType src_a, ConstColumnType src_b,
790 ConstColumnType src_c, ColumnType dst) KLEIDICV_STREAMING {
791 VectorType src_a_0, src_b_0, src_c_0;
792 operation().load(ctx, &src_a[0], src_a_0);
793 operation().load(ctx, &src_b[0], src_b_0);
794 operation().load(ctx, &src_c[0], src_c_0);
795 operation().vector_path(ctx, src_a_0, src_b_0, src_c_0, &dst[0]);
796 }
797
798 // Required method:
799 // void T::vector_path([ContextType,] VectorType, VectorType, VectorType,
800 // ScalarType *, ScalarType *);
801 template <typename T = ConcreteOperationType>
802 enable_if_has_vector_path_t<void, T, ContextType, VectorType, VectorType,
803 VectorType, ScalarType *, ScalarType *>
804 96 vector_path(ContextType ctx, ConstColumnType src_a, ConstColumnType src_b,
805 ConstColumnType src_c, ColumnType dst_a,
806 ColumnType dst_b) KLEIDICV_STREAMING {
807 96 VectorType src_a_0, src_b_0, src_c_0;
808 96 operation().load(ctx, &src_a[0], src_a_0);
809 96 operation().load(ctx, &src_b[0], src_b_0);
810 96 operation().load(ctx, &src_c[0], src_c_0);
811 192 operation().vector_path(ctx, src_a_0, src_b_0, src_c_0, &dst_a[0],
812 96 &dst_b[0]);
813 96 }
814
815 // Required method:
816 // void T::vector_path([ContextType,] VectorType, VectorType, VectorType,
817 // VectorType, ScalarType *);
818 template <typename T = ConcreteOperationType>
819 enable_if_has_vector_path_t<void, T, ContextType, VectorType, VectorType,
820 VectorType, VectorType, ScalarType *>
821 vector_path(ContextType ctx, ConstColumnType src_a, ConstColumnType src_b,
822 ConstColumnType src_c, ConstColumnType src_d,
823 ColumnType dst) KLEIDICV_STREAMING {
824 VectorType src_a_0, src_b_0, src_c_0, src_d_0;
825 operation().load(ctx, &src_a[0], src_a_0);
826 operation().load(ctx, &src_b[0], src_b_0);
827 operation().load(ctx, &src_c[0], src_c_0);
828 operation().load(ctx, &src_d[0], src_d_0);
829 operation().vector_path(ctx, src_a_0, src_b_0, src_c_0, src_d_0, &dst[0]);
830 }
831
832 // Required method:
833 // void T::vector_path([ContextType,] const ScalarType *, ScalarType *);
834 template <typename T = ConcreteOperationType>
835 enable_if_has_vector_path_t<void, T, ContextType, const ScalarType *,
836 ScalarType *>
837 12492 vector_path(ContextType ctx, ConstColumnType src,
838 ColumnType dst) KLEIDICV_STREAMING {
839 12492 operation().vector_path(ctx, &src[0], &dst[0]);
840 12492 }
841
842 // Required method:
843 // void T::vector_path([ContextType,] const ScalarType *, const ScalarType *,
844 // ScalarType *);
845 template <typename T = ConcreteOperationType>
846 enable_if_has_vector_path_t<void, T, ContextType, const ScalarType *,
847 const ScalarType *, ScalarType *>
848 vector_path(ContextType ctx, ConstColumnType src_a, ConstColumnType src_b,
849 ColumnType dst) KLEIDICV_STREAMING {
850 operation().vector_path(ctx, &src_a[0], &src_b[0], &dst[0]);
851 }
852
853 // Required method:
854 // void T::vector_path([ContextType,] const ScalarType *, const ScalarType *,
855 // const ScalarType *, ScalarType *, ScalarType *);
856 template <typename T = ConcreteOperationType>
857 enable_if_has_vector_path_t<void, T, ContextType, const ScalarType *,
858 const ScalarType *, const ScalarType *,
859 ScalarType *, ScalarType *>
860 3008 vector_path(ContextType ctx, ConstColumnType src_a, ConstColumnType src_b,
861 ConstColumnType src_c, ColumnType dst_a,
862 ColumnType dst_b) KLEIDICV_STREAMING {
863 6016 operation().vector_path(ctx, &src_a[0], &src_b[0], &src_c[0], &dst_a[0],
864 3008 &dst_b[0]);
865 3008 }
866
867 // Required method:
868 // void T::vector_path([ContextType,] const ScalarType *, ScalarType *,
869 // ScalarType *);
870 template <typename T = ConcreteOperationType>
871 enable_if_has_vector_path_t<void, T, ContextType, const ScalarType *,
872 ScalarType *, ScalarType *>
873 vector_path(ContextType ctx, ConstColumnType src, ColumnType dst_a,
874 ColumnType dst_b) KLEIDICV_STREAMING {
875 operation().vector_path(ctx, &src[0], &dst_a[0], &dst_b[0]);
876 }
877
878 // Required method:
879 // void T::vector_path([ContextType,] const ScalarType *, VectorType &,
880 // VectorType &);
881 template <typename T = ConcreteOperationType>
882 enable_if_has_vector_path_t<void, T, ContextType, const ScalarType *,
883 VectorType &, VectorType &>
884 vector_path(ContextType ctx, ConstColumnType src, ColumnType dst_a,
885 ColumnType dst_b) KLEIDICV_STREAMING {
886 VectorType vdst_a, vdst_b;
887
888 operation().vector_path(ctx, &src[0], vdst_a, vdst_b);
889
890 store(vdst_a, &dst_a[0]);
891 store(vdst_b, &dst_b[0]);
892 }
893
894 // Required method:
895 // void T::vector_path([ContextType,] Vector2Type, VectorType &,
896 // VectorType &);
897 template <typename T = ConcreteOperationType>
898 enable_if_has_vector_path_t<void, T, ContextType, Vector2Type, VectorType &,
899 VectorType &>
900 vector_path(ContextType ctx, ConstColumnType src, ColumnType dst_a,
901 ColumnType dst_b) KLEIDICV_STREAMING {
902 VectorType vdst_a, vdst_b;
903 Vector2Type vsrc;
904
905 load(&src[0], vsrc);
906
907 operation().vector_path(ctx, vsrc, vdst_a, vdst_b);
908
909 store(vdst_a, &dst_a[0]);
910 store(vdst_b, &dst_b[0]);
911 }
912
913 // Required method:
914 // void T::vector_path([ContextType,] const ScalarType *, ScalarType *,
915 // ScalarType *, ScalarType *);
916 template <typename T = ConcreteOperationType>
917 enable_if_has_vector_path_t<void, T, ContextType, const ScalarType *,
918 ScalarType *, ScalarType *, ScalarType *>
919 vector_path(ContextType ctx, ConstColumnType src, ColumnType dst_a,
920 ColumnType dst_b, ColumnType dst_c) KLEIDICV_STREAMING {
921 operation().vector_path(ctx, &src[0], &dst_a[0], &dst_b[0], &dst_c[0]);
922 }
923
924 // Required method:
925 // void T::vector_path([ContextType,] const ScalarType *, VectorType &,
926 // VectorType &, VectorType &);
927 template <typename T = ConcreteOperationType>
928 enable_if_has_vector_path_t<void, T, ContextType, const ScalarType *,
929 VectorType &, VectorType &, VectorType &>
930 vector_path(ContextType ctx, ConstColumnType src, ColumnType dst_a,
931 ColumnType dst_b, ColumnType dst_c) KLEIDICV_STREAMING {
932 VectorType vdst_a, vdst_b, vdst_c;
933
934 operation().vector_path(ctx, &src[0], vdst_a, vdst_b, vdst_c);
935
936 store(vdst_a, &dst_a[0]);
937 store(vdst_b, &dst_b[0]);
938 store(vdst_c, &dst_c[0]);
939 }
940
941 // Required method:
942 // void T::vector_path([ContextType,] Vector3Type, VectorType &,
943 // VectorType &, VectorType &);
944 template <typename T = ConcreteOperationType>
945 enable_if_has_vector_path_t<void, T, ContextType, Vector3Type, VectorType &,
946 VectorType &, VectorType &>
947 vector_path(ContextType ctx, ConstColumnType src, ColumnType dst_a,
948 ColumnType dst_b, ColumnType dst_c) KLEIDICV_STREAMING {
949 VectorType vdst_a, vdst_b, vdst_c;
950 Vector3Type vsrc;
951
952 load(&src[0], vsrc);
953
954 operation().vector_path(ctx, vsrc, vdst_a, vdst_b, vdst_c);
955
956 store(vdst_a, &dst_a[0]);
957 store(vdst_b, &dst_b[0]);
958 store(vdst_c, &dst_c[0]);
959 }
960
961 // Required method:
962 // void T::vector_path([ContextType,] const ScalarType *, ScalarType *,
963 // ScalarType *, ScalarType *, ScalarType *);
964 // Operation type with multiple destination, store in the vectorpath
965 template <typename T = ConcreteOperationType>
966 enable_if_has_vector_path_t<void, T, ContextType, const ScalarType *,
967 ScalarType *, ScalarType *, ScalarType *,
968 ScalarType *>
969 vector_path(ContextType ctx, ConstColumnType src, ColumnType dst_a,
970 ColumnType dst_b, ColumnType dst_c,
971 ColumnType dst_d) KLEIDICV_STREAMING {
972 operation().vector_path(ctx, &src[0], &dst_a[0], &dst_b[0], &dst_c[0],
973 &dst_d[0]);
974 }
975
976 // Required method:
977 // void T::vector_path([ContextType,] const ScalarType *,
978 // VectorType &, VectorType &, VectorType &,
979 // VectorType &);
980 // Operation type with multiple destination, and with store
981 template <typename T = ConcreteOperationType>
982 enable_if_has_vector_path_t<void, T, ContextType, const ScalarType *,
983 VectorType &, VectorType &, VectorType &,
984 VectorType &>
985 vector_path(ContextType ctx, ConstColumnType src, ColumnType dst_a,
986 ColumnType dst_b, ColumnType dst_c,
987 ColumnType dst_d) KLEIDICV_STREAMING {
988 VectorType vdst_a, vdst_b, vdst_c, vdst_d;
989
990 operation().vector_path(ctx, &src[0], vdst_a, vdst_b, vdst_c, vdst_d);
991
992 store(vdst_a, &dst_a[0]);
993 store(vdst_b, &dst_b[0]);
994 store(vdst_c, &dst_c[0]);
995 store(vdst_d, &dst_d[0]);
996 }
997
998 // Required method:
999 // void T::vector_path([ContextType,] const Vector4Type,
1000 // VectorType &, VectorType &, VectorType &,
1001 // VectorType &);
1002 // Operation type with multiple destination, and with load and store
1003 template <typename T = ConcreteOperationType>
1004 enable_if_has_vector_path_t<void, T, ContextType, const Vector4Type,
1005 VectorType &, VectorType &, VectorType &,
1006 VectorType &>
1007 vector_path(ContextType ctx, ConstColumnType src, ColumnType dst_a,
1008 ColumnType dst_b, ColumnType dst_c,
1009 ColumnType dst_d) KLEIDICV_STREAMING {
1010 VectorType vdst_a, vdst_b, vdst_c, vdst_d;
1011 Vector4Type vsrc;
1012
1013 load(&src[0], vsrc);
1014
1015 operation().vector_path(ctx, vsrc, vdst_a, vdst_b, vdst_c, vdst_d);
1016
1017 store(vdst_a, &dst_a[0]);
1018 store(vdst_b, &dst_b[0]);
1019 store(vdst_c, &dst_c[0]);
1020 store(vdst_d, &dst_d[0]);
1021 }
1022
1023 // ---------------------------------------------------------------------------
1024 // Forwarding implementations for tail_path().
1025 // ---------------------------------------------------------------------------
1026
1027 // Required method:
1028 // void T::tail_path([ContextType,] VectorType, ScalarType *);
1029 template <typename T = ConcreteOperationType>
1030 enable_if_has_tail_path_t<void, T, ContextType, VectorType, ScalarType *>
1031 372 tail_path(ContextType ctx, ConstColumnType src,
1032 ColumnType dst) KLEIDICV_STREAMING {
1033 372 VectorType src_0;
1034 372 operation().load(ctx, &src[0], src_0);
1035 372 operation().tail_path(ctx, src_0, &dst[0]);
1036 372 }
1037
1038 // Required method:
1039 // void T::tail_path([ContextType,] const ScalarType *, ScalarType *);
1040 template <typename T = ConcreteOperationType>
1041 enable_if_has_tail_path_t<void, T, ContextType, const ScalarType *,
1042 ScalarType *>
1043 336 tail_path(ContextType ctx, ConstColumnType src,
1044 ColumnType dst) KLEIDICV_STREAMING {
1045 336 operation().tail_path(ctx, &src[0], &dst[0]);
1046 336 }
1047
1048 // ---------------------------------------------------------------------------
1049 // Forwarding implementations for scalar_path().
1050 // ---------------------------------------------------------------------------
1051
1052 // Required method:
1053 // void T::scalar_path([ContextType,] ScalarType);
1054 template <typename T = ConcreteOperationType>
1055 33561 enable_if_has_scalar_path_t<void, T, ContextType, ScalarType> scalar_path(
1056 ContextType ctx, ConstColumnType src) KLEIDICV_STREAMING {
1057 33561 operation().scalar_path(ctx, src[0]);
1058 33561 }
1059
1060 // Required method:
1061 // ScalarType T::scalar_path([ContextType,] ScalarType);
1062 template <typename T = ConcreteOperationType>
1063 enable_if_has_scalar_path_t<ScalarType, T, ContextType, ScalarType>
1064 5460 scalar_path(ContextType ctx, ConstColumnType src,
1065 ColumnType dst) KLEIDICV_STREAMING {
1066 5460 dst[0] = operation().scalar_path(ctx, src[0]);
1067 5460 }
1068
1069 // Required method:
1070 // ScalarType T::scalar_path([ContextType,] ScalarType, ScalarType);
1071 template <typename T = ConcreteOperationType>
1072 enable_if_has_scalar_path_t<ScalarType, T, ContextType, ScalarType,
1073 ScalarType>
1074 14694 scalar_path(ContextType ctx, ConstColumnType src_a, ConstColumnType src_b,
1075 ColumnType dst) KLEIDICV_STREAMING {
1076 14694 dst[0] = operation().scalar_path(ctx, src_a[0], src_b[0]);
1077 14694 }
1078
1079 // Required method:
1080 // void T::scalar_path([ContextType,] const ScalarType *, ScalarType *);
1081 template <typename T = ConcreteOperationType>
1082 enable_if_has_scalar_path_t<void, T, ContextType, const ScalarType *,
1083 ScalarType *>
1084 2903 scalar_path(ContextType ctx, ConstColumnType src,
1085 ColumnType dst) KLEIDICV_STREAMING {
1086 2903 operation().scalar_path(ctx, &src[0], &dst[0]);
1087 2903 }
1088
1089 // Required method:
1090 // void T::scalar_path([ContextType,] const ScalarType *, const ScalarType *,
1091 // ScalarType *);
1092 template <typename T = ConcreteOperationType>
1093 enable_if_has_scalar_path_t<void, T, ContextType, const ScalarType *,
1094 const ScalarType *, ScalarType *>
1095 256 scalar_path(ContextType ctx, ConstColumnType src_a, ConstColumnType src_b,
1096 ColumnType dst) KLEIDICV_STREAMING {
1097 256 operation().scalar_path(ctx, &src_a[0], &src_b[0], &dst[0]);
1098 256 }
1099
1100 // Required method:
1101 // void T::scalar_path([ContextType,] const ScalarType *, const ScalarType *,
1102 // const ScalarType *, ScalarType *);
1103 template <typename T = ConcreteOperationType>
1104 enable_if_has_scalar_path_t<void, T, ContextType, const ScalarType *,
1105 const ScalarType *, const ScalarType *,
1106 ScalarType *>
1107 512 scalar_path(ContextType ctx, ConstColumnType src_a, ConstColumnType src_b,
1108 ConstColumnType src_c, ColumnType dst) KLEIDICV_STREAMING {
1109 512 operation().scalar_path(ctx, &src_a[0], &src_b[0], &src_c[0], &dst[0]);
1110 512 }
1111
1112 // Required method:
1113 // void T::scalar_path([ContextType,] const ScalarType *, const ScalarType *,
1114 // const ScalarType *, const ScalarType *, ScalarType *);
1115 template <typename T = ConcreteOperationType>
1116 enable_if_has_scalar_path_t<void, T, ContextType, const ScalarType *,
1117 const ScalarType *, const ScalarType *,
1118 const ScalarType *, ScalarType *>
1119 1024 scalar_path(ContextType ctx, ConstColumnType src_a, ConstColumnType src_b,
1120 ConstColumnType src_c, ConstColumnType src_d,
1121 ColumnType dst) KLEIDICV_STREAMING {
1122 2048 operation().scalar_path(ctx, &src_a[0], &src_b[0], &src_c[0], &src_d[0],
1123 1024 &dst[0]);
1124 1024 }
1125
1126 // Required method:
1127 // void T::scalar_path([ContextType,] size_t, const ScalarType *,
1128 // const ScalarType *, const ScalarType *, ScalarType *,
1129 // ScalarType *);
1130 template <typename T = ConcreteOperationType>
1131 enable_if_has_scalar_path_t<void, T, ContextType, size_t, const ScalarType *,
1132 const ScalarType *, const ScalarType *,
1133 ScalarType *, ScalarType *>
1134 656 scalar_path(ContextType ctx, size_t length, ConstColumnType src_a,
1135 ConstColumnType src_b, ConstColumnType src_c, ColumnType dst_a,
1136 ColumnType dst_b) KLEIDICV_STREAMING {
1137 1312 operation().scalar_path(ctx, length, &src_a[0], &src_b[0], &src_c[0],
1138 656 &dst_a[0], &dst_b[0]);
1139 656 }
1140
1141 // Required method:
1142 // void T::scalar_path([ContextType,] const ScalarType *, ScalarType *,
1143 // ScalarType *);
1144 template <typename T = ConcreteOperationType>
1145 enable_if_has_scalar_path_t<void, T, ContextType, const ScalarType *,
1146 ScalarType *, ScalarType *>
1147 256 scalar_path(ContextType ctx, ConstColumnType src, ColumnType dst_a,
1148 ColumnType dst_b) KLEIDICV_STREAMING {
1149 256 operation().scalar_path(ctx, &src[0], &dst_a[0], &dst_b[0]);
1150 256 }
1151
1152 // Required method:
1153 // void T::scalar_path([ContextType,] const ScalarType *, ScalarType *,
1154 // ScalarType *, ScalarType *);
1155 template <typename T = ConcreteOperationType>
1156 enable_if_has_scalar_path_t<void, T, ContextType, const ScalarType *,
1157 ScalarType *, ScalarType *, ScalarType *>
1158 512 scalar_path(ContextType ctx, ConstColumnType src, ColumnType dst_a,
1159 ColumnType dst_b, ColumnType dst_c) KLEIDICV_STREAMING {
1160 512 operation().scalar_path(ctx, &src[0], &dst_a[0], &dst_b[0], &dst_c[0]);
1161 512 }
1162
1163 // Required method:
1164 // void T::scalar_path([ContextType,] const ScalarType *, ScalarType *,
1165 // ScalarType *, ScalarType *, ScalarType *);
1166 template <typename T = ConcreteOperationType>
1167 enable_if_has_scalar_path_t<void, T, ContextType, const ScalarType *,
1168 ScalarType *, ScalarType *, ScalarType *,
1169 ScalarType *>
1170 512 scalar_path(ContextType ctx, ConstColumnType src, ColumnType dst_a,
1171 ColumnType dst_b, ColumnType dst_c,
1172 ColumnType dst_d) KLEIDICV_STREAMING {
1173 1024 operation().scalar_path(ctx, &src[0], &dst_a[0], &dst_b[0], &dst_c[0],
1174 512 &dst_d[0]);
1175 512 }
1176 }; // end of class OperationAdapter<OperationType>
1177
1178 // Removes context before forwarding calls.
1179 template <typename OperationType>
1180 class RemoveContextAdapter : public OperationBase<OperationType> {
1181 public:
1182 using ContextType = typename OperationBase<OperationType>::ContextType;
1183
1184 7255 explicit RemoveContextAdapter(OperationType &operation) KLEIDICV_STREAMING
1185 7255 : OperationBase<OperationType>(operation) {}
1186
1187 template <typename... ArgTypes>
1188 3280 decltype(auto) load(ContextType, ArgTypes &&...args) KLEIDICV_STREAMING {
1189 3280 return OperationBase<OperationType>::load(std::forward<ArgTypes>(args)...);
1190 }
1191
1192 template <typename... ArgTypes>
1193 490235 decltype(auto) load_consecutive(ContextType,
1194 ArgTypes &&...args) KLEIDICV_STREAMING {
1195 490235 return OperationBase<OperationType>::load_consecutive(
1196 490235 std::forward<ArgTypes>(args)...);
1197 }
1198
1199 template <typename... ArgTypes>
1200 2958 decltype(auto) store(ContextType, ArgTypes &&...args) KLEIDICV_STREAMING {
1201 2958 return OperationBase<OperationType>::store(std::forward<ArgTypes>(args)...);
1202 }
1203
1204 template <typename... ArgTypes>
1205 28513 decltype(auto) store_consecutive(ContextType,
1206 ArgTypes &&...args) KLEIDICV_STREAMING {
1207 28513 return OperationBase<OperationType>::store_consecutive(
1208 28513 std::forward<ArgTypes>(args)...);
1209 }
1210
1211 template <typename... ArgTypes>
1212 932704 decltype(auto) vector_path(ContextType,
1213 ArgTypes &&...args) KLEIDICV_STREAMING {
1214 932704 return this->operation().vector_path(std::forward<ArgTypes>(args)...);
1215 }
1216
1217 // Forwards remaining_path() calls to the inner operation.
1218 template <typename... ArgTypes>
1219 decltype(auto) remaining_path(ContextType,
1220 ArgTypes &&...args) KLEIDICV_STREAMING {
1221 return this->operation().remaining_path(std::forward<ArgTypes>(args)...);
1222 }
1223
1224 template <typename... ArgTypes>
1225 decltype(auto) tail_path(ContextType, ArgTypes &&...args) KLEIDICV_STREAMING {
1226 return this->operation().tail_path(std::forward<ArgTypes>(args)...);
1227 }
1228
1229 template <typename... ArgTypes>
1230 60346 decltype(auto) scalar_path(ContextType,
1231 ArgTypes &&...args) KLEIDICV_STREAMING {
1232 60346 return this->operation().scalar_path(std::forward<ArgTypes>(args)...);
1233 }
1234 }; // end of class RemoveContextAdapter<OperationType>
1235
1236 } // namespace KLEIDICV_TARGET_NAMESPACE
1237
1238 #endif // KLEIDICV_OPERATIONS_H
1239