KleidiCV Coverage Report

Directory: ./
File: kleidicv/src/filters/scharr_sc.h
Date: 2026-03-05 15:57:40
Exec Total Coverage
Lines: 189 189 100.0%
Functions: 57 57 100.0%
Branches: 18 18 100.0%
Line Branch Exec Source
1 // SPDX-FileCopyrightText: 2024 - 2026 Arm Limited and/or its affiliates <open-source-office@arm.com>
2 //
3 // SPDX-License-Identifier: Apache-2.0
4
5 #include <cstddef>
6 #include <cstdint>
7 #include <cstdlib>
8 #include <memory>
9
10 #include "kleidicv/config.h"
11 #include "kleidicv/ctypes.h"
12 #include "kleidicv/sve2.h"
13 #include "kleidicv/types.h"
14 #include "kleidicv/utils.h"
15
16 namespace KLEIDICV_TARGET_NAMESPACE {
17
18 // Scharr filtering in both horizontal and vertical directions, horizontal and
19 // vertical derivative approximations are stored interleaved.
20 //
21 // The applied weights for the horizontal approximation, as the kernel is
22 // mirrored both vertically and horizontally during the convolution:
23 // [ -3 0 3 ] [ 3 ]
24 // F = [ -10 0 10 ] = [ 10 ] * [ -1, 0, 1 ]
25 // [ -3 0 3 ] [ 3 ]
26 //
27 // The applied weights for the vertical approximation, as the kernel is mirrored
28 // both vertically and horizontally during the convolution:
29 // [ -3 -10 -3 ] [ -1 ]
30 // F = [ 0, 0, 0 ] = [ 0 ] * [ 3, 10, 3 ]
31 // [ 3 10 3 ] [ 1 ]
32 //
33 class ScharrInterleaved {
34 using SourceType = uint8_t;
35 using SourceVecTraits = VecTraits<SourceType>;
36 using SourceVectorType = typename SourceVecTraits::VectorType;
37 using SourceVector2Type = typename SourceVecTraits::Vector2Type;
38 using SourceVector3Type = typename SourceVecTraits::Vector3Type;
39 using BufferType = int16_t;
40 using BufferVecTraits = VecTraits<BufferType>;
41 using BufferVectorType = typename BufferVecTraits::VectorType;
42 using BufferVector2Type = typename BufferVecTraits::Vector2Type;
43 using BufferVector3Type = typename BufferVecTraits::Vector3Type;
44 using DestinationType = int16_t;
45 using DestinationVecTraits = VecTraits<DestinationType>;
46 using DestinationVectorType = typename DestinationVecTraits::VectorType;
47
48 public:
49 990 ScharrInterleaved(Rows<BufferType> hori_deriv_buffer,
50 Rows<BufferType> vert_deriv_buffer,
51 size_t width) KLEIDICV_STREAMING
52 990 : hori_deriv_buffer_(hori_deriv_buffer),
53 990 vert_deriv_buffer_(vert_deriv_buffer),
54 990 width_(width) {}
55
56 990 void process(Rows<const SourceType> src_rows, Rows<DestinationType> dst_rows,
57 size_t y_begin, size_t y_end) KLEIDICV_STREAMING {
58
2/2
✓ Branch 0 taken 990 times.
✓ Branch 1 taken 17472 times.
 18462 for (size_t i = y_begin; i < y_end; ++i) {
59 17472 process_vertical(src_rows.at(static_cast<ptrdiff_t>(i)));
60 17472 process_horizontal(dst_rows.at(static_cast<ptrdiff_t>(i)));
61 17472 }
62 990 }
63
64 private:
65 1334 void vertical_vector_path_x2(svbool_t pg, Rows<const SourceType> src_rows,
66 ptrdiff_t index) KLEIDICV_STREAMING {
67 #if KLEIDICV_TARGET_SME2
68 82 svcount_t pc8 = SourceVecTraits::svptrue_c();
69 82 SourceVector2Type src0 = svld1_x2(pc8, &src_rows.at(0)[index]);
70 82 SourceVector2Type src1 = svld1_x2(pc8, &src_rows.at(1)[index]);
71 82 SourceVector2Type src2 = svld1_x2(pc8, &src_rows.at(2)[index]);
72 164 SourceVector3Type src_a =
73 82 svcreate3(svget2(src0, 0), svget2(src1, 0), svget2(src2, 0));
74 164 SourceVector3Type src_b =
75 82 svcreate3(svget2(src0, 1), svget2(src1, 1), svget2(src2, 1));
76 #else
77 2504 SourceVector3Type src_a = svcreate3(svld1(pg, &src_rows.at(0)[index]),
78 1252 svld1(pg, &src_rows.at(1)[index]),
79 1252 svld1(pg, &src_rows.at(2)[index]));
80 2504 SourceVector3Type src_b =
81 2504 svcreate3(svld1_vnum(pg, &src_rows.at(0)[index], 1),
82 1252 svld1_vnum(pg, &src_rows.at(1)[index], 1),
83 1252 svld1_vnum(pg, &src_rows.at(2)[index], 1));
84 #endif
85
86 1334 svint16x2_t hori_interleaved_a = vertical_compute_hori_approx(pg, src_a);
87 1334 svint16x2_t vert_interleaved_a = vertical_compute_vert_approx(src_a);
88
89 1334 svint16x2_t hori_interleaved_b = vertical_compute_hori_approx(pg, src_b);
90 1334 svint16x2_t vert_interleaved_b = vertical_compute_vert_approx(src_b);
91
92 1334 svst2(pg, &hori_deriv_buffer_[index], hori_interleaved_a);
93 1334 svst2_vnum(pg, &hori_deriv_buffer_[index], 2, hori_interleaved_b);
94
95 1334 svst2(pg, &vert_deriv_buffer_[index], vert_interleaved_a);
96 1334 svst2_vnum(pg, &vert_deriv_buffer_[index], 2, vert_interleaved_b);
97 1334 }
98
99 18504 void vertical_vector_path(svbool_t pg, Rows<const SourceType> src_rows,
100 ptrdiff_t index) KLEIDICV_STREAMING {
101 18504 SourceVectorType src_0 = svld1(pg, &src_rows.at(0)[index]);
102 18504 SourceVectorType src_1 = svld1(pg, &src_rows.at(1)[index]);
103 18504 SourceVectorType src_2 = svld1(pg, &src_rows.at(2)[index]);
104
105 18504 SourceVector3Type src = svcreate3(src_0, src_1, src_2);
106 18504 svint16x2_t hori_interleaved = vertical_compute_hori_approx(pg, src);
107 18504 svint16x2_t vert_interleaved = vertical_compute_vert_approx(src);
108
109 18504 svst2(pg, &hori_deriv_buffer_[index], hori_interleaved);
110 18504 svst2(pg, &vert_deriv_buffer_[index], vert_interleaved);
111 18504 }
112
113 21172 svint16x2_t vertical_compute_hori_approx(svbool_t pg, SourceVector3Type src)
114 KLEIDICV_STREAMING {
115 21172 SourceVectorType src_0 = svget3(src, 0);
116 21172 SourceVectorType src_1 = svget3(src, 1);
117 21172 SourceVectorType src_2 = svget3(src, 2);
118 21172 svuint16_t hori_acc_b = svaddlb(src_0, src_2);
119 21172 svuint16_t hori_acc_t = svaddlt(src_0, src_2);
120
121 21172 hori_acc_b = svmul_n_u16_x(pg, hori_acc_b, 3);
122 21172 hori_acc_t = svmul_n_u16_x(pg, hori_acc_t, 3);
123
124 21172 hori_acc_b = svmlalb_n_u16(hori_acc_b, src_1, 10);
125 21172 hori_acc_t = svmlalt_n_u16(hori_acc_t, src_1, 10);
126
127 63516 return svcreate2(svreinterpret_s16(hori_acc_b),
128 21172 svreinterpret_s16(hori_acc_t));
129 21172 }
130
131 21172 svint16x2_t vertical_compute_vert_approx(SourceVector3Type src)
132 KLEIDICV_STREAMING {
133 21172 SourceVectorType src_0 = svget3(src, 0);
134 21172 SourceVectorType src_2 = svget3(src, 2);
135 21172 svuint16_t vert_acc_b = svsublb(src_2, src_0);
136 21172 svuint16_t vert_acc_t = svsublt(src_2, src_0);
137
138 63516 return svcreate2(svreinterpret_s16(vert_acc_b),
139 21172 svreinterpret_s16(vert_acc_t));
140 21172 }
141
142 17472 void process_vertical(Rows<const SourceType> src_rows) KLEIDICV_STREAMING {
143 34944 LoopUnroll2 loop{width_ * src_rows.channels(),
144 17472 SourceVecTraits::num_lanes()};
145 17472 svbool_t pg_all = SourceVecTraits::svptrue();
146
147 18806 loop.unroll_twice([&](ptrdiff_t index) KLEIDICV_STREAMING {
148 1334 vertical_vector_path_x2(pg_all, src_rows, index);
149 1334 });
150
151 19184 loop.unroll_once([&](ptrdiff_t index) KLEIDICV_STREAMING {
152 1712 vertical_vector_path(pg_all, src_rows, index);
153 1712 });
154
155 34264 loop.remaining([&](ptrdiff_t index, ptrdiff_t length) KLEIDICV_STREAMING {
156 16792 svbool_t pg = SourceVecTraits::svwhilelt(index, length);
157 16792 vertical_vector_path(pg, src_rows, index);
158 16792 });
159 17472 }
160
161 3518 void horizontal_vector_path_x2(svbool_t pg, Rows<DestinationType> dst_rows,
162 ptrdiff_t index,
163 ptrdiff_t channel) KLEIDICV_STREAMING {
164 #if KLEIDICV_TARGET_SME2
165 300 svcount_t pc16 = BufferVecTraits::svptrue_c();
166 300 BufferVector2Type hori_0 = svld1_x2(pc16, &hori_deriv_buffer_[index]);
167 600 BufferVector2Type hori_1 =
168 300 svld1_x2(pc16, &hori_deriv_buffer_[index + channel * 2]);
169 300 BufferVector2Type vert_0 = svld1_x2(pc16, &vert_deriv_buffer_[index]);
170 600 BufferVector2Type vert_1 =
171 300 svld1_x2(pc16, &vert_deriv_buffer_[index + channel]);
172 600 BufferVector2Type vert_2 =
173 300 svld1_x2(pc16, &vert_deriv_buffer_[index + channel * 2]);
174 300 BufferVector2Type hori_a = svcreate2(svget2(hori_0, 0), svget2(hori_1, 0));
175 300 BufferVector2Type hori_b = svcreate2(svget2(hori_0, 1), svget2(hori_1, 1));
176
177 600 BufferVector3Type vert_a =
178 300 svcreate3(svget2(vert_0, 0), svget2(vert_1, 0), svget2(vert_2, 0));
179 600 BufferVector3Type vert_b =
180 300 svcreate3(svget2(vert_0, 1), svget2(vert_1, 1), svget2(vert_2, 1));
181 #else
182 3218 BufferVectorType hori_buff_0 = svld1(pg, &hori_deriv_buffer_[index]);
183 6436 BufferVectorType hori_buff_3 =
184 3218 svld1_vnum(pg, &hori_deriv_buffer_[index], 1);
185 6436 BufferVectorType hori_buff_2 =
186 3218 svld1(pg, &hori_deriv_buffer_[index + channel * 2]);
187 6436 BufferVectorType hori_buff_4 =
188 3218 svld1_vnum(pg, &hori_deriv_buffer_[index + channel * 2], 1);
189
190 3218 BufferVectorType vert_buff_0 = svld1(pg, &vert_deriv_buffer_[index]);
191 6436 BufferVectorType vert_buff_3 =
192 3218 svld1_vnum(pg, &vert_deriv_buffer_[index], 1);
193 6436 BufferVectorType vert_buff_1 =
194 3218 svld1(pg, &vert_deriv_buffer_[index + channel]);
195 6436 BufferVectorType vert_buff_2 =
196 3218 svld1(pg, &vert_deriv_buffer_[index + channel * 2]);
197 6436 BufferVectorType vert_buff_4 =
198 3218 svld1_vnum(pg, &vert_deriv_buffer_[index + channel], 1);
199 6436 BufferVectorType vert_buff_5 =
200 3218 svld1_vnum(pg, &vert_deriv_buffer_[index + channel * 2], 1);
201
202 3218 BufferVector2Type hori_a = svcreate2(hori_buff_0, hori_buff_2);
203 3218 BufferVector2Type hori_b = svcreate2(hori_buff_3, hori_buff_4);
204
205 3218 BufferVector3Type vert_a = svcreate3(vert_buff_0, vert_buff_1, vert_buff_2);
206 3218 BufferVector3Type vert_b = svcreate3(vert_buff_3, vert_buff_4, vert_buff_5);
207 #endif
208
209 7036 svint16x2_t interleaved_result_a =
210 3518 horizontal_compute_interleaved(pg, hori_a, vert_a);
211
212 7036 svint16x2_t interleaved_result_b =
213 3518 horizontal_compute_interleaved(pg, hori_b, vert_b);
214
215 3518 svst2(pg, &dst_rows[index * 2], interleaved_result_a);
216 3518 svst2_vnum(pg, &dst_rows[index * 2], 2, interleaved_result_b);
217 3518 }
218
219 18324 void horizontal_vector_path(svbool_t pg, Rows<DestinationType> dst_rows,
220 ptrdiff_t index,
221 ptrdiff_t channel) KLEIDICV_STREAMING {
222 // Horizontal derivative approximation
223 36648 BufferVector2Type hori =
224 36648 svcreate2(svld1(pg, &hori_deriv_buffer_[index]),
225 18324 svld1(pg, &hori_deriv_buffer_[index + channel * 2]));
226
227 // Vertical derivative approximation
228 36648 BufferVector3Type vert =
229 36648 svcreate3(svld1(pg, &vert_deriv_buffer_[index]),
230 18324 svld1(pg, &vert_deriv_buffer_[index + channel]),
231 18324 svld1(pg, &vert_deriv_buffer_[index + channel * 2]));
232
233 36648 svint16x2_t interleaved_result =
234 18324 horizontal_compute_interleaved(pg, hori, vert);
235 18324 svst2(pg, &dst_rows[index * 2], interleaved_result);
236 18324 }
237
238 25360 svint16x2_t horizontal_compute_interleaved(
239 svbool_t pg, BufferVector2Type hori,
240 BufferVector3Type vert) KLEIDICV_STREAMING {
241 25360 BufferVectorType hori_buff_0 = svget2(hori, 0);
242 25360 BufferVectorType hori_buff_2 = svget2(hori, 1);
243 25360 BufferVectorType vert_buff_0 = svget3(vert, 0);
244 25360 BufferVectorType vert_buff_1 = svget3(vert, 1);
245 25360 BufferVectorType vert_buff_2 = svget3(vert, 2);
246 25360 DestinationVectorType hori_result = svsub_x(pg, hori_buff_2, hori_buff_0);
247
248 25360 DestinationVectorType vert_result = svadd_x(pg, vert_buff_0, vert_buff_2);
249 25360 vert_result = svmul_n_s16_x(pg, vert_result, 3);
250 25360 vert_result = svmad_s16_x(pg, vert_buff_1, svdup_n_s16(10), vert_result);
251
252 50720 return svcreate2(hori_result, vert_result);
253 25360 }
254
255 17472 void process_horizontal(Rows<DestinationType> dst_rows) KLEIDICV_STREAMING {
256 // width is decremented by 2 as the result has less columns.
257 34944 LoopUnroll2 loop{(width_ - 2) * hori_deriv_buffer_.channels(),
258 17472 BufferVecTraits::num_lanes()};
259 17472 svbool_t pg_all = BufferVecTraits::svptrue();
260 34944 const ptrdiff_t channel =
261 17472 static_cast<ptrdiff_t>(hori_deriv_buffer_.channels());
262
263 20990 loop.unroll_twice([&](ptrdiff_t index) KLEIDICV_STREAMING {
264 3518 horizontal_vector_path_x2(pg_all, dst_rows, index, channel);
265 3518 });
266
267 19888 loop.unroll_once([&](ptrdiff_t index) KLEIDICV_STREAMING {
268 2416 horizontal_vector_path(pg_all, dst_rows, index, channel);
269 2416 });
270
271 33380 loop.remaining([&](ptrdiff_t index, ptrdiff_t length) KLEIDICV_STREAMING {
272 15908 svbool_t pg = BufferVecTraits::svwhilelt(index, length);
273 15908 horizontal_vector_path(pg, dst_rows, index, channel);
274 15908 });
275 17472 }
276
277 Rows<BufferType> hori_deriv_buffer_;
278 Rows<BufferType> vert_deriv_buffer_;
279 size_t width_;
280 }; // end of class ScharrInterleaved
281
282 class ScharrBufferDeleter {
283 public:
284 990 void operator()(void *ptr) const { std::free(ptr); }
285 };
286
287 1008 static kleidicv_error_t kleidicv_scharr_interleaved_stripe_s16_u8_sc(
288 const uint8_t *src, size_t src_stride, size_t src_width, size_t src_height,
289 size_t src_channels, int16_t *dst, size_t dst_stride, size_t y_begin,
290 size_t y_end) KLEIDICV_STREAMING {
291 // Does not include checks for whether the operation is implemented.
292 // This must be done earlier, by scharr_interleaved_is_implemented.
293
4/4
✓ Branch 0 taken 3 times.
✓ Branch 1 taken 1005 times.
✓ Branch 2 taken 3 times.
✓ Branch 3 taken 1005 times.
 1008 CHECK_POINTER_AND_STRIDE(src, src_stride, src_height);
294
4/4
✓ Branch 0 taken 6 times.
✓ Branch 1 taken 999 times.
✓ Branch 2 taken 6 times.
✓ Branch 3 taken 999 times.
 1005 CHECK_POINTER_AND_STRIDE(dst, dst_stride, src_height);
295
6/6
✓ Branch 0 taken 3 times.
✓ Branch 1 taken 996 times.
✓ Branch 2 taken 3 times.
✓ Branch 3 taken 993 times.
✓ Branch 4 taken 6 times.
✓ Branch 5 taken 993 times.
 999 CHECK_IMAGE_SIZE(src_width, src_height);
296
297 // Allocating more elements because in case of SVE interleaving stores are
298 // governed by one predicate. For example, if a predicate requires 7 uint8_t
299 // elements and an algorithm performs widening to 16 bits, the resulting
300 // interleaving store will still be governed by the same predicate, thus
301 // storing 8 elements. Choosing '3' to account for svst4().
302 993 size_t buffer_stride = ((src_width * src_channels) + 3) * sizeof(int16_t);
303 // Buffer has two rows, one for the horizontal derivative approximation, one
304 // for the vertical one.
305 993 size_t buffer_height = 2;
306 // Memory is allocated with malloc to avoid its initialization.
307 993 void *allocation = std::malloc(buffer_stride * buffer_height);
308
309
2/2
✓ Branch 0 taken 990 times.
✓ Branch 1 taken 3 times.
 993 if (!allocation) {
310 3 return KLEIDICV_ERROR_ALLOCATION;
311 }
312
313 1980 std::unique_ptr<int16_t, ScharrBufferDeleter> buffer(
314 990 reinterpret_cast<int16_t *>(allocation));
315
316 990 Rows<const uint8_t> src_rows{src, src_stride, src_channels};
317
318 // Result is treated as it has double the channel number compared to the
319 // input.
320 990 Rows<int16_t> dst_rows{dst, dst_stride, src_channels * 2};
321
322 990 Rows<int16_t> hori_deriv_buffer{buffer.get(), buffer_stride, src_channels};
323
324 1980 int16_t *vert_deriv_ptr = reinterpret_cast<int16_t *>(
325 990 reinterpret_cast<uint8_t *>(buffer.get()) + buffer_stride);
326 990 Rows<int16_t> vert_deriv_buffer{vert_deriv_ptr, buffer_stride, src_channels};
327
328 1980 ScharrInterleaved(hori_deriv_buffer, vert_deriv_buffer, src_width)
329 990 .process(src_rows, dst_rows, y_begin, y_end);
330
331 990 return KLEIDICV_OK;
332 1008 }
333 } // namespace KLEIDICV_TARGET_NAMESPACE
334