| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | // SPDX-FileCopyrightText: 2024 - 2026 Arm Limited and/or its affiliates <open-source-office@arm.com> | ||
| 2 | // | ||
| 3 | // SPDX-License-Identifier: Apache-2.0 | ||
| 4 | |||
| 5 | #include "kleidicv/ctypes.h" | ||
| 6 | #include "kleidicv/filters/blur_and_downsample.h" | ||
| 7 | #include "kleidicv/kleidicv.h" | ||
| 8 | #include "kleidicv/sve2.h" | ||
| 9 | #include "kleidicv/utils.h" | ||
| 10 | #include "kleidicv/workspace/blur_and_downsample_ws.h" | ||
| 11 | #include "kleidicv/workspace/border_5x5.h" | ||
| 12 | |||
| 13 | namespace KLEIDICV_TARGET_NAMESPACE { | ||
| 14 | |||
| 15 | // Applies Gaussian Blur binomial filter to even rows and columns | ||
| 16 | // | ||
| 17 | // [ 1, 4, 6, 4, 1 ] [ 1 ] | ||
| 18 | // [ 4, 16, 24, 16, 4 ] [ 4 ] | ||
| 19 | // F = 1/256 * [ 6, 24, 36, 24, 6 ] = 1/256 * [ 6 ] * [ 1, 4, 6, 4, 1 ] | ||
| 20 | // [ 4, 16, 24, 16, 4 ] [ 4 ] | ||
| 21 | // [ 1, 4, 6, 4, 1 ] [ 1 ] | ||
| 22 | class BlurAndDownsample { | ||
| 23 | public: | ||
| 24 | using SourceType = uint8_t; | ||
| 25 | using BufferType = uint16_t; | ||
| 26 | using DestinationType = uint8_t; | ||
| 27 | using SourceVecTraits = | ||
| 28 | typename ::KLEIDICV_TARGET_NAMESPACE::VecTraits<SourceType>; | ||
| 29 | using SourceVectorType = typename SourceVecTraits::VectorType; | ||
| 30 | using SourceVector2Type = typename SourceVecTraits::Vector2Type; | ||
| 31 | using BufferVecTraits = | ||
| 32 | typename ::KLEIDICV_TARGET_NAMESPACE::VecTraits<BufferType>; | ||
| 33 | using BufferVectorType = typename BufferVecTraits::VectorType; | ||
| 34 | using BorderInfoType = | ||
| 35 | typename ::KLEIDICV_TARGET_NAMESPACE::FixedBorderInfo5x5<SourceType>; | ||
| 36 | using BorderType = FixedBorderType; | ||
| 37 | using BorderOffsets = typename BorderInfoType::Offsets; | ||
| 38 | |||
| 39 | static constexpr size_t margin = 2UL; | ||
| 40 | |||
| 41 | 7533 | void process_vertical(size_t width, Rows<const SourceType> src_rows, | |
| 42 | Rows<BufferType> dst_rows, | ||
| 43 | BorderOffsets border_offsets) const KLEIDICV_STREAMING { | ||
| 44 | 7533 | LoopUnroll2 loop{width * src_rows.channels(), SourceVecTraits::num_lanes()}; | |
| 45 | |||
| 46 | 37063 | loop.unroll_twice([&](ptrdiff_t index) KLEIDICV_STREAMING { | |
| 47 | 29530 | svbool_t pg_all = SourceVecTraits::svptrue(); | |
| 48 | 59060 | vertical_vector_path_2x(pg_all, src_rows, dst_rows, border_offsets, | |
| 49 | 29530 | index); | |
| 50 | 29530 | }); | |
| 51 | |||
| 52 | 9575 | loop.unroll_once([&](ptrdiff_t index) KLEIDICV_STREAMING { | |
| 53 | 2042 | svbool_t pg_all = SourceVecTraits::svptrue(); | |
| 54 | 4084 | vertical_vector_path_1x(pg_all, src_rows, dst_rows, border_offsets, | |
| 55 | 2042 | index); | |
| 56 | 2042 | }); | |
| 57 | |||
| 58 | 12860 | loop.remaining([&](ptrdiff_t index, ptrdiff_t length) KLEIDICV_STREAMING { | |
| 59 | 5327 | svbool_t pg = SourceVecTraits::svwhilelt(index, length); | |
| 60 | 5327 | vertical_vector_path_1x(pg, src_rows, dst_rows, border_offsets, index); | |
| 61 | 5327 | }); | |
| 62 | 7533 | } | |
| 63 | |||
| 64 | 7533 | void process_horizontal(size_t width, Rows<const BufferType> src_rows, | |
| 65 | Rows<DestinationType> dst_rows, | ||
| 66 | BorderOffsets border_offsets) const | ||
| 67 | KLEIDICV_STREAMING { | ||
| 68 |
4/4✓ Branch 0 taken 2220 times.
✓ Branch 1 taken 2280 times.
✓ Branch 2 taken 1497 times.
✓ Branch 3 taken 1536 times.
|
7533 | switch (src_rows.channels()) { |
| 69 | case 1: | ||
| 70 | 1497 | process_horizontal<1>(width, src_rows, dst_rows, border_offsets); | |
| 71 | 1497 | break; | |
| 72 | case 2: | ||
| 73 | 2220 | process_horizontal<2>(width, src_rows, dst_rows, border_offsets); | |
| 74 | 2220 | break; | |
| 75 | case 3: | ||
| 76 | 1536 | process_horizontal_3channels(width, src_rows, dst_rows, border_offsets); | |
| 77 | 1536 | break; | |
| 78 | default /* channel == 4 */: | ||
| 79 | 2280 | process_horizontal<4>(width, src_rows, dst_rows, border_offsets); | |
| 80 | 2280 | break; | |
| 81 | } | ||
| 82 | 7533 | } | |
| 83 | |||
| 84 | 15066 | void process_horizontal_borders( | |
| 85 | Rows<const BufferType> src_rows, Rows<DestinationType> dst_rows, | ||
| 86 | BorderOffsets border_offsets) const KLEIDICV_STREAMING { | ||
| 87 |
2/2✓ Branch 0 taken 15066 times.
✓ Branch 1 taken 39330 times.
|
54396 | for (ptrdiff_t index = 0; |
| 88 | 54396 | index < static_cast<ptrdiff_t>(src_rows.channels()); ++index) { | |
| 89 | 39330 | disable_loop_vectorization(); | |
| 90 | 39330 | svbool_t pg = svptrue_pat_b8(SV_VL1); | |
| 91 | 39330 | horizontal_border_path(pg, src_rows, dst_rows, border_offsets, index); | |
| 92 | 39330 | } | |
| 93 | 15066 | } | |
| 94 | |||
| 95 | private: | ||
| 96 | template <ptrdiff_t Channels> | ||
| 97 | 5997 | void process_horizontal(size_t width, Rows<const BufferType> src_rows, | |
| 98 | Rows<DestinationType> dst_rows, | ||
| 99 | BorderOffsets border_offsets) const | ||
| 100 | KLEIDICV_STREAMING { | ||
| 101 | static_assert(Channels == 1 || Channels == 2 || Channels == 4); | ||
| 102 | 5997 | svbool_t pg_all = BufferVecTraits::svptrue(); | |
| 103 | 5997 | LoopUnroll2 loop{width * src_rows.channels(), BufferVecTraits::num_lanes()}; | |
| 104 | |||
| 105 | 51831 | loop.unroll_twice([&](size_t index) KLEIDICV_STREAMING { | |
| 106 | 91668 | horizontal_vector_path_2x<Channels>(pg_all, pg_all, src_rows, pg_all, | |
| 107 | 45834 | dst_rows, border_offsets, | |
| 108 | 45834 | static_cast<ptrdiff_t>(index)); | |
| 109 | 45834 | }); | |
| 110 | |||
| 111 | 10794 | loop.remaining([&](size_t index, size_t length) KLEIDICV_STREAMING { | |
| 112 | 4797 | svbool_t pg_src_0 = BufferVecTraits::svwhilelt(index, length); | |
| 113 | 9594 | svbool_t pg_src_1 = BufferVecTraits::svwhilelt( | |
| 114 | 4797 | index + BufferVecTraits::num_lanes(), length); | |
| 115 | 4797 | size_t dst_length = ((width + 1) / 2) * src_rows.channels(); | |
| 116 | 4797 | svbool_t pg_dst = BufferVecTraits::svwhilelt(index / 2, dst_length); | |
| 117 | 9594 | horizontal_vector_path_2x<Channels>(pg_src_0, pg_src_1, src_rows, pg_dst, | |
| 118 | 4797 | dst_rows, border_offsets, | |
| 119 | 4797 | static_cast<ptrdiff_t>(index)); | |
| 120 | 4797 | }); | |
| 121 | 5997 | } | |
| 122 | |||
| 123 | 29530 | void vertical_vector_path_2x(svbool_t pg, Rows<const SourceType> src_rows, | |
| 124 | Rows<BufferType> dst_rows, | ||
| 125 | BorderOffsets border_offsets, | ||
| 126 | ptrdiff_t index) const KLEIDICV_STREAMING { | ||
| 127 | 29530 | const auto *src_row_0 = &src_rows.at(border_offsets.c0())[index]; | |
| 128 | 29530 | const auto *src_row_1 = &src_rows.at(border_offsets.c1())[index]; | |
| 129 | 29530 | const auto *src_row_2 = &src_rows.at(border_offsets.c2())[index]; | |
| 130 | 29530 | const auto *src_row_3 = &src_rows.at(border_offsets.c3())[index]; | |
| 131 | 29530 | const auto *src_row_4 = &src_rows.at(border_offsets.c4())[index]; | |
| 132 | |||
| 133 | 29530 | SourceVector2Type src_0; | |
| 134 | 29530 | SourceVector2Type src_1; | |
| 135 | 29530 | SourceVector2Type src_2; | |
| 136 | 29530 | SourceVector2Type src_3; | |
| 137 | 29530 | SourceVector2Type src_4; | |
| 138 | |||
| 139 | 29530 | src_0 = | |
| 140 | 29530 | svcreate2(svld1(pg, &src_row_0[0]), svld1_vnum(pg, &src_row_0[0], 1)); | |
| 141 | 29530 | src_1 = | |
| 142 | 29530 | svcreate2(svld1(pg, &src_row_1[0]), svld1_vnum(pg, &src_row_1[0], 1)); | |
| 143 | 29530 | src_2 = | |
| 144 | 29530 | svcreate2(svld1(pg, &src_row_2[0]), svld1_vnum(pg, &src_row_2[0], 1)); | |
| 145 | 29530 | src_3 = | |
| 146 | 29530 | svcreate2(svld1(pg, &src_row_3[0]), svld1_vnum(pg, &src_row_3[0], 1)); | |
| 147 | 29530 | src_4 = | |
| 148 | 29530 | svcreate2(svld1(pg, &src_row_4[0]), svld1_vnum(pg, &src_row_4[0], 1)); | |
| 149 | |||
| 150 | 59060 | vertical_vector_path(pg, svget2(src_0, 0), svget2(src_1, 0), | |
| 151 | 29530 | svget2(src_2, 0), svget2(src_3, 0), svget2(src_4, 0), | |
| 152 | 29530 | &dst_rows[index]); | |
| 153 | 59060 | vertical_vector_path(pg, svget2(src_0, 1), svget2(src_1, 1), | |
| 154 | 29530 | svget2(src_2, 1), svget2(src_3, 1), svget2(src_4, 1), | |
| 155 | 59060 | &dst_rows[index + static_cast<ptrdiff_t>( | |
| 156 | 29530 | SourceVecTraits::num_lanes())]); | |
| 157 | 29530 | } | |
| 158 | |||
| 159 | 7369 | void vertical_vector_path_1x(svbool_t pg, Rows<const SourceType> src_rows, | |
| 160 | Rows<BufferType> dst_rows, | ||
| 161 | BorderOffsets border_offsets, | ||
| 162 | ptrdiff_t index) const KLEIDICV_STREAMING { | ||
| 163 | 14738 | SourceVectorType src_0 = | |
| 164 | 7369 | svld1(pg, &src_rows.at(border_offsets.c0())[index]); | |
| 165 | 14738 | SourceVectorType src_1 = | |
| 166 | 7369 | svld1(pg, &src_rows.at(border_offsets.c1())[index]); | |
| 167 | 14738 | SourceVectorType src_2 = | |
| 168 | 7369 | svld1(pg, &src_rows.at(border_offsets.c2())[index]); | |
| 169 | 14738 | SourceVectorType src_3 = | |
| 170 | 7369 | svld1(pg, &src_rows.at(border_offsets.c3())[index]); | |
| 171 | 14738 | SourceVectorType src_4 = | |
| 172 | 7369 | svld1(pg, &src_rows.at(border_offsets.c4())[index]); | |
| 173 | 14738 | vertical_vector_path(pg, src_0, src_1, src_2, src_3, src_4, | |
| 174 | 7369 | &dst_rows[index]); | |
| 175 | 7369 | } | |
| 176 | |||
| 177 | // Applies vertical filtering vector using SIMD operations. | ||
| 178 | // | ||
| 179 | // DST = [ SRC0, SRC1, SRC2, SRC3, SRC4 ] * [ 1, 4, 6, 4, 1 ]T | ||
| 180 | 66429 | void vertical_vector_path(svbool_t pg, svuint8_t src_0, svuint8_t src_1, | |
| 181 | svuint8_t src_2, svuint8_t src_3, svuint8_t src_4, | ||
| 182 | BufferType *dst) const KLEIDICV_STREAMING { | ||
| 183 | 66429 | svuint16_t acc_0_4_b = svaddlb_u16(src_0, src_4); | |
| 184 | 66429 | svuint16_t acc_0_4_t = svaddlt_u16(src_0, src_4); | |
| 185 | 66429 | svuint16_t acc_1_3_b = svaddlb_u16(src_1, src_3); | |
| 186 | 66429 | svuint16_t acc_1_3_t = svaddlt_u16(src_1, src_3); | |
| 187 | |||
| 188 | 66429 | svuint16_t acc_u16_b = svmlalb_n_u16(acc_0_4_b, src_2, 6); | |
| 189 | 66429 | svuint16_t acc_u16_t = svmlalt_n_u16(acc_0_4_t, src_2, 6); | |
| 190 | 66429 | acc_u16_b = svmla_n_u16_x(pg, acc_u16_b, acc_1_3_b, 4); | |
| 191 | 66429 | acc_u16_t = svmla_n_u16_x(pg, acc_u16_t, acc_1_3_t, 4); | |
| 192 | |||
| 193 | 66429 | svuint16x2_t interleaved = svcreate2(acc_u16_b, acc_u16_t); | |
| 194 | 66429 | svst2(pg, &dst[0], interleaved); | |
| 195 | 66429 | } | |
| 196 | |||
| 197 | template <ptrdiff_t Channels> | ||
| 198 | 50631 | void horizontal_vector_path_2x(svbool_t pg_src_0, svbool_t pg_src_1, | |
| 199 | Rows<const BufferType> src_rows, | ||
| 200 | svbool_t pg_dst, | ||
| 201 | Rows<DestinationType> dst_rows, | ||
| 202 | BorderOffsets border_offsets, | ||
| 203 | ptrdiff_t index) const KLEIDICV_STREAMING { | ||
| 204 | 50631 | const auto *src_0 = &src_rows.at(0, border_offsets.c0())[index]; | |
| 205 | 50631 | const auto *src_1 = &src_rows.at(0, border_offsets.c1())[index]; | |
| 206 | 50631 | const auto *src_2 = &src_rows.at(0, border_offsets.c2())[index]; | |
| 207 | 50631 | const auto *src_3 = &src_rows.at(0, border_offsets.c3())[index]; | |
| 208 | 50631 | const auto *src_4 = &src_rows.at(0, border_offsets.c4())[index]; | |
| 209 | |||
| 210 | 50631 | BufferVectorType src_0_0 = svld1(pg_src_0, &src_0[0]); | |
| 211 | 50631 | BufferVectorType src_1_0 = svld1_vnum(pg_src_1, &src_0[0], 1); | |
| 212 | 50631 | BufferVectorType src_0_1 = svld1(pg_src_0, &src_1[0]); | |
| 213 | 50631 | BufferVectorType src_1_1 = svld1_vnum(pg_src_1, &src_1[0], 1); | |
| 214 | 50631 | BufferVectorType src_0_2 = svld1(pg_src_0, &src_2[0]); | |
| 215 | 50631 | BufferVectorType src_1_2 = svld1_vnum(pg_src_1, &src_2[0], 1); | |
| 216 | 50631 | BufferVectorType src_0_3 = svld1(pg_src_0, &src_3[0]); | |
| 217 | 50631 | BufferVectorType src_1_3 = svld1_vnum(pg_src_1, &src_3[0], 1); | |
| 218 | 50631 | BufferVectorType src_0_4 = svld1(pg_src_0, &src_4[0]); | |
| 219 | 50631 | BufferVectorType src_1_4 = svld1_vnum(pg_src_1, &src_4[0], 1); | |
| 220 | |||
| 221 | 101262 | svuint16_t res_0 = horizontal_vector_path(pg_src_0, src_0_0, src_0_1, | |
| 222 | 50631 | src_0_2, src_0_3, src_0_4); | |
| 223 | 101262 | svuint16_t res_1 = horizontal_vector_path(pg_src_1, src_1_0, src_1_1, | |
| 224 | 50631 | src_1_2, src_1_3, src_1_4); | |
| 225 | |||
| 226 | if constexpr (Channels == 1) { | ||
| 227 | 4427 | svuint16_t res_even_only = svuzp1(res_0, res_1); | |
| 228 | 4427 | svst1b(pg_dst, &dst_rows[index / 2], res_even_only); | |
| 229 | 4427 | } else if constexpr (Channels == 2) { | |
| 230 | 30468 | svuint16_t res_even_only = svreinterpret_u16( | |
| 231 | 15234 | svuzp1(svreinterpret_u32(res_0), svreinterpret_u32(res_1))); | |
| 232 | 15234 | svst1b(pg_dst, &dst_rows[index / 2], res_even_only); | |
| 233 | 15234 | } else if constexpr (Channels == 4) { | |
| 234 | 61940 | svuint16_t res_even_only = svreinterpret_u16( | |
| 235 | 30970 | svuzp1(svreinterpret_u64(res_0), svreinterpret_u64(res_1))); | |
| 236 | 30970 | svst1b(pg_dst, &dst_rows[index / 2], res_even_only); | |
| 237 | 30970 | } | |
| 238 | 50631 | } | |
| 239 | |||
| 240 | // Applies horizontal filtering vector using SIMD operations. | ||
| 241 | // | ||
| 242 | // DST = 1/256 * [ SRC0, SRC1, SRC2, SRC3, SRC4 ] * [ 1, 4, 6, 4, 1 ]T | ||
| 243 | 129306 | svuint16_t horizontal_vector_path(svbool_t pg, svuint16_t src_0, | |
| 244 | svuint16_t src_1, svuint16_t src_2, | ||
| 245 | svuint16_t src_3, | ||
| 246 | svuint16_t src_4) const KLEIDICV_STREAMING { | ||
| 247 | 129306 | svuint16_t acc_0_4 = svadd_x(pg, src_0, src_4); | |
| 248 | 129306 | svuint16_t acc_1_3 = svadd_x(pg, src_1, src_3); | |
| 249 | 129306 | svuint16_t acc = svmla_n_u16_x(pg, acc_0_4, src_2, 6); | |
| 250 | 129306 | acc = svmla_n_u16_x(pg, acc, acc_1_3, 4); | |
| 251 | 129306 | acc = svrshr_x(pg, acc, 8); | |
| 252 | 258612 | return acc; | |
| 253 | 129306 | } | |
| 254 | |||
| 255 | // 3-channel data is interleaved per RGB triplet, so even-column decimation | ||
| 256 | // cannot use the shared 1/2/4 packed unzip path. | ||
| 257 | 4674 | void horizontal_vector_path_3_channel_2x( | |
| 258 | svbool_t pg_src_0, svbool_t pg_src_1, svbool_t pg_dst, | ||
| 259 | Rows<const BufferType> src_rows, Rows<DestinationType> dst_rows, | ||
| 260 | BorderOffsets border_offsets, ptrdiff_t column, | ||
| 261 | ptrdiff_t vec_stride) const KLEIDICV_STREAMING { | ||
| 262 | 4674 | constexpr ptrdiff_t channels = 3; | |
| 263 | 4674 | const auto *src_0 = &src_rows.at(0, border_offsets.c0())[column * channels]; | |
| 264 | 4674 | const auto *src_1 = &src_rows.at(0, border_offsets.c1())[column * channels]; | |
| 265 | 4674 | const auto *src_2 = &src_rows.at(0, border_offsets.c2())[column * channels]; | |
| 266 | 4674 | const auto *src_3 = &src_rows.at(0, border_offsets.c3())[column * channels]; | |
| 267 | 4674 | const auto *src_4 = &src_rows.at(0, border_offsets.c4())[column * channels]; | |
| 268 | |||
| 269 | 4674 | svuint16x3_t src0_a = svld3(pg_src_0, &src_0[0]); | |
| 270 | 4674 | svuint16x3_t src1_a = svld3(pg_src_0, &src_1[0]); | |
| 271 | 4674 | svuint16x3_t src2_a = svld3(pg_src_0, &src_2[0]); | |
| 272 | 4674 | svuint16x3_t src3_a = svld3(pg_src_0, &src_3[0]); | |
| 273 | 4674 | svuint16x3_t src4_a = svld3(pg_src_0, &src_4[0]); | |
| 274 | |||
| 275 | 4674 | svuint16x3_t src0_b = svld3(pg_src_1, &src_0[vec_stride]); | |
| 276 | 4674 | svuint16x3_t src1_b = svld3(pg_src_1, &src_1[vec_stride]); | |
| 277 | 4674 | svuint16x3_t src2_b = svld3(pg_src_1, &src_2[vec_stride]); | |
| 278 | 4674 | svuint16x3_t src3_b = svld3(pg_src_1, &src_3[vec_stride]); | |
| 279 | 4674 | svuint16x3_t src4_b = svld3(pg_src_1, &src_4[vec_stride]); | |
| 280 | |||
| 281 | 9348 | svuint16_t res0_a = horizontal_vector_path( | |
| 282 | 4674 | pg_src_0, svget3(src0_a, 0), svget3(src1_a, 0), svget3(src2_a, 0), | |
| 283 | 4674 | svget3(src3_a, 0), svget3(src4_a, 0)); | |
| 284 | 9348 | svuint16_t res0_b = horizontal_vector_path( | |
| 285 | 4674 | pg_src_1, svget3(src0_b, 0), svget3(src1_b, 0), svget3(src2_b, 0), | |
| 286 | 4674 | svget3(src3_b, 0), svget3(src4_b, 0)); | |
| 287 | 9348 | svuint16_t res1_a = horizontal_vector_path( | |
| 288 | 4674 | pg_src_0, svget3(src0_a, 1), svget3(src1_a, 1), svget3(src2_a, 1), | |
| 289 | 4674 | svget3(src3_a, 1), svget3(src4_a, 1)); | |
| 290 | 9348 | svuint16_t res1_b = horizontal_vector_path( | |
| 291 | 4674 | pg_src_1, svget3(src0_b, 1), svget3(src1_b, 1), svget3(src2_b, 1), | |
| 292 | 4674 | svget3(src3_b, 1), svget3(src4_b, 1)); | |
| 293 | 9348 | svuint16_t res2_a = horizontal_vector_path( | |
| 294 | 4674 | pg_src_0, svget3(src0_a, 2), svget3(src1_a, 2), svget3(src2_a, 2), | |
| 295 | 4674 | svget3(src3_a, 2), svget3(src4_a, 2)); | |
| 296 | 9348 | svuint16_t res2_b = horizontal_vector_path( | |
| 297 | 4674 | pg_src_1, svget3(src0_b, 2), svget3(src1_b, 2), svget3(src2_b, 2), | |
| 298 | 4674 | svget3(src3_b, 2), svget3(src4_b, 2)); | |
| 299 | |||
| 300 | 4674 | svuint16_t out0 = svuzp1(res0_a, res0_b); | |
| 301 | 4674 | svuint16_t out1 = svuzp1(res1_a, res1_b); | |
| 302 | 4674 | svuint16_t out2 = svuzp1(res2_a, res2_b); | |
| 303 | |||
| 304 | 4674 | svuint8_t out0_u8 = svuzp1(svreinterpret_u8(out0), svreinterpret_u8(out0)); | |
| 305 | 4674 | svuint8_t out1_u8 = svuzp1(svreinterpret_u8(out1), svreinterpret_u8(out1)); | |
| 306 | 4674 | svuint8_t out2_u8 = svuzp1(svreinterpret_u8(out2), svreinterpret_u8(out2)); | |
| 307 | |||
| 308 | 4674 | svuint8x3_t out_rgb = svcreate3(out0_u8, out1_u8, out2_u8); | |
| 309 | 4674 | svst3(pg_dst, &dst_rows.at(0, column / 2)[0], out_rgb); | |
| 310 | 4674 | } | |
| 311 | |||
| 312 | 1536 | void process_horizontal_3channels( | |
| 313 | size_t width, Rows<const BufferType> src_rows, | ||
| 314 | Rows<DestinationType> dst_rows, | ||
| 315 | BorderOffsets border_offsets) const KLEIDICV_STREAMING { | ||
| 316 | 1536 | constexpr ptrdiff_t channels = 3; | |
| 317 | 3072 | const ptrdiff_t vec_lanes = | |
| 318 | 1536 | static_cast<ptrdiff_t>(BufferVecTraits::num_lanes()); | |
| 319 | 1536 | const ptrdiff_t vec_stride = vec_lanes * channels; | |
| 320 | 1536 | svbool_t pg_all = BufferVecTraits::svptrue(); | |
| 321 | 3072 | svbool_t pg_dst_all = svwhilelt_b8(static_cast<uint64_t>(0), | |
| 322 | 1536 | static_cast<uint64_t>(vec_lanes)); | |
| 323 | 1536 | LoopUnroll2 loop{width, static_cast<size_t>(vec_lanes)}; | |
| 324 | |||
| 325 | 4932 | loop.unroll_twice([&](ptrdiff_t column) KLEIDICV_STREAMING { | |
| 326 | 6792 | horizontal_vector_path_3_channel_2x(pg_all, pg_all, pg_dst_all, src_rows, | |
| 327 | 3396 | dst_rows, border_offsets, column, | |
| 328 | 3396 | vec_stride); | |
| 329 | 3396 | }); | |
| 330 | |||
| 331 | 2814 | loop.remaining([&](ptrdiff_t column, ptrdiff_t length) KLEIDICV_STREAMING { | |
| 332 | 1278 | column = align_up(column, 2UL); | |
| 333 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 1278 times.
|
1278 | if (column < length) { |
| 334 | 1278 | svbool_t pg_src_0 = BufferVecTraits::svwhilelt(column, length); | |
| 335 | 2556 | svbool_t pg_src_1 = | |
| 336 | 1278 | BufferVecTraits::svwhilelt(column + vec_lanes, length); | |
| 337 | 1278 | size_t dst_length = (width + 1) / 2; | |
| 338 | 2556 | svbool_t pg_dst = svwhilelt_b8(static_cast<uint64_t>(column / 2), | |
| 339 | 1278 | static_cast<uint64_t>(dst_length)); | |
| 340 | 2556 | horizontal_vector_path_3_channel_2x(pg_src_0, pg_src_1, pg_dst, | |
| 341 | 1278 | src_rows, dst_rows, border_offsets, | |
| 342 | 1278 | column, vec_stride); | |
| 343 | 1278 | } | |
| 344 | 1278 | }); | |
| 345 | 1536 | } | |
| 346 | |||
| 347 | // Applies horizontal filtering for the borders using SIMD operations. | ||
| 348 | // | ||
| 349 | // DST = 1/256 * [ SRC0, SRC1, SRC2, SRC3, SRC4 ] * [ 1, 4, 6, 4, 1 ]T | ||
| 350 | 39330 | void horizontal_border_path(svbool_t pg, Rows<const BufferType> src_rows, | |
| 351 | Rows<DestinationType> dst_rows, | ||
| 352 | BorderOffsets border_offsets, | ||
| 353 | ptrdiff_t index) const KLEIDICV_STREAMING { | ||
| 354 | 78660 | BufferVectorType src_0 = | |
| 355 | 39330 | svld1(pg, &src_rows.at(0, border_offsets.c0())[index]); | |
| 356 | 78660 | BufferVectorType src_1 = | |
| 357 | 39330 | svld1(pg, &src_rows.at(0, border_offsets.c1())[index]); | |
| 358 | 78660 | BufferVectorType src_2 = | |
| 359 | 39330 | svld1(pg, &src_rows.at(0, border_offsets.c2())[index]); | |
| 360 | 78660 | BufferVectorType src_3 = | |
| 361 | 39330 | svld1(pg, &src_rows.at(0, border_offsets.c3())[index]); | |
| 362 | 78660 | BufferVectorType src_4 = | |
| 363 | 39330 | svld1(pg, &src_rows.at(0, border_offsets.c4())[index]); | |
| 364 | |||
| 365 | 39330 | svuint16_t acc_0_4 = svadd_x(pg, src_0, src_4); | |
| 366 | 39330 | svuint16_t acc_1_3 = svadd_x(pg, src_1, src_3); | |
| 367 | 39330 | svuint16_t acc = svmla_n_u16_x(pg, acc_0_4, src_2, 6); | |
| 368 | 39330 | acc = svmla_n_u16_x(pg, acc, acc_1_3, 4); | |
| 369 | 39330 | acc = svrshr_x(pg, acc, 8); | |
| 370 | |||
| 371 | 39330 | svst1b(pg, &dst_rows[index], acc); | |
| 372 | 39330 | } | |
| 373 | }; // end of class BlurAndDownsample | ||
| 374 | |||
| 375 | 426 | static kleidicv_error_t blur_and_downsample_stripe_u8_sc( | |
| 376 | const uint8_t *src, size_t src_stride, size_t src_width, size_t src_height, | ||
| 377 | uint8_t *dst, size_t dst_stride, size_t y_begin, size_t y_end, | ||
| 378 | size_t channels, FixedBorderType fixed_border_type) KLEIDICV_STREAMING { | ||
| 379 |
4/4✓ Branch 0 taken 3 times.
✓ Branch 1 taken 423 times.
✓ Branch 2 taken 3 times.
✓ Branch 3 taken 423 times.
|
426 | CHECK_POINTER_AND_STRIDE(src, src_stride, src_height); |
| 380 |
4/4✓ Branch 0 taken 3 times.
✓ Branch 1 taken 420 times.
✓ Branch 2 taken 3 times.
✓ Branch 3 taken 420 times.
|
423 | CHECK_POINTER_AND_STRIDE(dst, dst_stride, (src_height + 1) / 2); |
| 381 |
6/6✓ Branch 0 taken 3 times.
✓ Branch 1 taken 417 times.
✓ Branch 2 taken 3 times.
✓ Branch 3 taken 414 times.
✓ Branch 4 taken 6 times.
✓ Branch 5 taken 414 times.
|
420 | CHECK_IMAGE_SIZE(src_width, src_height); |
| 382 | 414 | Rectangle rect{src_width, src_height}; | |
| 383 | 414 | constexpr size_t intermediate_size{ | |
| 384 | sizeof(typename BlurAndDownsample::BufferType)}; | ||
| 385 | |||
| 386 | 828 | auto workspace_variant = BlurAndDownsampleFilterWorkspace::create( | |
| 387 | 414 | rect, channels, intermediate_size); | |
| 388 |
4/4✓ Branch 0 taken 3 times.
✓ Branch 1 taken 411 times.
✓ Branch 2 taken 3 times.
✓ Branch 3 taken 411 times.
|
417 | if (auto *err = std::get_if<kleidicv_error_t>(&workspace_variant)) { |
| 389 | 3 | return *err; | |
| 390 | } | ||
| 391 | 822 | auto &workspace = | |
| 392 | 411 | *std::get_if<BlurAndDownsampleFilterWorkspace>(&workspace_variant); | |
| 393 | |||
| 394 | 411 | Rows<const uint8_t> src_rows{src, src_stride, channels}; | |
| 395 | 411 | Rows<uint8_t> dst_rows{dst, dst_stride, channels}; | |
| 396 | 411 | workspace.process(y_begin, y_end, src_rows, dst_rows, fixed_border_type, | |
| 397 | BlurAndDownsample{}); | ||
| 398 | |||
| 399 | 411 | return KLEIDICV_OK; | |
| 400 | 426 | } | |
| 401 | |||
| 402 | } // namespace KLEIDICV_TARGET_NAMESPACE | ||
| 403 |