KleidiCV Coverage Report

Directory:	./
File:	kleidicv/include/kleidicv/filters/separable_filter_21x21_sc.h
Date:	2025-09-25 14:13:34

	Exec	Total	Coverage
Lines:	233	233	100.0%
Functions:	26	26	100.0%
Branches:	2	2	100.0%

  
      Line
      Branch
      Exec
      Source
    
      // SPDX-FileCopyrightText: 2025 Arm Limited and/or its affiliates <open-source-office@arm.com>
    
      //
    
      // SPDX-License-Identifier: Apache-2.0
    
      #ifndef KLEIDICV_SEPARABLE_FILTER_21X21_SC_H
    
      #define KLEIDICV_SEPARABLE_FILTER_21X21_SC_H
    
      #include "kleidicv/sve2.h"
    
      #include "kleidicv/workspace/border_21x21.h"
    
      // It is used by SVE2 and SME, the actual namespace will reflect it.
    
      namespace KLEIDICV_TARGET_NAMESPACE {
    
      // Template for drivers of separable NxM filters.
    
      template <typename FilterType, const size_t S>
    
      class SeparableFilter;
    
      // Driver for a separable 21x21 filter.
    
      template <typename FilterType>
    
      class SeparableFilter<FilterType, 21UL> {
    
       public:
    
        using SourceType = typename FilterType::SourceType;
    
        using BufferType = typename FilterType::BufferType;
    
        using DestinationType = typename FilterType::DestinationType;
    
        using SourceVecTraits =
    
            typename ::KLEIDICV_TARGET_NAMESPACE::VecTraits<SourceType>;
    
        using SourceVectorType = typename SourceVecTraits::VectorType;
    
        using BufferVecTraits =
    
            typename ::KLEIDICV_TARGET_NAMESPACE::VecTraits<BufferType>;
    
        using BufferVectorType = typename BufferVecTraits::VectorType;
    
        using BorderInfoType =
    
            typename ::KLEIDICV_TARGET_NAMESPACE::FixedBorderInfo21x21<SourceType>;
    
        using BorderType = FixedBorderType;
    
        using BorderOffsets = typename BorderInfoType::Offsets;
    
      64
        explicit SeparableFilter(FilterType filter) KLEIDICV_STREAMING
    
      64
            : filter_{filter} {}
    
        static constexpr size_t margin = 10UL;
    
      1296
        void process_vertical(size_t width, Rows<const SourceType> src_rows,
    
                              Rows<BufferType> dst_rows,
    
                              BorderOffsets border_offsets) const KLEIDICV_STREAMING {
    
      1296
          LoopUnroll2 loop{width * src_rows.channels(), SourceVecTraits::num_lanes()};
    
      3384
          loop.unroll_once([&](size_t index) KLEIDICV_STREAMING {
    
      2088
            svbool_t pg_all = SourceVecTraits::svptrue();
    
      2088
            vertical_vector_path(pg_all, src_rows, dst_rows, border_offsets, index);
    
      2088
          });
    
      2592
          loop.remaining([&](size_t index, size_t length) KLEIDICV_STREAMING {
    
      1296
            svbool_t pg = SourceVecTraits::svwhilelt(index, length);
    
      1296
            vertical_vector_path(pg, src_rows, dst_rows, border_offsets, index);
    
      1296
          });
    
      1296
        }
    
      1296
        void process_horizontal(size_t width, Rows<const BufferType> src_rows,
    
                                Rows<DestinationType> dst_rows,
    
                                BorderOffsets border_offsets) const
    
            KLEIDICV_STREAMING {
    
      1296
          svbool_t pg_all = BufferVecTraits::svptrue();
    
      1296
          LoopUnroll2 loop{width * src_rows.channels(), BufferVecTraits::num_lanes()};
    
      1616
          loop.unroll_twice([&](size_t index) KLEIDICV_STREAMING {
    
      640
            horizontal_vector_path_2x(pg_all, src_rows, dst_rows, border_offsets,
    
      320
                                      index);
    
      320
          });
    
      1616
          loop.unroll_once([&](size_t index) KLEIDICV_STREAMING {
    
      320
            horizontal_vector_path(pg_all, src_rows, dst_rows, border_offsets, index);
    
      320
          });
    
      2272
          loop.remaining([&](size_t index, size_t length) KLEIDICV_STREAMING {
    
      976
            svbool_t pg = BufferVecTraits::svwhilelt(index, length);
    
      976
            horizontal_vector_path(pg, src_rows, dst_rows, border_offsets, index);
    
      976
          });
    
      1296
        }
    
        // Processing of horizontal borders is always scalar because border offsets
    
        // change for each and every element in the border.
    
      25920
        void process_horizontal_borders(
    
            Rows<const BufferType> src_rows, Rows<DestinationType> dst_rows,
    
            BorderOffsets border_offsets) const KLEIDICV_STREAMING {
    
        2/2✓ Branch 0 taken 25920 times.
✓ Branch 1 taken 45120 times.

      71040
          for (size_t index = 0; index < src_rows.channels(); ++index) {
    
      45120
            disable_loop_vectorization();
    
      45120
            process_horizontal_border(src_rows, dst_rows, border_offsets, index);
    
      45120
          }
    
      25920
        }
    
       private:
    
      3384
        void vertical_vector_path(svbool_t pg, Rows<const SourceType> src_rows,
    
                                  Rows<BufferType> dst_rows,
    
                                  BorderOffsets border_offsets,
    
                                  size_t index) const KLEIDICV_STREAMING {
    
      6768
          SourceVectorType src_0 =
    
      3384
              svld1(pg, &src_rows.at(border_offsets.c0())[index]);
    
      6768
          SourceVectorType src_1 =
    
      3384
              svld1(pg, &src_rows.at(border_offsets.c1())[index]);
    
      6768
          SourceVectorType src_2 =
    
      3384
              svld1(pg, &src_rows.at(border_offsets.c2())[index]);
    
      6768
          SourceVectorType src_3 =
    
      3384
              svld1(pg, &src_rows.at(border_offsets.c3())[index]);
    
      6768
          SourceVectorType src_4 =
    
      3384
              svld1(pg, &src_rows.at(border_offsets.c4())[index]);
    
      6768
          SourceVectorType src_5 =
    
      3384
              svld1(pg, &src_rows.at(border_offsets.c5())[index]);
    
      6768
          SourceVectorType src_6 =
    
      3384
              svld1(pg, &src_rows.at(border_offsets.c6())[index]);
    
      6768
          SourceVectorType src_7 =
    
      3384
              svld1(pg, &src_rows.at(border_offsets.c7())[index]);
    
      6768
          SourceVectorType src_8 =
    
      3384
              svld1(pg, &src_rows.at(border_offsets.c8())[index]);
    
      6768
          SourceVectorType src_9 =
    
      3384
              svld1(pg, &src_rows.at(border_offsets.c9())[index]);
    
      6768
          SourceVectorType src_10 =
    
      3384
              svld1(pg, &src_rows.at(border_offsets.c10())[index]);
    
      6768
          SourceVectorType src_11 =
    
      3384
              svld1(pg, &src_rows.at(border_offsets.c11())[index]);
    
      6768
          SourceVectorType src_12 =
    
      3384
              svld1(pg, &src_rows.at(border_offsets.c12())[index]);
    
      6768
          SourceVectorType src_13 =
    
      3384
              svld1(pg, &src_rows.at(border_offsets.c13())[index]);
    
      6768
          SourceVectorType src_14 =
    
      3384
              svld1(pg, &src_rows.at(border_offsets.c14())[index]);
    
      6768
          SourceVectorType src_15 =
    
      3384
              svld1(pg, &src_rows.at(border_offsets.c15())[index]);
    
      6768
          SourceVectorType src_16 =
    
      3384
              svld1(pg, &src_rows.at(border_offsets.c16())[index]);
    
      6768
          SourceVectorType src_17 =
    
      3384
              svld1(pg, &src_rows.at(border_offsets.c17())[index]);
    
      6768
          SourceVectorType src_18 =
    
      3384
              svld1(pg, &src_rows.at(border_offsets.c18())[index]);
    
      6768
          SourceVectorType src_19 =
    
      3384
              svld1(pg, &src_rows.at(border_offsets.c19())[index]);
    
      6768
          SourceVectorType src_20 =
    
      3384
              svld1(pg, &src_rows.at(border_offsets.c20())[index]);
    
      71064
          std::reference_wrapper<SourceVectorType> sources[21] = {
    
      23688
              src_0,  src_1,  src_2,  src_3,  src_4,  src_5,  src_6,
    
      23688
              src_7,  src_8,  src_9,  src_10, src_11, src_12, src_13,
    
      23688
              src_14, src_15, src_16, src_17, src_18, src_19, src_20};
    
      3384
          filter_.vertical_vector_path(pg, sources, &dst_rows[index]);
    
      3384
        }
    
      320
        void horizontal_vector_path_2x(svbool_t pg, Rows<const BufferType> src_rows,
    
                                       Rows<DestinationType> dst_rows,
    
                                       BorderOffsets border_offsets,
    
                                       size_t index) const KLEIDICV_STREAMING {
    
      320
          auto src_0 = &src_rows.at(0, border_offsets.c0())[index];
    
      320
          auto src_1 = &src_rows.at(0, border_offsets.c1())[index];
    
      320
          auto src_2 = &src_rows.at(0, border_offsets.c2())[index];
    
      320
          auto src_3 = &src_rows.at(0, border_offsets.c3())[index];
    
      320
          auto src_4 = &src_rows.at(0, border_offsets.c4())[index];
    
      320
          auto src_5 = &src_rows.at(0, border_offsets.c5())[index];
    
      320
          auto src_6 = &src_rows.at(0, border_offsets.c6())[index];
    
      320
          auto src_7 = &src_rows.at(0, border_offsets.c7())[index];
    
      320
          auto src_8 = &src_rows.at(0, border_offsets.c8())[index];
    
      320
          auto src_9 = &src_rows.at(0, border_offsets.c9())[index];
    
      320
          auto src_10 = &src_rows.at(0, border_offsets.c10())[index];
    
      320
          auto src_11 = &src_rows.at(0, border_offsets.c11())[index];
    
      320
          auto src_12 = &src_rows.at(0, border_offsets.c12())[index];
    
      320
          auto src_13 = &src_rows.at(0, border_offsets.c13())[index];
    
      320
          auto src_14 = &src_rows.at(0, border_offsets.c14())[index];
    
      320
          auto src_15 = &src_rows.at(0, border_offsets.c15())[index];
    
      320
          auto src_16 = &src_rows.at(0, border_offsets.c16())[index];
    
      320
          auto src_17 = &src_rows.at(0, border_offsets.c17())[index];
    
      320
          auto src_18 = &src_rows.at(0, border_offsets.c18())[index];
    
      320
          auto src_19 = &src_rows.at(0, border_offsets.c19())[index];
    
      320
          auto src_20 = &src_rows.at(0, border_offsets.c20())[index];
    
      320
          BufferVectorType src_0_0 = svld1(pg, &src_0[0]);
    
      320
          BufferVectorType src_1_0 = svld1_vnum(pg, &src_0[0], 1);
    
      320
          BufferVectorType src_0_1 = svld1(pg, &src_1[0]);
    
      320
          BufferVectorType src_1_1 = svld1_vnum(pg, &src_1[0], 1);
    
      320
          BufferVectorType src_0_2 = svld1(pg, &src_2[0]);
    
      320
          BufferVectorType src_1_2 = svld1_vnum(pg, &src_2[0], 1);
    
      320
          BufferVectorType src_0_3 = svld1(pg, &src_3[0]);
    
      320
          BufferVectorType src_1_3 = svld1_vnum(pg, &src_3[0], 1);
    
      320
          BufferVectorType src_0_4 = svld1(pg, &src_4[0]);
    
      320
          BufferVectorType src_1_4 = svld1_vnum(pg, &src_4[0], 1);
    
      320
          BufferVectorType src_0_5 = svld1(pg, &src_5[0]);
    
      320
          BufferVectorType src_1_5 = svld1_vnum(pg, &src_5[0], 1);
    
      320
          BufferVectorType src_0_6 = svld1(pg, &src_6[0]);
    
      320
          BufferVectorType src_1_6 = svld1_vnum(pg, &src_6[0], 1);
    
      320
          BufferVectorType src_0_7 = svld1(pg, &src_7[0]);
    
      320
          BufferVectorType src_1_7 = svld1_vnum(pg, &src_7[0], 1);
    
      320
          BufferVectorType src_0_8 = svld1(pg, &src_8[0]);
    
      320
          BufferVectorType src_1_8 = svld1_vnum(pg, &src_8[0], 1);
    
      320
          BufferVectorType src_0_9 = svld1(pg, &src_9[0]);
    
      320
          BufferVectorType src_1_9 = svld1_vnum(pg, &src_9[0], 1);
    
      320
          BufferVectorType src_0_10 = svld1(pg, &src_10[0]);
    
      320
          BufferVectorType src_1_10 = svld1_vnum(pg, &src_10[0], 1);
    
      320
          BufferVectorType src_0_11 = svld1(pg, &src_11[0]);
    
      320
          BufferVectorType src_1_11 = svld1_vnum(pg, &src_11[0], 1);
    
      320
          BufferVectorType src_0_12 = svld1(pg, &src_12[0]);
    
      320
          BufferVectorType src_1_12 = svld1_vnum(pg, &src_12[0], 1);
    
      320
          BufferVectorType src_0_13 = svld1(pg, &src_13[0]);
    
      320
          BufferVectorType src_1_13 = svld1_vnum(pg, &src_13[0], 1);
    
      320
          BufferVectorType src_0_14 = svld1(pg, &src_14[0]);
    
      320
          BufferVectorType src_1_14 = svld1_vnum(pg, &src_14[0], 1);
    
      320
          BufferVectorType src_0_15 = svld1(pg, &src_15[0]);
    
      320
          BufferVectorType src_1_15 = svld1_vnum(pg, &src_15[0], 1);
    
      320
          BufferVectorType src_0_16 = svld1(pg, &src_16[0]);
    
      320
          BufferVectorType src_1_16 = svld1_vnum(pg, &src_16[0], 1);
    
      320
          BufferVectorType src_0_17 = svld1(pg, &src_17[0]);
    
      320
          BufferVectorType src_1_17 = svld1_vnum(pg, &src_17[0], 1);
    
      320
          BufferVectorType src_0_18 = svld1(pg, &src_18[0]);
    
      320
          BufferVectorType src_1_18 = svld1_vnum(pg, &src_18[0], 1);
    
      320
          BufferVectorType src_0_19 = svld1(pg, &src_19[0]);
    
      320
          BufferVectorType src_1_19 = svld1_vnum(pg, &src_19[0], 1);
    
      320
          BufferVectorType src_0_20 = svld1(pg, &src_20[0]);
    
      320
          BufferVectorType src_1_20 = svld1_vnum(pg, &src_20[0], 1);
    
      6720
          std::reference_wrapper<BufferVectorType> sources_0[21] = {
    
      2240
              src_0_0,  src_0_1,  src_0_2,  src_0_3,  src_0_4,  src_0_5,  src_0_6,
    
      2240
              src_0_7,  src_0_8,  src_0_9,  src_0_10, src_0_11, src_0_12, src_0_13,
    
      2240
              src_0_14, src_0_15, src_0_16, src_0_17, src_0_18, src_0_19, src_0_20};
    
      320
          filter_.horizontal_vector_path(pg, sources_0, &dst_rows[index]);
    
      6720
          std::reference_wrapper<BufferVectorType> sources_1[21] = {
    
      2240
              src_1_0,  src_1_1,  src_1_2,  src_1_3,  src_1_4,  src_1_5,  src_1_6,
    
      2240
              src_1_7,  src_1_8,  src_1_9,  src_1_10, src_1_11, src_1_12, src_1_13,
    
      2240
              src_1_14, src_1_15, src_1_16, src_1_17, src_1_18, src_1_19, src_1_20};
    
      640
          filter_.horizontal_vector_path(
    
      320
              pg, sources_1, &dst_rows[index + BufferVecTraits::num_lanes()]);
    
      320
        }
    
      1296
        void horizontal_vector_path(svbool_t pg, Rows<const BufferType> src_rows,
    
                                    Rows<DestinationType> dst_rows,
    
                                    BorderOffsets border_offsets,
    
                                    size_t index) const KLEIDICV_STREAMING {
    
      2592
          BufferVectorType src_0 =
    
      1296
              svld1(pg, &src_rows.at(0, border_offsets.c0())[index]);
    
      2592
          BufferVectorType src_1 =
    
      1296
              svld1(pg, &src_rows.at(0, border_offsets.c1())[index]);
    
      2592
          BufferVectorType src_2 =
    
      1296
              svld1(pg, &src_rows.at(0, border_offsets.c2())[index]);
    
      2592
          BufferVectorType src_3 =
    
      1296
              svld1(pg, &src_rows.at(0, border_offsets.c3())[index]);
    
      2592
          BufferVectorType src_4 =
    
      1296
              svld1(pg, &src_rows.at(0, border_offsets.c4())[index]);
    
      2592
          BufferVectorType src_5 =
    
      1296
              svld1(pg, &src_rows.at(0, border_offsets.c5())[index]);
    
      2592
          BufferVectorType src_6 =
    
      1296
              svld1(pg, &src_rows.at(0, border_offsets.c6())[index]);
    
      2592
          BufferVectorType src_7 =
    
      1296
              svld1(pg, &src_rows.at(0, border_offsets.c7())[index]);
    
      2592
          BufferVectorType src_8 =
    
      1296
              svld1(pg, &src_rows.at(0, border_offsets.c8())[index]);
    
      2592
          BufferVectorType src_9 =
    
      1296
              svld1(pg, &src_rows.at(0, border_offsets.c9())[index]);
    
      2592
          BufferVectorType src_10 =
    
      1296
              svld1(pg, &src_rows.at(0, border_offsets.c10())[index]);
    
      2592
          BufferVectorType src_11 =
    
      1296
              svld1(pg, &src_rows.at(0, border_offsets.c11())[index]);
    
      2592
          BufferVectorType src_12 =
    
      1296
              svld1(pg, &src_rows.at(0, border_offsets.c12())[index]);
    
      2592
          BufferVectorType src_13 =
    
      1296
              svld1(pg, &src_rows.at(0, border_offsets.c13())[index]);
    
      2592
          BufferVectorType src_14 =
    
      1296
              svld1(pg, &src_rows.at(0, border_offsets.c14())[index]);
    
      2592
          BufferVectorType src_15 =
    
      1296
              svld1(pg, &src_rows.at(0, border_offsets.c15())[index]);
    
      2592
          BufferVectorType src_16 =
    
      1296
              svld1(pg, &src_rows.at(0, border_offsets.c16())[index]);
    
      2592
          BufferVectorType src_17 =
    
      1296
              svld1(pg, &src_rows.at(0, border_offsets.c17())[index]);
    
      2592
          BufferVectorType src_18 =
    
      1296
              svld1(pg, &src_rows.at(0, border_offsets.c18())[index]);
    
      2592
          BufferVectorType src_19 =
    
      1296
              svld1(pg, &src_rows.at(0, border_offsets.c19())[index]);
    
      2592
          BufferVectorType src_20 =
    
      1296
              svld1(pg, &src_rows.at(0, border_offsets.c20())[index]);
    
      27216
          std::reference_wrapper<BufferVectorType> sources[21] = {
    
      9072
              src_0,  src_1,  src_2,  src_3,  src_4,  src_5,  src_6,
    
      9072
              src_7,  src_8,  src_9,  src_10, src_11, src_12, src_13,
    
      9072
              src_14, src_15, src_16, src_17, src_18, src_19, src_20};
    
      1296
          filter_.horizontal_vector_path(pg, sources, &dst_rows[index]);
    
      1296
        }
    
      45120
        void process_horizontal_border(Rows<const BufferType> src_rows,
    
                                       Rows<DestinationType> dst_rows,
    
                                       BorderOffsets border_offsets,
    
                                       size_t index) const KLEIDICV_STREAMING {
    
      45120
          BufferType src[21];
    
      45120
          src[0] = src_rows.at(0, border_offsets.c0())[index];
    
      45120
          src[1] = src_rows.at(0, border_offsets.c1())[index];
    
      45120
          src[2] = src_rows.at(0, border_offsets.c2())[index];
    
      45120
          src[3] = src_rows.at(0, border_offsets.c3())[index];
    
      45120
          src[4] = src_rows.at(0, border_offsets.c4())[index];
    
      45120
          src[5] = src_rows.at(0, border_offsets.c5())[index];
    
      45120
          src[6] = src_rows.at(0, border_offsets.c6())[index];
    
      45120
          src[7] = src_rows.at(0, border_offsets.c7())[index];
    
      45120
          src[8] = src_rows.at(0, border_offsets.c8())[index];
    
      45120
          src[9] = src_rows.at(0, border_offsets.c9())[index];
    
      45120
          src[10] = src_rows.at(0, border_offsets.c10())[index];
    
      45120
          src[11] = src_rows.at(0, border_offsets.c11())[index];
    
      45120
          src[12] = src_rows.at(0, border_offsets.c12())[index];
    
      45120
          src[13] = src_rows.at(0, border_offsets.c13())[index];
    
      45120
          src[14] = src_rows.at(0, border_offsets.c14())[index];
    
      45120
          src[15] = src_rows.at(0, border_offsets.c15())[index];
    
      45120
          src[16] = src_rows.at(0, border_offsets.c16())[index];
    
      45120
          src[17] = src_rows.at(0, border_offsets.c17())[index];
    
      45120
          src[18] = src_rows.at(0, border_offsets.c18())[index];
    
      45120
          src[19] = src_rows.at(0, border_offsets.c19())[index];
    
      45120
          src[20] = src_rows.at(0, border_offsets.c20())[index];
    
      45120
          filter_.horizontal_scalar_path(src, &dst_rows[index]);
    
      45120
        }
    
        FilterType filter_;
    
      };  // end of class SeparableFilter<FilterType, 21UL>
    
      // Shorthand for 21x21 separable filters driver type.
    
      template <class FilterType>
    
      using SeparableFilter21x21 = SeparableFilter<FilterType, 21UL>;
    
      }  // namespace KLEIDICV_TARGET_NAMESPACE
    
      #endif  // KLEIDICV_SEPARABLE_FILTER_21X21_SC_H

Line	Branch	Exec	Source
1			// SPDX-FileCopyrightText: 2025 Arm Limited and/or its affiliates <open-source-office@arm.com>
2			//
3			// SPDX-License-Identifier: Apache-2.0
4
5			#ifndef KLEIDICV_SEPARABLE_FILTER_21X21_SC_H
6			#define KLEIDICV_SEPARABLE_FILTER_21X21_SC_H
7
8			#include "kleidicv/sve2.h"
9			#include "kleidicv/workspace/border_21x21.h"
10
11			// It is used by SVE2 and SME, the actual namespace will reflect it.
12			namespace KLEIDICV_TARGET_NAMESPACE {
13
14			// Template for drivers of separable NxM filters.
15			template <typename FilterType, const size_t S>
16			class SeparableFilter;
17
18			// Driver for a separable 21x21 filter.
19			template <typename FilterType>
20			class SeparableFilter<FilterType, 21UL> {
21			public:
22			using SourceType = typename FilterType::SourceType;
23			using BufferType = typename FilterType::BufferType;
24			using DestinationType = typename FilterType::DestinationType;
25			using SourceVecTraits =
26			typename ::KLEIDICV_TARGET_NAMESPACE::VecTraits<SourceType>;
27			using SourceVectorType = typename SourceVecTraits::VectorType;
28			using BufferVecTraits =
29			typename ::KLEIDICV_TARGET_NAMESPACE::VecTraits<BufferType>;
30			using BufferVectorType = typename BufferVecTraits::VectorType;
31			using BorderInfoType =
32			typename ::KLEIDICV_TARGET_NAMESPACE::FixedBorderInfo21x21<SourceType>;
33			using BorderType = FixedBorderType;
34			using BorderOffsets = typename BorderInfoType::Offsets;
35
36		64	explicit SeparableFilter(FilterType filter) KLEIDICV_STREAMING
37		64	: filter_{filter} {}
38
39			static constexpr size_t margin = 10UL;
40
41		1296	void process_vertical(size_t width, Rows<const SourceType> src_rows,
42			Rows<BufferType> dst_rows,
43			BorderOffsets border_offsets) const KLEIDICV_STREAMING {
44		1296	LoopUnroll2 loop{width * src_rows.channels(), SourceVecTraits::num_lanes()};
45
46		3384	loop.unroll_once([&](size_t index) KLEIDICV_STREAMING {
47		2088	svbool_t pg_all = SourceVecTraits::svptrue();
48		2088	vertical_vector_path(pg_all, src_rows, dst_rows, border_offsets, index);
49		2088	});
50
51		2592	loop.remaining([&](size_t index, size_t length) KLEIDICV_STREAMING {
52		1296	svbool_t pg = SourceVecTraits::svwhilelt(index, length);
53		1296	vertical_vector_path(pg, src_rows, dst_rows, border_offsets, index);
54		1296	});
55		1296	}
56
57		1296	void process_horizontal(size_t width, Rows<const BufferType> src_rows,
58			Rows<DestinationType> dst_rows,
59			BorderOffsets border_offsets) const
60			KLEIDICV_STREAMING {
61		1296	svbool_t pg_all = BufferVecTraits::svptrue();
62		1296	LoopUnroll2 loop{width * src_rows.channels(), BufferVecTraits::num_lanes()};
63
64		1616	loop.unroll_twice([&](size_t index) KLEIDICV_STREAMING {
65		640	horizontal_vector_path_2x(pg_all, src_rows, dst_rows, border_offsets,
66		320	index);
67		320	});
68
69		1616	loop.unroll_once([&](size_t index) KLEIDICV_STREAMING {
70		320	horizontal_vector_path(pg_all, src_rows, dst_rows, border_offsets, index);
71		320	});
72
73		2272	loop.remaining([&](size_t index, size_t length) KLEIDICV_STREAMING {
74		976	svbool_t pg = BufferVecTraits::svwhilelt(index, length);
75		976	horizontal_vector_path(pg, src_rows, dst_rows, border_offsets, index);
76		976	});
77		1296	}
78
79			// Processing of horizontal borders is always scalar because border offsets
80			// change for each and every element in the border.
81		25920	void process_horizontal_borders(
82			Rows<const BufferType> src_rows, Rows<DestinationType> dst_rows,
83			BorderOffsets border_offsets) const KLEIDICV_STREAMING {
84	2/2 ✓ Branch 0 taken 25920 times. ✓ Branch 1 taken 45120 times.	71040	for (size_t index = 0; index < src_rows.channels(); ++index) {
85		45120	disable_loop_vectorization();
86		45120	process_horizontal_border(src_rows, dst_rows, border_offsets, index);
87		45120	}
88		25920	}
89
90			private:
91		3384	void vertical_vector_path(svbool_t pg, Rows<const SourceType> src_rows,
92			Rows<BufferType> dst_rows,
93			BorderOffsets border_offsets,
94			size_t index) const KLEIDICV_STREAMING {
95		6768	SourceVectorType src_0 =
96		3384	svld1(pg, &src_rows.at(border_offsets.c0())[index]);
97		6768	SourceVectorType src_1 =
98		3384	svld1(pg, &src_rows.at(border_offsets.c1())[index]);
99		6768	SourceVectorType src_2 =
100		3384	svld1(pg, &src_rows.at(border_offsets.c2())[index]);
101		6768	SourceVectorType src_3 =
102		3384	svld1(pg, &src_rows.at(border_offsets.c3())[index]);
103		6768	SourceVectorType src_4 =
104		3384	svld1(pg, &src_rows.at(border_offsets.c4())[index]);
105		6768	SourceVectorType src_5 =
106		3384	svld1(pg, &src_rows.at(border_offsets.c5())[index]);
107		6768	SourceVectorType src_6 =
108		3384	svld1(pg, &src_rows.at(border_offsets.c6())[index]);
109		6768	SourceVectorType src_7 =
110		3384	svld1(pg, &src_rows.at(border_offsets.c7())[index]);
111		6768	SourceVectorType src_8 =
112		3384	svld1(pg, &src_rows.at(border_offsets.c8())[index]);
113		6768	SourceVectorType src_9 =
114		3384	svld1(pg, &src_rows.at(border_offsets.c9())[index]);
115		6768	SourceVectorType src_10 =
116		3384	svld1(pg, &src_rows.at(border_offsets.c10())[index]);
117		6768	SourceVectorType src_11 =
118		3384	svld1(pg, &src_rows.at(border_offsets.c11())[index]);
119		6768	SourceVectorType src_12 =
120		3384	svld1(pg, &src_rows.at(border_offsets.c12())[index]);
121		6768	SourceVectorType src_13 =
122		3384	svld1(pg, &src_rows.at(border_offsets.c13())[index]);
123		6768	SourceVectorType src_14 =
124		3384	svld1(pg, &src_rows.at(border_offsets.c14())[index]);
125		6768	SourceVectorType src_15 =
126		3384	svld1(pg, &src_rows.at(border_offsets.c15())[index]);
127		6768	SourceVectorType src_16 =
128		3384	svld1(pg, &src_rows.at(border_offsets.c16())[index]);
129		6768	SourceVectorType src_17 =
130		3384	svld1(pg, &src_rows.at(border_offsets.c17())[index]);
131		6768	SourceVectorType src_18 =
132		3384	svld1(pg, &src_rows.at(border_offsets.c18())[index]);
133		6768	SourceVectorType src_19 =
134		3384	svld1(pg, &src_rows.at(border_offsets.c19())[index]);
135		6768	SourceVectorType src_20 =
136		3384	svld1(pg, &src_rows.at(border_offsets.c20())[index]);
137		71064	std::reference_wrapper<SourceVectorType> sources[21] = {
138		23688	src_0, src_1, src_2, src_3, src_4, src_5, src_6,
139		23688	src_7, src_8, src_9, src_10, src_11, src_12, src_13,
140		23688	src_14, src_15, src_16, src_17, src_18, src_19, src_20};
141		3384	filter_.vertical_vector_path(pg, sources, &dst_rows[index]);
142		3384	}
143
144		320	void horizontal_vector_path_2x(svbool_t pg, Rows<const BufferType> src_rows,
145			Rows<DestinationType> dst_rows,
146			BorderOffsets border_offsets,
147			size_t index) const KLEIDICV_STREAMING {
148		320	auto src_0 = &src_rows.at(0, border_offsets.c0())[index];
149		320	auto src_1 = &src_rows.at(0, border_offsets.c1())[index];
150		320	auto src_2 = &src_rows.at(0, border_offsets.c2())[index];
151		320	auto src_3 = &src_rows.at(0, border_offsets.c3())[index];
152		320	auto src_4 = &src_rows.at(0, border_offsets.c4())[index];
153		320	auto src_5 = &src_rows.at(0, border_offsets.c5())[index];
154		320	auto src_6 = &src_rows.at(0, border_offsets.c6())[index];
155		320	auto src_7 = &src_rows.at(0, border_offsets.c7())[index];
156		320	auto src_8 = &src_rows.at(0, border_offsets.c8())[index];
157		320	auto src_9 = &src_rows.at(0, border_offsets.c9())[index];
158		320	auto src_10 = &src_rows.at(0, border_offsets.c10())[index];
159		320	auto src_11 = &src_rows.at(0, border_offsets.c11())[index];
160		320	auto src_12 = &src_rows.at(0, border_offsets.c12())[index];
161		320	auto src_13 = &src_rows.at(0, border_offsets.c13())[index];
162		320	auto src_14 = &src_rows.at(0, border_offsets.c14())[index];
163		320	auto src_15 = &src_rows.at(0, border_offsets.c15())[index];
164		320	auto src_16 = &src_rows.at(0, border_offsets.c16())[index];
165		320	auto src_17 = &src_rows.at(0, border_offsets.c17())[index];
166		320	auto src_18 = &src_rows.at(0, border_offsets.c18())[index];
167		320	auto src_19 = &src_rows.at(0, border_offsets.c19())[index];
168		320	auto src_20 = &src_rows.at(0, border_offsets.c20())[index];
169
170		320	BufferVectorType src_0_0 = svld1(pg, &src_0[0]);
171		320	BufferVectorType src_1_0 = svld1_vnum(pg, &src_0[0], 1);
172		320	BufferVectorType src_0_1 = svld1(pg, &src_1[0]);
173		320	BufferVectorType src_1_1 = svld1_vnum(pg, &src_1[0], 1);
174		320	BufferVectorType src_0_2 = svld1(pg, &src_2[0]);
175		320	BufferVectorType src_1_2 = svld1_vnum(pg, &src_2[0], 1);
176		320	BufferVectorType src_0_3 = svld1(pg, &src_3[0]);
177		320	BufferVectorType src_1_3 = svld1_vnum(pg, &src_3[0], 1);
178		320	BufferVectorType src_0_4 = svld1(pg, &src_4[0]);
179		320	BufferVectorType src_1_4 = svld1_vnum(pg, &src_4[0], 1);
180		320	BufferVectorType src_0_5 = svld1(pg, &src_5[0]);
181		320	BufferVectorType src_1_5 = svld1_vnum(pg, &src_5[0], 1);
182		320	BufferVectorType src_0_6 = svld1(pg, &src_6[0]);
183		320	BufferVectorType src_1_6 = svld1_vnum(pg, &src_6[0], 1);
184		320	BufferVectorType src_0_7 = svld1(pg, &src_7[0]);
185		320	BufferVectorType src_1_7 = svld1_vnum(pg, &src_7[0], 1);
186		320	BufferVectorType src_0_8 = svld1(pg, &src_8[0]);
187		320	BufferVectorType src_1_8 = svld1_vnum(pg, &src_8[0], 1);
188		320	BufferVectorType src_0_9 = svld1(pg, &src_9[0]);
189		320	BufferVectorType src_1_9 = svld1_vnum(pg, &src_9[0], 1);
190		320	BufferVectorType src_0_10 = svld1(pg, &src_10[0]);
191		320	BufferVectorType src_1_10 = svld1_vnum(pg, &src_10[0], 1);
192		320	BufferVectorType src_0_11 = svld1(pg, &src_11[0]);
193		320	BufferVectorType src_1_11 = svld1_vnum(pg, &src_11[0], 1);
194		320	BufferVectorType src_0_12 = svld1(pg, &src_12[0]);
195		320	BufferVectorType src_1_12 = svld1_vnum(pg, &src_12[0], 1);
196		320	BufferVectorType src_0_13 = svld1(pg, &src_13[0]);
197		320	BufferVectorType src_1_13 = svld1_vnum(pg, &src_13[0], 1);
198		320	BufferVectorType src_0_14 = svld1(pg, &src_14[0]);
199		320	BufferVectorType src_1_14 = svld1_vnum(pg, &src_14[0], 1);
200		320	BufferVectorType src_0_15 = svld1(pg, &src_15[0]);
201		320	BufferVectorType src_1_15 = svld1_vnum(pg, &src_15[0], 1);
202		320	BufferVectorType src_0_16 = svld1(pg, &src_16[0]);
203		320	BufferVectorType src_1_16 = svld1_vnum(pg, &src_16[0], 1);
204		320	BufferVectorType src_0_17 = svld1(pg, &src_17[0]);
205		320	BufferVectorType src_1_17 = svld1_vnum(pg, &src_17[0], 1);
206		320	BufferVectorType src_0_18 = svld1(pg, &src_18[0]);
207		320	BufferVectorType src_1_18 = svld1_vnum(pg, &src_18[0], 1);
208		320	BufferVectorType src_0_19 = svld1(pg, &src_19[0]);
209		320	BufferVectorType src_1_19 = svld1_vnum(pg, &src_19[0], 1);
210		320	BufferVectorType src_0_20 = svld1(pg, &src_20[0]);
211		320	BufferVectorType src_1_20 = svld1_vnum(pg, &src_20[0], 1);
212
213		6720	std::reference_wrapper<BufferVectorType> sources_0[21] = {
214		2240	src_0_0, src_0_1, src_0_2, src_0_3, src_0_4, src_0_5, src_0_6,
215		2240	src_0_7, src_0_8, src_0_9, src_0_10, src_0_11, src_0_12, src_0_13,
216		2240	src_0_14, src_0_15, src_0_16, src_0_17, src_0_18, src_0_19, src_0_20};
217		320	filter_.horizontal_vector_path(pg, sources_0, &dst_rows[index]);
218		6720	std::reference_wrapper<BufferVectorType> sources_1[21] = {
219		2240	src_1_0, src_1_1, src_1_2, src_1_3, src_1_4, src_1_5, src_1_6,
220		2240	src_1_7, src_1_8, src_1_9, src_1_10, src_1_11, src_1_12, src_1_13,
221		2240	src_1_14, src_1_15, src_1_16, src_1_17, src_1_18, src_1_19, src_1_20};
222		640	filter_.horizontal_vector_path(
223		320	pg, sources_1, &dst_rows[index + BufferVecTraits::num_lanes()]);
224		320	}
225
226		1296	void horizontal_vector_path(svbool_t pg, Rows<const BufferType> src_rows,
227			Rows<DestinationType> dst_rows,
228			BorderOffsets border_offsets,
229			size_t index) const KLEIDICV_STREAMING {
230		2592	BufferVectorType src_0 =
231		1296	svld1(pg, &src_rows.at(0, border_offsets.c0())[index]);
232		2592	BufferVectorType src_1 =
233		1296	svld1(pg, &src_rows.at(0, border_offsets.c1())[index]);
234		2592	BufferVectorType src_2 =
235		1296	svld1(pg, &src_rows.at(0, border_offsets.c2())[index]);
236		2592	BufferVectorType src_3 =
237		1296	svld1(pg, &src_rows.at(0, border_offsets.c3())[index]);
238		2592	BufferVectorType src_4 =
239		1296	svld1(pg, &src_rows.at(0, border_offsets.c4())[index]);
240		2592	BufferVectorType src_5 =
241		1296	svld1(pg, &src_rows.at(0, border_offsets.c5())[index]);
242		2592	BufferVectorType src_6 =
243		1296	svld1(pg, &src_rows.at(0, border_offsets.c6())[index]);
244		2592	BufferVectorType src_7 =
245		1296	svld1(pg, &src_rows.at(0, border_offsets.c7())[index]);
246		2592	BufferVectorType src_8 =
247		1296	svld1(pg, &src_rows.at(0, border_offsets.c8())[index]);
248		2592	BufferVectorType src_9 =
249		1296	svld1(pg, &src_rows.at(0, border_offsets.c9())[index]);
250		2592	BufferVectorType src_10 =
251		1296	svld1(pg, &src_rows.at(0, border_offsets.c10())[index]);
252		2592	BufferVectorType src_11 =
253		1296	svld1(pg, &src_rows.at(0, border_offsets.c11())[index]);
254		2592	BufferVectorType src_12 =
255		1296	svld1(pg, &src_rows.at(0, border_offsets.c12())[index]);
256		2592	BufferVectorType src_13 =
257		1296	svld1(pg, &src_rows.at(0, border_offsets.c13())[index]);
258		2592	BufferVectorType src_14 =
259		1296	svld1(pg, &src_rows.at(0, border_offsets.c14())[index]);
260		2592	BufferVectorType src_15 =
261		1296	svld1(pg, &src_rows.at(0, border_offsets.c15())[index]);
262		2592	BufferVectorType src_16 =
263		1296	svld1(pg, &src_rows.at(0, border_offsets.c16())[index]);
264		2592	BufferVectorType src_17 =
265		1296	svld1(pg, &src_rows.at(0, border_offsets.c17())[index]);
266		2592	BufferVectorType src_18 =
267		1296	svld1(pg, &src_rows.at(0, border_offsets.c18())[index]);
268		2592	BufferVectorType src_19 =
269		1296	svld1(pg, &src_rows.at(0, border_offsets.c19())[index]);
270		2592	BufferVectorType src_20 =
271		1296	svld1(pg, &src_rows.at(0, border_offsets.c20())[index]);
272
273		27216	std::reference_wrapper<BufferVectorType> sources[21] = {
274		9072	src_0, src_1, src_2, src_3, src_4, src_5, src_6,
275		9072	src_7, src_8, src_9, src_10, src_11, src_12, src_13,
276		9072	src_14, src_15, src_16, src_17, src_18, src_19, src_20};
277		1296	filter_.horizontal_vector_path(pg, sources, &dst_rows[index]);
278		1296	}
279
280		45120	void process_horizontal_border(Rows<const BufferType> src_rows,
281			Rows<DestinationType> dst_rows,
282			BorderOffsets border_offsets,
283			size_t index) const KLEIDICV_STREAMING {
284		45120	BufferType src[21];
285		45120	src[0] = src_rows.at(0, border_offsets.c0())[index];
286		45120	src[1] = src_rows.at(0, border_offsets.c1())[index];
287		45120	src[2] = src_rows.at(0, border_offsets.c2())[index];
288		45120	src[3] = src_rows.at(0, border_offsets.c3())[index];
289		45120	src[4] = src_rows.at(0, border_offsets.c4())[index];
290		45120	src[5] = src_rows.at(0, border_offsets.c5())[index];
291		45120	src[6] = src_rows.at(0, border_offsets.c6())[index];
292		45120	src[7] = src_rows.at(0, border_offsets.c7())[index];
293		45120	src[8] = src_rows.at(0, border_offsets.c8())[index];
294		45120	src[9] = src_rows.at(0, border_offsets.c9())[index];
295		45120	src[10] = src_rows.at(0, border_offsets.c10())[index];
296		45120	src[11] = src_rows.at(0, border_offsets.c11())[index];
297		45120	src[12] = src_rows.at(0, border_offsets.c12())[index];
298		45120	src[13] = src_rows.at(0, border_offsets.c13())[index];
299		45120	src[14] = src_rows.at(0, border_offsets.c14())[index];
300		45120	src[15] = src_rows.at(0, border_offsets.c15())[index];
301		45120	src[16] = src_rows.at(0, border_offsets.c16())[index];
302		45120	src[17] = src_rows.at(0, border_offsets.c17())[index];
303		45120	src[18] = src_rows.at(0, border_offsets.c18())[index];
304		45120	src[19] = src_rows.at(0, border_offsets.c19())[index];
305		45120	src[20] = src_rows.at(0, border_offsets.c20())[index];
306		45120	filter_.horizontal_scalar_path(src, &dst_rows[index]);
307		45120	}
308
309			FilterType filter_;
310			}; // end of class SeparableFilter<FilterType, 21UL>
311
312			// Shorthand for 21x21 separable filters driver type.
313			template <class FilterType>
314			using SeparableFilter21x21 = SeparableFilter<FilterType, 21UL>;
315
316			} // namespace KLEIDICV_TARGET_NAMESPACE
317
318			#endif // KLEIDICV_SEPARABLE_FILTER_21X21_SC_H
319