| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | // SPDX-FileCopyrightText: 2023 - 2026 Arm Limited and/or its affiliates <open-source-office@arm.com> | ||
| 2 | // | ||
| 3 | // SPDX-License-Identifier: Apache-2.0 | ||
| 4 | |||
| 5 | #ifndef KLEIDICV_DISPATCH_H | ||
| 6 | #define KLEIDICV_DISPATCH_H | ||
| 7 | |||
| 8 | #include "kleidicv/config.h" | ||
| 9 | |||
| 10 | #if KLEIDICV_ENABLE_SME2 || KLEIDICV_ENABLE_SME || KLEIDICV_ENABLE_SVE2 | ||
| 11 | |||
| 12 | #include <cstdint> | ||
| 13 | #include <type_traits> | ||
| 14 | |||
| 15 | namespace kleidicv { | ||
| 16 | #if KLEIDICV_ENABLE_SVE2 | ||
| 17 | uint64_t svcntb_sve(); | ||
| 18 | #endif | ||
| 19 | #if KLEIDICV_ENABLE_SME2 || KLEIDICV_ENABLE_SME | ||
| 20 | uint64_t svcntb_sme(); | ||
| 21 | #endif | ||
| 22 | } // namespace kleidicv | ||
| 23 | |||
| 24 | #ifdef __APPLE__ | ||
| 25 | |||
| 26 | #include <sys/sysctl.h> | ||
| 27 | |||
| 28 | static inline bool query_sysctl(const char* attribute_name) { | ||
| 29 | uint64_t attribute_value = 0; | ||
| 30 | size_t max_attribute_size = sizeof(attribute_value); | ||
| 31 | if (sysctlbyname(attribute_name, &attribute_value, &max_attribute_size, NULL, | ||
| 32 | 0)) { | ||
| 33 | return false; | ||
| 34 | } | ||
| 35 | return attribute_value; | ||
| 36 | } | ||
| 37 | |||
| 38 | static inline bool is_sve2_supported() { | ||
| 39 | return query_sysctl("hw.optional.arm.FEAT_SVE2"); | ||
| 40 | } | ||
| 41 | |||
| 42 | static inline bool is_sme_supported() { | ||
| 43 | return query_sysctl("hw.optional.arm.FEAT_SME"); | ||
| 44 | } | ||
| 45 | |||
| 46 | static inline bool is_sme2_supported() { | ||
| 47 | return query_sysctl("hw.optional.arm.FEAT_SME2"); | ||
| 48 | } | ||
| 49 | |||
| 50 | #else // __APPLE__ | ||
| 51 | |||
| 52 | #include <sys/auxv.h> | ||
| 53 | |||
| 54 | 302 | static inline bool is_sve2_supported() { | |
| 55 | 302 | return getauxval(AT_HWCAP2) & (1UL << 1); | |
| 56 | } | ||
| 57 | |||
| 58 | 380 | static inline bool is_sme_supported() { | |
| 59 | 380 | return getauxval(AT_HWCAP2) & (1UL << 23); | |
| 60 | } | ||
| 61 | |||
| 62 | #if KLEIDICV_ENABLE_SVE2 | ||
| 63 | 325 | static inline bool is_sme2_supported() { | |
| 64 | 325 | return getauxval(AT_HWCAP2) & (1UL << 37); | |
| 65 | } | ||
| 66 | |||
| 67 | #endif // __APPLE__ | ||
| 68 | |||
| 69 | #if KLEIDICV_ENABLE_SVE2 | ||
| 70 | #define KLEIDICV_SVE2_RESOLVE(sve2_impl) \ | ||
| 71 | if (!std::is_null_pointer_v<decltype(sve2_impl)> && is_sve2_supported()) { \ | ||
| 72 | return sve2_impl; \ | ||
| 73 | } | ||
| 74 | #define KLEIDICV_SVE2_RESOLVE_VECLEN(sve2_impl, sve_len) \ | ||
| 75 | if (!std::is_null_pointer_v<decltype(sve2_impl)> && is_sve2_supported() && \ | ||
| 76 | kleidicv::svcntb_sve() == (sve_len)) { \ | ||
| 77 | return sve2_impl; \ | ||
| 78 | } | ||
| 79 | #else | ||
| 80 | #define KLEIDICV_SVE2_RESOLVE(x) | ||
| 81 | #define KLEIDICV_SVE2_RESOLVE_VECLEN(x, minlen, maxlen) | ||
| 82 | #endif // KLEIDICV_ENABLE_SVE2 | ||
| 83 | |||
| 84 | #if KLEIDICV_ENABLE_SME | ||
| 85 | #define KLEIDICV_SME_RESOLVE(sme_impl) \ | ||
| 86 | if (!std::is_null_pointer_v<decltype(sme_impl)> && is_sme_supported()) { \ | ||
| 87 | return sme_impl; \ | ||
| 88 | } | ||
| 89 | #define KLEIDICV_SME_RESOLVE_VECLEN(sme_impl, sme_len) \ | ||
| 90 | if (!std::is_null_pointer_v<decltype(sme_impl)> && is_sme_supported() && \ | ||
| 91 | kleidicv::svcntb_sme() == (sme_len)) { \ | ||
| 92 | return sme_impl; \ | ||
| 93 | } | ||
| 94 | #else | ||
| 95 | #define KLEIDICV_SME_RESOLVE(x) | ||
| 96 | #define KLEIDICV_SME_RESOLVE_VECLEN(x, sme_len) | ||
| 97 | #endif // KLEIDICV_ENABLE_SME | ||
| 98 | |||
| 99 | #if KLEIDICV_ENABLE_SME2 | ||
| 100 | #define KLEIDICV_SME2_RESOLVE(sme2_impl) \ | ||
| 101 | if (!std::is_null_pointer_v<decltype(sme2_impl)> && is_sme2_supported()) { \ | ||
| 102 | return sme2_impl; \ | ||
| 103 | } | ||
| 104 | #define KLEIDICV_SME2_RESOLVE_VECLEN(sme2_impl, sme_len) \ | ||
| 105 | if (!std::is_null_pointer_v<decltype(sme2_impl)> && is_sme2_supported() && \ | ||
| 106 | kleidicv::svcntb_sme() == (sme_len)) { \ | ||
| 107 | return sme2_impl; \ | ||
| 108 | } | ||
| 109 | #else | ||
| 110 | #define KLEIDICV_SME2_RESOLVE(x) | ||
| 111 | #define KLEIDICV_SME2_RESOLVE_VECLEN(x, sme_len) | ||
| 112 | #endif // KLEIDICV_ENABLE_SME2 | ||
| 113 | |||
| 114 | #define KLEIDICV_MULTIVERSION_C_API(api_name, neon_impl, sve2_impl, sme_impl, \ | ||
| 115 | sme2_impl) \ | ||
| 116 | static decltype(neon_impl) api_name##_resolver() { \ | ||
| 117 | KLEIDICV_SME2_RESOLVE(sme2_impl); \ | ||
| 118 | KLEIDICV_SME_RESOLVE(sme_impl); \ | ||
| 119 | KLEIDICV_SVE2_RESOLVE(sve2_impl); \ | ||
| 120 | return neon_impl; \ | ||
| 121 | } \ | ||
| 122 | extern "C" { \ | ||
| 123 | decltype(neon_impl) api_name = api_name##_resolver(); \ | ||
| 124 | } | ||
| 125 | |||
| 126 | #define KLEIDICV_MULTIVERSION_C_API_VECLEN( \ | ||
| 127 | api_name, neon_impl, sve2_impl, sme_impl, sme2_impl, sve_len, sme_len) \ | ||
| 128 | static decltype(neon_impl) api_name##_resolver() { \ | ||
| 129 | KLEIDICV_SME2_RESOLVE_VECLEN(sme2_impl, sme_len); \ | ||
| 130 | KLEIDICV_SME_RESOLVE_VECLEN(sme_impl, sme_len); \ | ||
| 131 | KLEIDICV_SVE2_RESOLVE_VECLEN(sve2_impl, sve_len); \ | ||
| 132 | return neon_impl; \ | ||
| 133 | } \ | ||
| 134 | extern "C" { \ | ||
| 135 | decltype(neon_impl) api_name = api_name##_resolver(); \ | ||
| 136 | } | ||
| 137 | |||
| 138 | #endif // __APPLE__ | ||
| 139 | |||
| 140 | #else // KLEIDICV_HAVE_SVE2 || KLEIDICV_HAVE_SME || KLEIDICV_HAVE_SME2 | ||
| 141 | |||
| 142 | #define KLEIDICV_MULTIVERSION_C_API(api_name, neon_impl, sve2_impl, sme_impl, \ | ||
| 143 | sme2_impl) \ | ||
| 144 | \ | ||
| 145 | extern "C" { \ | ||
| 146 | decltype(neon_impl) api_name = neon_impl; \ | ||
| 147 | } | ||
| 148 | |||
| 149 | #define KLEIDICV_MULTIVERSION_C_API_VECLEN( \ | ||
| 150 | api_name, neon_impl, sve2_impl, sme_impl, sme2_impl, minlen, maxlen) \ | ||
| 151 | KLEIDICV_MULTIVERSION_C_API(api_name, neon_impl, sve2_impl, sme_impl, \ | ||
| 152 | sme2_impl) | ||
| 153 | |||
| 154 | #endif // KLEIDICV_ENABLE_SME2 || KLEIDICV_ENABLE_SME || KLEIDICV_ENABLE_SVE2 | ||
| 155 | |||
| 156 | #if KLEIDICV_ALWAYS_ENABLE_SME2 | ||
| 157 | #define KLEIDICV_SME2_IMPL_IF(func) func | ||
| 158 | #else | ||
| 159 | #define KLEIDICV_SME2_IMPL_IF(func) nullptr | ||
| 160 | #endif // KLEIDICV_ALWAYS_ENABLE_SME2 | ||
| 161 | |||
| 162 | #if KLEIDICV_ALWAYS_ENABLE_SME | ||
| 163 | #define KLEIDICV_SME_IMPL_IF(func) func | ||
| 164 | #else | ||
| 165 | #define KLEIDICV_SME_IMPL_IF(func) nullptr | ||
| 166 | #endif // KLEIDICV_ALWAYS_ENABLE_SME | ||
| 167 | |||
| 168 | #if KLEIDICV_ALWAYS_ENABLE_SVE2 | ||
| 169 | #define KLEIDICV_SVE2_IMPL_IF(func) func | ||
| 170 | #else | ||
| 171 | #define KLEIDICV_SVE2_IMPL_IF(func) nullptr | ||
| 172 | #endif // KLEIDICV_ALWAYS_ENABLE_SVE2 | ||
| 173 | |||
| 174 | #endif // KLEIDICV_DISPATCH_H | ||
| 175 |