1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
|
#ifdef ARM_FEATURES
#include "zbuild.h"
#include "arm_features.h"
#if defined(HAVE_SYS_AUXV_H)
# include <sys/auxv.h>
# ifdef ARM_ASM_HWCAP
# include <asm/hwcap.h>
# endif
#elif defined(__FreeBSD__) && defined(ARCH_64BIT)
# include <machine/armreg.h>
# ifndef ID_AA64ISAR0_CRC32_VAL
# define ID_AA64ISAR0_CRC32_VAL ID_AA64ISAR0_CRC32
# endif
#elif defined(__OpenBSD__) && defined(ARCH_64BIT)
# include <machine/armreg.h>
# include <machine/cpu.h>
# include <sys/sysctl.h>
# include <sys/types.h>
#elif defined(__APPLE__)
# if !defined(_DARWIN_C_SOURCE)
# define _DARWIN_C_SOURCE /* enable types aliases (eg u_int) */
# endif
# include <sys/sysctl.h>
#elif defined(_WIN32)
# include <windows.h>
#endif
static int arm_has_crc32(void) {
int has_crc32 = 0;
#if defined(__ARM_FEATURE_CRC32)
/* Compile-time check */
has_crc32 = 1;
#elif defined(__linux__) && defined(HAVE_SYS_AUXV_H)
# ifdef HWCAP_CRC32
has_crc32 = (getauxval(AT_HWCAP) & HWCAP_CRC32) != 0;
# elif defined(HWCAP2_CRC32)
has_crc32 = (getauxval(AT_HWCAP2) & HWCAP2_CRC32) != 0;
# endif
#elif (defined(__FreeBSD__) || defined(__OpenBSD__)) && defined(HAVE_SYS_AUXV_H)
# ifdef HWCAP_CRC32
unsigned long hwcap = 0;
elf_aux_info(AT_HWCAP, &hwcap, sizeof(hwcap));
has_crc32 = (hwcap & HWCAP_CRC32) != 0;
# elif defined(HWCAP2_CRC32)
unsigned long hwcap2 = 0;
elf_aux_info(AT_HWCAP2, &hwcap2, sizeof(hwcap2));
has_crc32 = (hwcap2 & HWCAP2_CRC32) != 0;
# endif
#elif defined(__FreeBSD__) && defined(ARCH_64BIT)
has_crc32 = getenv("QEMU_EMULATING") == NULL
&& ID_AA64ISAR0_CRC32_VAL(READ_SPECIALREG(id_aa64isar0_el1)) >= ID_AA64ISAR0_CRC32_BASE;
#elif defined(__OpenBSD__) && defined(ARCH_64BIT)
int isar0_mib[] = { CTL_MACHDEP, CPU_ID_AA64ISAR0 };
uint64_t isar0 = 0;
size_t len = sizeof(isar0);
if (sysctl(isar0_mib, 2, &isar0, &len, NULL, 0) != -1) {
has_crc32 = ID_AA64ISAR0_CRC32(isar0) >= ID_AA64ISAR0_CRC32_BASE;
}
#elif defined(__APPLE__)
int has_feat = 0;
size_t size = sizeof(has_feat);
has_crc32 = sysctlbyname("hw.optional.armv8_crc32", &has_feat, &size, NULL, 0) == 0
&& has_feat == 1;
#elif defined(_WIN32)
has_crc32 = IsProcessorFeaturePresent(PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE);
#endif
return has_crc32;
}
static int arm_has_pmull(void) {
int has_pmull = 0;
#if defined(__ARM_FEATURE_CRYPTO) || defined(__ARM_FEATURE_AES)
/* Compile-time check */
has_pmull = 1;
#elif defined(__linux__) && defined(HAVE_SYS_AUXV_H)
# ifdef HWCAP_PMULL
has_pmull = (getauxval(AT_HWCAP) & HWCAP_PMULL) != 0;
# elif defined(HWCAP_AES)
/* PMULL is part of crypto extension, check for AES as proxy */
has_pmull = (getauxval(AT_HWCAP) & HWCAP_AES) != 0;
# endif
#elif (defined(__FreeBSD__) || defined(__OpenBSD__)) && defined(HAVE_SYS_AUXV_H)
# ifdef HWCAP_PMULL
unsigned long hwcap = 0;
elf_aux_info(AT_HWCAP, &hwcap, sizeof(hwcap));
has_pmull = (hwcap & HWCAP_PMULL) != 0;
# elif defined(HWCAP_AES)
/* PMULL is part of crypto extension, check for AES as proxy */
unsigned long hwcap = 0;
elf_aux_info(AT_HWCAP, &hwcap, sizeof(hwcap));
has_pmull = (hwcap & HWCAP_AES) != 0;
# endif
#elif defined(__FreeBSD__) && defined(ARCH_64BIT)
/* Check for AES feature as PMULL is part of crypto extension */
has_pmull = getenv("QEMU_EMULATING") == NULL
&& ID_AA64ISAR0_AES_VAL(READ_SPECIALREG(id_aa64isar0_el1)) >= ID_AA64ISAR0_AES_BASE;
#elif defined(__OpenBSD__) && defined(ARCH_64BIT)
int isar0_mib[] = { CTL_MACHDEP, CPU_ID_AA64ISAR0 };
uint64_t isar0 = 0;
size_t len = sizeof(isar0);
if (sysctl(isar0_mib, 2, &isar0, &len, NULL, 0) != -1) {
has_pmull = ID_AA64ISAR0_AES(isar0) >= ID_AA64ISAR0_AES_BASE;
}
#elif defined(__APPLE__)
int has_feat = 0;
size_t size = sizeof(has_feat);
has_pmull = sysctlbyname("hw.optional.arm.FEAT_PMULL", &has_feat, &size, NULL, 0) == 0
&& has_feat == 1;
#elif defined(_WIN32)
/* Windows checks for crypto/AES support */
# ifdef PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE
has_pmull = IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE);
# endif
#endif
return has_pmull;
}
static int arm_has_eor3(void) {
int has_eor3 = 0;
#if defined(__ARM_FEATURE_SHA3)
/* Compile-time check */
has_eor3 = 1;
#elif defined(__linux__) && defined(HAVE_SYS_AUXV_H)
/* EOR3 is part of SHA3 extension, check HWCAP2_SHA3 */
# ifdef HWCAP2_SHA3
has_eor3 = (getauxval(AT_HWCAP2) & HWCAP2_SHA3) != 0;
# elif defined(HWCAP_SHA3)
has_eor3 = (getauxval(AT_HWCAP) & HWCAP_SHA3) != 0;
# endif
#elif (defined(__FreeBSD__) || defined(__OpenBSD__)) && defined(HAVE_SYS_AUXV_H)
# ifdef HWCAP2_SHA3
unsigned long hwcap2 = 0;
elf_aux_info(AT_HWCAP2, &hwcap2, sizeof(hwcap2));
has_eor3 = (hwcap2 & HWCAP2_SHA3) != 0;
# elif defined(HWCAP_SHA3)
unsigned long hwcap = 0;
elf_aux_info(AT_HWCAP, &hwcap, sizeof(hwcap));
has_eor3 = (hwcap & HWCAP_SHA3) != 0;
# endif
#elif defined(__FreeBSD__) && defined(ARCH_64BIT)
/* FreeBSD: check for SHA3 in id_aa64isar0_el1 */
# ifdef ID_AA64ISAR0_SHA3_VAL
has_eor3 = getenv("QEMU_EMULATING") == NULL
&& ID_AA64ISAR0_SHA3_VAL(READ_SPECIALREG(id_aa64isar0_el1)) >= ID_AA64ISAR0_SHA3_BASE;
# endif
#elif defined(__OpenBSD__) && defined(ARCH_64BIT)
# ifdef ID_AA64ISAR0_SHA3
int isar0_mib[] = { CTL_MACHDEP, CPU_ID_AA64ISAR0 };
uint64_t isar0 = 0;
size_t len = sizeof(isar0);
if (sysctl(isar0_mib, 2, &isar0, &len, NULL, 0) != -1) {
has_eor3 = ID_AA64ISAR0_SHA3(isar0) >= ID_AA64ISAR0_SHA3_IMPL;
}
# endif
#elif defined(__APPLE__)
/* All Apple Silicon (M1+) has SHA3/EOR3 support */
int has_feat = 0;
size_t size = sizeof(has_feat);
has_eor3 = sysctlbyname("hw.optional.arm.FEAT_SHA3", &has_feat, &size, NULL, 0) == 0
&& has_feat == 1;
/* Fallback to legacy name for older macOS versions */
if (!has_eor3) {
size = sizeof(has_feat);
has_eor3 = sysctlbyname("hw.optional.armv8_2_sha3", &has_feat, &size, NULL, 0) == 0
&& has_feat == 1;
}
#elif defined(_WIN32)
# ifdef PF_ARM_SHA3_INSTRUCTIONS_AVAILABLE
has_eor3 = IsProcessorFeaturePresent(PF_ARM_SHA3_INSTRUCTIONS_AVAILABLE);
# endif
#endif
return has_eor3;
}
/* AArch64 has neon. */
#ifdef ARCH_32BIT
static inline int arm_has_neon(void) {
int has_neon = 0;
#if defined(__ARM_NEON__) || defined(__ARM_NEON)
/* Compile-time check */
has_neon = 1;
#elif defined(__linux__) && defined(HAVE_SYS_AUXV_H)
# ifdef HWCAP_ARM_NEON
has_neon = (getauxval(AT_HWCAP) & HWCAP_ARM_NEON) != 0;
# elif defined(HWCAP_NEON)
has_neon = (getauxval(AT_HWCAP) & HWCAP_NEON) != 0;
# endif
#elif (defined(__FreeBSD__) || defined(__OpenBSD__)) && defined(HAVE_SYS_AUXV_H)
# ifdef HWCAP_NEON
unsigned long hwcap = 0;
elf_aux_info(AT_HWCAP, &hwcap, sizeof(hwcap));
has_neon = (hwcap & HWCAP_NEON) != 0;
# endif
#elif defined(__APPLE__)
int has_feat = 0;
size_t size = sizeof(has_feat);
has_neon = sysctlbyname("hw.optional.neon", &has_feat, &size, NULL, 0) == 0
&& has_feat == 1;
#elif defined(_M_ARM) && defined(WINAPI_FAMILY_PARTITION)
# if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_PHONE_APP)
has_neon = 1; /* Always supported */
# endif
#endif
return has_neon;
}
#endif
/* AArch64 does not have ARMv6 SIMD. */
#ifdef ARCH_32BIT
static inline int arm_has_simd(void) {
int has_simd = 0;
#if defined(__ARM_FEATURE_SIMD32)
/* Compile-time check for ARMv6 SIMD */
has_simd = 1;
#elif defined(__linux__) && defined(HAVE_SYS_AUXV_H)
const char *platform = (const char *)getauxval(AT_PLATFORM);
has_simd = platform
&& (strncmp(platform, "v6l", 3) == 0
|| strncmp(platform, "v7l", 3) == 0
|| strncmp(platform, "v8l", 3) == 0);
#endif
return has_simd;
}
#endif
#if defined(ARCH_64BIT) && !defined(__APPLE__) && !defined(_WIN32)
/* MIDR_EL1 bit field definitions */
#define MIDR_IMPLEMENTOR(midr) (((midr) & (0xffU << 24)) >> 24)
#define MIDR_PARTNUM(midr) (((midr) & (0xfffU << 4)) >> 4)
/* ARM CPU Implementer IDs */
#define ARM_IMPLEMENTER_ARM 0x41
#define ARM_IMPLEMENTER_QUALCOMM 0x51
#define ARM_IMPLEMENTER_APPLE 0x61
/* ARM CPU Part Numbers */
/* Cortex-X series - Multiple PMULL lanes */
#define ARM_PART_CORTEX_X1 0xd44
#define ARM_PART_CORTEX_X1C 0xd4c
#define ARM_PART_CORTEX_X2 0xd48
#define ARM_PART_CORTEX_X3 0xd4e
#define ARM_PART_CORTEX_X4 0xd82
#define ARM_PART_CORTEX_X925 0xd85
/* Neoverse V/N2 series - Multiple PMULL lanes */
#define ARM_PART_NEOVERSE_N2 0xd49
#define ARM_PART_NEOVERSE_V1 0xd40
#define ARM_PART_NEOVERSE_V2 0xd4f
#define ARM_PART_NEOVERSE_V3 0xd8e
/* Snapdragon X Elite/Plus - Custom core */
#define QUALCOMM_PART_ORYON 0x001
static inline int arm_has_cpuid(void) {
int has_cpuid = 0;
#if defined(__linux__) && defined(HAVE_SYS_AUXV_H)
# ifdef HWCAP_CPUID
has_cpuid = (getauxval(AT_HWCAP) & HWCAP_CPUID) != 0;
# elif defined(HWCAP2_CPUID)
has_cpuid = (getauxval(AT_HWCAP2) & HWCAP2_CPUID) != 0;
# endif
#elif (defined(__FreeBSD__) || defined(__OpenBSD__)) && defined(HAVE_SYS_AUXV_H)
# ifdef HWCAP_CPUID
unsigned long hwcap = 0;
elf_aux_info(AT_HWCAP, &hwcap, sizeof(hwcap));
has_cpuid = (hwcap & HWCAP_CPUID) != 0;
# endif
#endif
return has_cpuid;
}
#endif
/* Determine if CPU has fast PMULL (multiple execution units) */
static inline int arm_cpu_has_fast_pmull(void) {
int has_fast_pmull = 0;
#if defined(__APPLE__)
/* On macOS, all Apple Silicon has fast PMULL */
has_fast_pmull = 1;
#elif defined(ARCH_64BIT) && !defined(_WIN32)
/* We need CPUID feature to read MIDR register */
if (!arm_has_cpuid())
return has_fast_pmull;
uint64_t midr;
__asm__ ("mrs %0, midr_el1" : "=r" (midr));
uint32_t implementer = MIDR_IMPLEMENTOR(midr);
uint32_t part = MIDR_PARTNUM(midr);
if (implementer == ARM_IMPLEMENTER_APPLE) {
/* All Apple Silicon (M1+) have fast PMULL */
has_fast_pmull = 1;
} else if (implementer == ARM_IMPLEMENTER_ARM) {
/* ARM Cortex-X and Neoverse V/N2 series have multi-lane PMULL */
switch (part) {
case ARM_PART_CORTEX_X1:
case ARM_PART_CORTEX_X1C:
case ARM_PART_CORTEX_X2:
case ARM_PART_CORTEX_X3:
case ARM_PART_CORTEX_X4:
case ARM_PART_CORTEX_X925:
case ARM_PART_NEOVERSE_N2:
case ARM_PART_NEOVERSE_V1:
case ARM_PART_NEOVERSE_V2:
case ARM_PART_NEOVERSE_V3:
has_fast_pmull = 1;
}
} else if (implementer == ARM_IMPLEMENTER_QUALCOMM) {
/* Qualcomm Oryon (Snapdragon X Elite/Plus) has fast PMULL */
if (part == QUALCOMM_PART_ORYON)
has_fast_pmull = 1;
}
#endif
return has_fast_pmull;
}
void Z_INTERNAL arm_check_features(struct arm_cpu_features *features) {
#ifdef ARCH_64BIT
features->has_simd = 0; /* never available */
features->has_neon = 1; /* always available */
#else
features->has_simd = arm_has_simd();
features->has_neon = arm_has_neon();
#endif
features->has_crc32 = arm_has_crc32();
features->has_pmull = arm_has_pmull();
features->has_eor3 = arm_has_eor3();
features->has_fast_pmull = features->has_pmull && arm_cpu_has_fast_pmull();
}
#endif
|
