1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// This patch implements the support routines for the SME ABI,
// described here:
// https://github.com/ARM-software/abi-aa/blob/main/aapcs64/aapcs64.rst#sme-support-routines
#include "../assembly.h"
#if !defined(__APPLE__)
#define TPIDR2_SYMBOL SYMBOL_NAME(__aarch64_has_sme_and_tpidr2_el0)
#define TPIDR2_SYMBOL_OFFSET :lo12:SYMBOL_NAME(__aarch64_has_sme_and_tpidr2_el0)
#else
// MachO requires @page/@pageoff directives because the global is defined
// in a different file. Otherwise this file may fail to build.
#define TPIDR2_SYMBOL SYMBOL_NAME(__aarch64_has_sme_and_tpidr2_el0)@page
#define TPIDR2_SYMBOL_OFFSET SYMBOL_NAME(__aarch64_has_sme_and_tpidr2_el0)@pageoff
#endif
.arch armv9-a+sme
// Utility function which calls a system's abort() routine. Because the function
// is streaming-compatible it should disable streaming-SVE mode before calling
// abort(). Note that there is no need to preserve any state before the call,
// because the function does not return.
DEFINE_COMPILERRT_PRIVATE_FUNCTION(do_abort)
.cfi_startproc
.variant_pcs SYMBOL_NAME(do_abort)
BTI_C
stp x29, x30, [sp, #-32]!
cntd x0
// Store VG to a stack location that we describe with .cfi_offset
str x0, [sp, #16]
.cfi_def_cfa_offset 32
.cfi_offset w30, -24
.cfi_offset w29, -32
.cfi_offset 46, -16
bl __arm_sme_state
tbz x0, #0, 2f
1:
smstop sm
2:
// We can't make this into a tail-call because the unwinder would
// need to restore the value of VG.
bl SYMBOL_NAME(abort)
.cfi_endproc
END_COMPILERRT_FUNCTION(do_abort)
// __arm_sme_state fills the result registers based on a local
// that is set as part of the compiler-rt startup code.
// __aarch64_has_sme_and_tpidr2_el0
DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(__arm_sme_state)
.variant_pcs __arm_sme_state
BTI_C
mov x0, xzr
mov x1, xzr
adrp x16, TPIDR2_SYMBOL
ldrb w16, [x16, TPIDR2_SYMBOL_OFFSET]
cbz w16, 1f
0:
orr x0, x0, #0xC000000000000000
mrs x16, SVCR
bfxil x0, x16, #0, #2
mrs x1, TPIDR2_EL0
1:
ret
END_COMPILERRT_OUTLINE_FUNCTION(__arm_sme_state)
DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(__arm_tpidr2_restore)
.variant_pcs __arm_tpidr2_restore
BTI_C
// If TPIDR2_EL0 is nonnull, the subroutine aborts in some platform-specific
// manner.
mrs x14, TPIDR2_EL0
cbnz x14, 2f
// If any of the reserved bytes in the first 16 bytes of BLK are nonzero,
// the subroutine [..] aborts in some platform-defined manner.
ldrh w14, [x0, #10]
cbnz w14, 2f
ldr w14, [x0, #12]
cbnz w14, 2f
// If BLK.za_save_buffer is NULL, the subroutine does nothing.
ldr x16, [x0]
cbz x16, 1f
// If BLK.num_za_save_slices is zero, the subroutine does nothing.
ldrh w14, [x0, #8]
cbz x14, 1f
mov x15, xzr
0:
ldr za[w15,0], [x16]
addsvl x16, x16, #1
add x15, x15, #1
cmp x14, x15
b.ne 0b
1:
ret
2:
b SYMBOL_NAME(do_abort)
END_COMPILERRT_OUTLINE_FUNCTION(__arm_tpidr2_restore)
DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(__arm_tpidr2_save)
.variant_pcs __arm_tpidr2_restore
BTI_C
// If the current thread does not have access to TPIDR2_EL0, the subroutine
// does nothing.
adrp x14, TPIDR2_SYMBOL
ldrb w14, [x14, TPIDR2_SYMBOL_OFFSET]
cbz w14, 1f
// If TPIDR2_EL0 is null, the subroutine does nothing.
mrs x16, TPIDR2_EL0
cbz x16, 1f
// If any of the reserved bytes in the first 16 bytes of the TPIDR2 block are
// nonzero, the subroutine [..] aborts in some platform-defined manner.
ldrh w14, [x16, #10]
cbnz w14, 2f
ldr w14, [x16, #12]
cbnz w14, 2f
// If num_za_save_slices is zero, the subroutine does nothing.
ldrh w14, [x16, #8]
cbz x14, 1f
// If za_save_buffer is NULL, the subroutine does nothing.
ldr x16, [x16]
cbz x16, 1f
mov x15, xzr
0:
str za[w15,0], [x16]
addsvl x16, x16, #1
add x15, x15, #1
cmp x14, x15
b.ne 0b
1:
ret
2:
b SYMBOL_NAME(do_abort)
END_COMPILERRT_OUTLINE_FUNCTION(__arm_tpidr2_save)
DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(__arm_za_disable)
.variant_pcs __arm_tpidr2_restore
BTI_C
// If the current thread does not have access to SME, the subroutine does
// nothing.
adrp x14, TPIDR2_SYMBOL
ldrb w14, [x14, TPIDR2_SYMBOL_OFFSET]
cbz w14, 0f
// Otherwise, the subroutine behaves as if it did the following:
// * Call __arm_tpidr2_save.
stp x29, x30, [sp, #-16]!
.cfi_def_cfa_offset 16
mov x29, sp
.cfi_def_cfa w29, 16
.cfi_offset w30, -8
.cfi_offset w29, -16
bl __arm_tpidr2_save
// * Set TPIDR2_EL0 to null.
msr TPIDR2_EL0, xzr
// * Set PSTATE.ZA to 0.
smstop za
.cfi_def_cfa wsp, 16
ldp x29, x30, [sp], #16
.cfi_def_cfa_offset 0
.cfi_restore w30
.cfi_restore w29
0:
ret
END_COMPILERRT_OUTLINE_FUNCTION(__arm_za_disable)
NO_EXEC_STACK_DIRECTIVE
// GNU property note for BTI and PAC
GNU_PROPERTY_BTI_PAC
|
