1-- A cell that holds one u8, u16, u32 or u64 and is read and written only
2-- atomically, over the compiler's memory builtins (D227, D500). Each
3-- operation carries its ordering in its name, because an ordering is a
4-- compiler atom a call cannot pass on: a load acquires and a store
5-- releases, an exchange, an add and a compare-exchange acquire and
6-- release, and the relaxed family is for counters and caches whose value
7-- orders nothing else. Sequential consistency stays with the builtins.
8--
9-- The cell is an ordinary struct: module storage starts it `zeroed`, and
10-- `new` makes one for a local or a field. Every access after it is
11-- shared goes through the operations here and the cell's address, so
12-- pass a cell by pointer and never copy one that another context uses.
13--
14-- Cortex-M0 has no atomic read-modify-write and no eight-byte atomic
15-- access (D227), so on it `u64` has no conformance to `scalar` and no
16-- scalar one to `exchangeable`, and the compiler refuses a cell over u64,
17-- and an exchange, add or compare-exchange, at the caller's use (D510).
18-- What it has instead is `sole_add`, a relaxed load and a release store,
19-- correct while one context alone writes the cell, which every target
20-- has; and platform/cortex_m's critical section for state several
21-- contexts write.
22
23--- The scalars a cell can hold, with the accesses every target admits for
24--- them: u8, u16 and u32 everywhere, and u64 where the target has eight-byte
25--- atomics, which Cortex-M0 has not. The entries are the cell's
26--- operations' own, over the scalar's address; use the operations below.
27public scalar: type = concept (item: type)
28 load: (cell: ptr item) -> (value: item)
29 load_relaxed: (cell: ptr item) -> (value: item)
30 store: (cell: ptr mut item, value: item) -> none
31 store_relaxed: (cell: ptr mut item, value: item) -> none
32 sole_add: (cell: ptr mut item, value: item) -> (previous: item)
33end scalar
34
35--- The scalars whose cells also take a read-modify-write: each of u8, u16,
36--- u32 and u64 where the target has atomic read-modify-write, which
37--- Cortex-M0 has not.
38public exchangeable: type = concept (item: type) is scalar
39 exchange: (cell: ptr mut item, value: item) -> (previous: item)
40 add: (cell: ptr mut item, value: item) -> (previous: item)
41 add_relaxed: (cell: ptr mut item, value: item) -> (previous: item)
42 compare_exchange: (cell: ptr mut item, expected: item, desired: item)
43 -> (observed: item)
44end exchangeable
45
46load_u8: (cell: ptr u8) -> (value: u8) =
47 value = compiler.atomic_load(cell, compiler.acquire)
48end load_u8
49load_relaxed_u8: (cell: ptr u8) -> (value: u8) =
50 value = compiler.atomic_load(cell, compiler.relaxed)
51end load_relaxed_u8
52store_u8: (cell: ptr mut u8, value: u8) -> none =
53 compiler.atomic_store(cell, value, compiler.release)
54end store_u8
55store_relaxed_u8: (cell: ptr mut u8, value: u8) -> none =
56 compiler.atomic_store(cell, value, compiler.relaxed)
57end store_relaxed_u8
58sole_add_u8: (cell: ptr mut u8, value: u8) -> (previous: u8) =
59 previous = compiler.atomic_load(cell, compiler.relaxed)
60 compiler.atomic_store(cell, previous +% value, compiler.release)
61end sole_add_u8
62load_u16: (cell: ptr u16) -> (value: u16) =
63 value = compiler.atomic_load(cell, compiler.acquire)
64end load_u16
65load_relaxed_u16: (cell: ptr u16) -> (value: u16) =
66 value = compiler.atomic_load(cell, compiler.relaxed)
67end load_relaxed_u16
68store_u16: (cell: ptr mut u16, value: u16) -> none =
69 compiler.atomic_store(cell, value, compiler.release)
70end store_u16
71store_relaxed_u16: (cell: ptr mut u16, value: u16) -> none =
72 compiler.atomic_store(cell, value, compiler.relaxed)
73end store_relaxed_u16
74sole_add_u16: (cell: ptr mut u16, value: u16) -> (previous: u16) =
75 previous = compiler.atomic_load(cell, compiler.relaxed)
76 compiler.atomic_store(cell, previous +% value, compiler.release)
77end sole_add_u16
78load_u32: (cell: ptr u32) -> (value: u32) =
79 value = compiler.atomic_load(cell, compiler.acquire)
80end load_u32
81load_relaxed_u32: (cell: ptr u32) -> (value: u32) =
82 value = compiler.atomic_load(cell, compiler.relaxed)
83end load_relaxed_u32
84store_u32: (cell: ptr mut u32, value: u32) -> none =
85 compiler.atomic_store(cell, value, compiler.release)
86end store_u32
87store_relaxed_u32: (cell: ptr mut u32, value: u32) -> none =
88 compiler.atomic_store(cell, value, compiler.relaxed)
89end store_relaxed_u32
90sole_add_u32: (cell: ptr mut u32, value: u32) -> (previous: u32) =
91 previous = compiler.atomic_load(cell, compiler.relaxed)
92 compiler.atomic_store(cell, previous +% value, compiler.release)
93end sole_add_u32
94u8 is scalar (load: load_u8, load_relaxed: load_relaxed_u8,
95 store: store_u8, store_relaxed: store_relaxed_u8,
96 sole_add: sole_add_u8)
97u16 is scalar (load: load_u16, load_relaxed: load_relaxed_u16,
98 store: store_u16, store_relaxed: store_relaxed_u16,
99 sole_add: sole_add_u16)
100u32 is scalar (load: load_u32, load_relaxed: load_relaxed_u32,
101 store: store_u32, store_relaxed: store_relaxed_u32,
102 sole_add: sole_add_u32)
103
104-- Cortex-M0 has neither the eight-byte accesses nor any read-modify-write
105-- (D227): what it lacks has no conformance there, and D510 refuses its use.
106fixed if compiler.arch <> cortex_m0 then
107 load_u64: (cell: ptr u64) -> (value: u64) =
108 value = compiler.atomic_load(cell, compiler.acquire)
109 end load_u64
110 load_relaxed_u64: (cell: ptr u64) -> (value: u64) =
111 value = compiler.atomic_load(cell, compiler.relaxed)
112 end load_relaxed_u64
113 store_u64: (cell: ptr mut u64, value: u64) -> none =
114 compiler.atomic_store(cell, value, compiler.release)
115 end store_u64
116 store_relaxed_u64: (cell: ptr mut u64, value: u64) -> none =
117 compiler.atomic_store(cell, value, compiler.relaxed)
118 end store_relaxed_u64
119 sole_add_u64: (cell: ptr mut u64, value: u64) -> (previous: u64) =
120 previous = compiler.atomic_load(cell, compiler.relaxed)
121 compiler.atomic_store(cell, previous +% value, compiler.release)
122 end sole_add_u64
123 u64 is scalar (load: load_u64, load_relaxed: load_relaxed_u64,
124 store: store_u64, store_relaxed: store_relaxed_u64,
125 sole_add: sole_add_u64)
126 exchange_u8: (cell: ptr mut u8, value: u8) -> (previous: u8) =
127 previous = compiler.atomic_exchange(cell, value, compiler.acq_rel)
128 end exchange_u8
129 add_u8: (cell: ptr mut u8, value: u8) -> (previous: u8) =
130 previous = compiler.atomic_add(cell, value, compiler.acq_rel)
131 end add_u8
132 add_relaxed_u8: (cell: ptr mut u8, value: u8) -> (previous: u8) =
133 previous = compiler.atomic_add(cell, value, compiler.relaxed)
134 end add_relaxed_u8
135 compare_exchange_u8: (cell: ptr mut u8, expected: u8, desired: u8)
136 -> (observed: u8) =
137 observed = compiler.atomic_compare_exchange(cell, expected, desired,
138 compiler.acq_rel,
139 compiler.acquire)
140 end compare_exchange_u8
141 exchange_u16: (cell: ptr mut u16, value: u16) -> (previous: u16) =
142 previous = compiler.atomic_exchange(cell, value, compiler.acq_rel)
143 end exchange_u16
144 add_u16: (cell: ptr mut u16, value: u16) -> (previous: u16) =
145 previous = compiler.atomic_add(cell, value, compiler.acq_rel)
146 end add_u16
147 add_relaxed_u16: (cell: ptr mut u16, value: u16) -> (previous: u16) =
148 previous = compiler.atomic_add(cell, value, compiler.relaxed)
149 end add_relaxed_u16
150 compare_exchange_u16: (cell: ptr mut u16, expected: u16, desired: u16)
151 -> (observed: u16) =
152 observed = compiler.atomic_compare_exchange(cell, expected, desired,
153 compiler.acq_rel,
154 compiler.acquire)
155 end compare_exchange_u16
156 exchange_u32: (cell: ptr mut u32, value: u32) -> (previous: u32) =
157 previous = compiler.atomic_exchange(cell, value, compiler.acq_rel)
158 end exchange_u32
159 add_u32: (cell: ptr mut u32, value: u32) -> (previous: u32) =
160 previous = compiler.atomic_add(cell, value, compiler.acq_rel)
161 end add_u32
162 add_relaxed_u32: (cell: ptr mut u32, value: u32) -> (previous: u32) =
163 previous = compiler.atomic_add(cell, value, compiler.relaxed)
164 end add_relaxed_u32
165 compare_exchange_u32: (cell: ptr mut u32, expected: u32, desired: u32)
166 -> (observed: u32) =
167 observed = compiler.atomic_compare_exchange(cell, expected, desired,
168 compiler.acq_rel,
169 compiler.acquire)
170 end compare_exchange_u32
171 exchange_u64: (cell: ptr mut u64, value: u64) -> (previous: u64) =
172 previous = compiler.atomic_exchange(cell, value, compiler.acq_rel)
173 end exchange_u64
174 add_u64: (cell: ptr mut u64, value: u64) -> (previous: u64) =
175 previous = compiler.atomic_add(cell, value, compiler.acq_rel)
176 end add_u64
177 add_relaxed_u64: (cell: ptr mut u64, value: u64) -> (previous: u64) =
178 previous = compiler.atomic_add(cell, value, compiler.relaxed)
179 end add_relaxed_u64
180 compare_exchange_u64: (cell: ptr mut u64, expected: u64, desired: u64)
181 -> (observed: u64) =
182 observed = compiler.atomic_compare_exchange(cell, expected, desired,
183 compiler.acq_rel,
184 compiler.acquire)
185 end compare_exchange_u64
186 u8 is exchangeable (exchange: exchange_u8, add: add_u8,
187 add_relaxed: add_relaxed_u8,
188 compare_exchange: compare_exchange_u8)
189 u16 is exchangeable (exchange: exchange_u16, add: add_u16,
190 add_relaxed: add_relaxed_u16,
191 compare_exchange: compare_exchange_u16)
192 u32 is exchangeable (exchange: exchange_u32, add: add_u32,
193 add_relaxed: add_relaxed_u32,
194 compare_exchange: compare_exchange_u32)
195 u64 is exchangeable (exchange: exchange_u64, add: add_u64,
196 add_relaxed: add_relaxed_u64,
197 compare_exchange: compare_exchange_u64)
198end if
199
200cell_value: type (item: type) = struct
201 value: item
202end cell_value
203
204--- One u8, u16, u32 or u64 read and written atomically. A zeroed cell holds
205--- zero.
206public cell: type (item: type is scalar) = cell_value(item)
207
208--- A cell holding `value`, for a local or a field; module storage writes
209--- `zeroed` instead. Making it is not atomic: make it before another
210--- context can reach it.
211public new: (item: type is scalar, value: item) -> (made: cell(item)) =
212 made = (value: value)
213end new
214
215--- The value `at` holds, with acquire ordering: what the store that wrote
216--- it released is visible after it.
217public load: (item: type is scalar, at: ptr cell(item)) -> (value: item) =
218 value = item.load(addr at.val.value)
219end load
220
221--- The value `at` holds, with no ordering beyond the cell's own.
222public load_relaxed: (item: type is scalar, at: ptr cell(item))
223 -> (value: item) =
224 value = item.load_relaxed(addr at.val.value)
225end load_relaxed
226
227--- Store `value` into `at`, with release ordering: whatever this context
228--- wrote before it is visible to a load that acquires this value.
229public store: (item: type is scalar, at: ptr mut cell(item), value: item)
230 -> none =
231 item.store(addr at.val.value, value)
232end store
233
234--- Store `value` into `at`, with no ordering beyond the cell's own.
235public store_relaxed: (item: type is scalar, at: ptr mut cell(item),
236 value: item) -> none =
237 item.store_relaxed(addr at.val.value, value)
238end store_relaxed
239
240--- Add `value` to `at`, wrapping, as the cell's one writer, and return what
241--- it held: a relaxed load and a release store, so a reader that acquires
242--- the sum sees what the writer did before. It is correct only while one
243--- context alone writes the cell, an interrupt handler or a thread, since a
244--- second writer's store between the two is lost; then it needs no
245--- read-modify-write, and every target has it, Cortex-M0 included. Where
246--- several contexts write, use `add`.
247public sole_add: (item: type is scalar, at: ptr mut cell(item), value: item)
248 -> (previous: item) =
249 previous = item.sole_add(addr at.val.value, value)
250end sole_add
251
252--- Store `value` into `at` and return what it held, in one indivisible
253--- step that acquires and releases. Refused on Cortex-M0.
254public exchange: (item: type is exchangeable, at: ptr mut cell(item),
255 value: item) -> (previous: item) =
256 previous = item.exchange(addr at.val.value, value)
257end exchange
258
259--- Add `value` to `at`, wrapping, and return what it held, in one
260--- indivisible step that acquires and releases. Refused on Cortex-M0.
261public add: (item: type is exchangeable, at: ptr mut cell(item), value: item)
262 -> (previous: item) =
263 previous = item.add(addr at.val.value, value)
264end add
265
266--- Add `value` to `at`, wrapping, and return what it held, in one
267--- indivisible step with no ordering beyond the cell's own: a counter
268--- several contexts bump. Refused on Cortex-M0.
269public add_relaxed: (item: type is exchangeable, at: ptr mut cell(item),
270 value: item) -> (previous: item) =
271 previous = item.add_relaxed(addr at.val.value, value)
272end add_relaxed
273
274--- Store `desired` into `at` if it holds `expected`, and return what it
275--- held, in one indivisible step: the store happened exactly when the
276--- result equals `expected`, and never fails spuriously. A store acquires
277--- and releases; a comparison that fails only acquires. Refused on
278--- Cortex-M0.
279public compare_exchange: (item: type is exchangeable, at: ptr mut cell(item),
280 expected: item, desired: item)
281 -> (observed: item) =
282 observed = item.compare_exchange(addr at.val.value, expected, desired)
283end compare_exchange