1-- Waiting on an address and waking whoever waits there: the system's own
2-- sleep on a word of memory, which a lock waits through when it is
3-- taken (D501). The word is a `core/atomic` cell of a `u32`, so whoever
4-- changes it does so atomically, and only this process maps it. It is
5-- Linux's futex, FreeBSD's `_umtx_op` and Darwin's
6-- `os_sync_wait_on_address`; POSIX itself names none, so each system's
7-- own is here under one name.
8
9import core/atomic
10import platform/libc
11import platform/linux
12
13--- Sleep while the cell at `word` holds `expected`, until `wake_one` or
14--- `wake_all` wakes this thread. The system compares and sleeps in one
15--- step, so a wake that follows a change to the cell is never lost; it
16--- returns at once when the cell does not hold `expected`. It may also
17--- return with nobody having woken it, for a signal or for nothing, so
18--- whoever waits reads the cell again and waits again while what it
19--- waits for has not happened. Waiting blocks the whole thread, and with
20--- it every fibre of its round.
21public wait_on:
22 (word: ptr atomic.cell(u32), expected: u32) -> none =
23 address_wait(word, expected)
24end wait_on
25
26--- `wait_on`, for at most `span`, measured on the monotonic clock: it
27--- fails `etimedout` when the span passed with the cell still holding
28--- `expected`, as far as the system saw, and `einval` for nanoseconds of a
29--- second or more or a negative span. Any other return is `wait_on`'s.
30public wait_on_within:
31 (word: ptr atomic.cell(u32), expected: u32, span: time_value)
32 -> none ! errno =
33 fail einval when span.seconds < 0 or span.nanoseconds >= 1000000000
34 try address_wait_within(word, expected, span)
35end wait_on_within
36
37--- Wake one of the threads waiting on the cell at `word`, if any is.
38--- Change the cell first: a thread that has not yet begun to wait then
39--- finds it changed and does not sleep.
40public wake_one:
41 (word: ptr atomic.cell(u32)) -> none =
42 address_wake(word, false)
43end wake_one
44
45--- Wake every thread waiting on the cell at `word`.
46public wake_all:
47 (word: ptr atomic.cell(u32)) -> none =
48 address_wake(word, true)
49end wake_all
50
51-- The cell's one field is its first, so its address is the word's.
52
53fixed if compiler.os == linux then
54 address_wait: (word: ptr atomic.cell(u32), expected: u32) -> none =
55 at: ptr u32 = ptr(usize(word))
56 _ = linux.futex_wait(at, expected, linux.no_limit)
57 end address_wait
58
59 address_wait_within:
60 (word: ptr atomic.cell(u32), expected: u32, span: time_value)
61 -> none ! errno =
62 at: ptr u32 = ptr(usize(word))
63 limit: linux.timespec = (seconds: span.seconds,
64 nanoseconds: i64(span.nanoseconds))
65 result: i32 = linux.futex_wait(at, expected, addr limit)
66 fail etimedout when result < 0 and errno_of(-result) == etimedout
67 end address_wait_within
68
69 address_wake: (word: ptr atomic.cell(u32), every: bool) -> none =
70 at: ptr u32 = ptr(usize(word))
71 count: i32 = if every then 2147483647 else 1 end if
72 _ = linux.futex_wake(at, count)
73 end address_wake
74elsif compiler.os == freebsd then
75 address_wait: (word: ptr atomic.cell(u32), expected: u32) -> none =
76 _ = libc._umtx_op(usize(word), libc.umtx_op_wait_uint_private,
77 u64(expected), 0, 0)
78 end address_wait
79
80 address_wait_within:
81 (word: ptr atomic.cell(u32), expected: u32, span: time_value)
82 -> none ! errno =
83 limit: libc.umtx_time = (timeout: (seconds: span.seconds,
84 nanoseconds:
85 i64(span.nanoseconds)),
86 flags: 0, clock: u32(libc.clock_monotonic))
87 status: i32 = libc._umtx_op(usize(word),
88 libc.umtx_op_wait_uint_private,
89 u64(expected), sizeof libc.umtx_time,
90 usize(addr limit))
91 return when status == 0
92 problem: errno = failure()
93 fail etimedout when problem == etimedout
94 end address_wait_within
95
96 address_wake: (word: ptr atomic.cell(u32), every: bool) -> none =
97 count: u64 = if every then 2147483647 else 1 end if
98 _ = libc._umtx_op(usize(word), libc.umtx_op_wake_private, count, 0,
99 0)
100 end address_wake
101elsif compiler.os == darwin then
102 address_wait: (word: ptr atomic.cell(u32), expected: u32) -> none =
103 _ = libc.os_sync_wait_on_address(usize(word), u64(expected), 4,
104 libc.os_sync_none)
105 end address_wait
106
107 -- The span in nanoseconds, as long as a u64 holds.
108 address_wait_within:
109 (word: ptr atomic.cell(u32), expected: u32, span: time_value)
110 -> none ! errno =
111 whole: u64 = u64(span.seconds)
112 most: u64 = 18446744073
113 nanoseconds: u64 = if whole >= most then 0 -% u64(1)
114 else whole * 1000000000 + u64(span.nanoseconds)
115 end if
116 result: i32 = libc.os_sync_wait_on_address_with_timeout(
117 usize(word), u64(expected), 4, libc.os_sync_none,
118 libc.os_clock_mach_absolute_time, nanoseconds)
119 return when result >= 0
120 problem: errno = failure()
121 fail etimedout when problem == etimedout
122 end address_wait_within
123
124 address_wake: (word: ptr atomic.cell(u32), every: bool) -> none =
125 if every then
126 _ = libc.os_sync_wake_by_address_all(usize(word), 4,
127 libc.os_sync_none)
128 else
129 _ = libc.os_sync_wake_by_address_any(usize(word), 4,
130 libc.os_sync_none)
131 end if
132 end address_wake
133end if