1-- The vDSO: a small shared object the kernel maps into every process,
2-- whose functions answer some calls, the clocks among them, without
3-- entering the kernel. The auxiliary vector names where it is mapped,
4-- AT_SYSINFO_EHDR, and its functions are found the way a dynamic loader
5-- finds any: through its ELF program headers, its dynamic table, its
6-- symbols and their versions. Nothing here trusts the image: every
7-- offset is held to lie within it before a byte is read, so an image that
8-- is malformed, truncated or of another kind finds nothing rather than
9-- trapping, and the caller makes the system call instead.
10
11fixed if compiler.os == linux then
12 --- The auxiliary-vector tag of the vDSO's address, AT_SYSINFO_EHDR.
13 public at_sysinfo_ehdr: u64 = 33
14
15 fixed if compiler.arch == x86_64 then
16 --- The name the vDSO gives `clock_gettime` on this architecture.
17 public vdso_clock_name: []u8 = "__vdso_clock_gettime"
18 --- The symbol version it has: x86-64's is `LINUX_2.6`.
19 public vdso_version: []u8 = "LINUX_2.6"
20 elsif compiler.arch == arm64 then
21 --- The name the vDSO gives `clock_gettime` on this architecture.
22 public vdso_clock_name: []u8 = "__kernel_clock_gettime"
23 --- The symbol version it has: arm64's is `LINUX_2.6.39`.
24 public vdso_version: []u8 = "LINUX_2.6.39"
25 else
26 --- The name the vDSO gives `clock_gettime` on this architecture.
27 public vdso_clock_name: []u8 = "__vdso_clock_gettime"
28 --- The symbol version it has: RV64's is `LINUX_4.15`.
29 public vdso_version: []u8 = "LINUX_4.15"
30 end if
31
32 --- The vDSO's `clock_gettime`, a C function: it fills `into` with the
33 --- clock `clock`'s time and returns zero, or minus the errno, making
34 --- the system call itself for a clock it cannot read.
35 public vdso_clock_gettime: type = extern(c) (clock: i32, into: ptr mut timespec) -> (result: i32)
36
37 --- The most of an image `vdso_symbol` believes: the kernel's vDSO is a
38 --- few pages, and a load segment claiming more is not one.
39 public vdso_largest: usize = 1048576
40
41 -- The least of any page size Linux uses, which the vDSO's first page,
42 -- holding its ELF header and program headers, is at least.
43 first_page: usize = 4096
44
45 -- Whether `size` bytes from `at` lie within `image`, asked without an
46 -- addition that could overflow.
47 fits: (image: []u8, at: u64, size: u64) -> (yes: bool) =
48 length: u64 = u64(lenof image)
49 yes = at <= length and size <= length - at
50 end fits
51
52 -- The little-endian integers at `at`, which the caller has held to lie
53 -- within the image.
54 half_at: (image: []u8, at: u64) -> (value: u64) =
55 first: usize = usize(at)
56 value = u64(image[first]) | (u64(image[first + 1]) << 8)
57 end half_at
58
59 word_at: (image: []u8, at: u64) -> (value: u64) =
60 value = half_at(image, at) | (half_at(image, at + 2) << 16)
61 end word_at
62
63 double_at: (image: []u8, at: u64) -> (value: u64) =
64 value = word_at(image, at) | (word_at(image, at + 4) << 32)
65 end double_at
66
67 -- Whether the NUL-terminated string `at` bytes into the string table,
68 -- which starts at `table` and holds `size` bytes, is `wanted`.
69 string_is: (image: []u8, table: u64, size: u64, at: u64, wanted: []u8)
70 -> (yes: bool) =
71 yes = false
72 length: u64 = u64(lenof wanted)
73 return when at >= size or length >= size - at
74 start: u64 = table + at
75 return when not fits(image, start, length + 1)
76 for index in usize(0)..<lenof wanted do
77 return when image[usize(start) + index] <> wanted[index]
78 end for
79 yes = image[usize(start + length)] == 0
80 end string_is
81
82 -- What the dynamic table says, as offsets into the image: zero for a
83 -- table it does not name, since no table lies at the image's start.
84 tables: type = struct
85 hash: u64
86 gnu_hash: u64
87 strings: u64
88 string_size: u64
89 symbols: u64
90 versions: u64
91 definitions: u64
92 end tables
93
94 -- Where in the image the address `address` lies, given the first load
95 -- segment's file offset and address: its offset, or the image's
96 -- length when it lies outside.
97 placed: (image: []u8, load_offset: u64, load_address: u64, address: u64)
98 -> (offset: u64) =
99 length: u64 = u64(lenof image)
100 offset = length
101 return when address < load_address
102 distance: u64 = address - load_address
103 return when distance > length or load_offset >= length - distance
104 offset = load_offset + distance
105 end placed
106
107 -- How many symbols the symbol table holds: DT_HASH's chain count, or
108 -- where DT_GNU_HASH's last chain ends. Zero when neither says.
109 symbol_count: (image: []u8, found: tables) -> (count: u64) =
110 count = 0
111 if found.hash <> 0 then
112 return when not fits(image, found.hash, 8)
113 count = word_at(image, found.hash + 4)
114 return
115 end if
116 return when found.gnu_hash == 0 or not fits(image, found.gnu_hash, 16)
117 buckets: u64 = word_at(image, found.gnu_hash)
118 first: u64 = word_at(image, found.gnu_hash + 4)
119 bloom: u64 = word_at(image, found.gnu_hash + 8)
120 start: u64 = found.gnu_hash + 16
121 length: u64 = u64(lenof image)
122 return when bloom > length / 8 or not fits(image, start, bloom * 8)
123 bucket_start: u64 = start + bloom * 8
124 return when buckets > length / 4
125 or not fits(image, bucket_start, buckets * 4)
126 mut last: u64 = 0
127 for index in u64(0)..<buckets do
128 value: u64 = word_at(image, bucket_start + index * 4)
129 if value > last then last = value end if
130 end for
131 if last < first then
132 count = first
133 return
134 end if
135 chain_start: u64 = bucket_start + buckets * 4
136 loop do
137 return when last - first > length / 4
138 at: u64 = chain_start + (last - first) * 4
139 return when not fits(image, at, 4)
140 break when (word_at(image, at) & 1) <> 0
141 inc last
142 end loop
143 count = last + 1
144 end symbol_count
145
146 -- Whether the symbol at `index` has the version named `version`: the
147 -- version definitions are walked, as a dynamic loader walks them, for
148 -- the one whose index the symbol's version entry gives. A table of
149 -- definitions that runs on is cut off after 256.
150 has_version: (image: []u8, found: tables, index: u64, version: []u8)
151 -> (yes: bool) =
152 yes = false
153 entry: u64 = found.versions + index * 2
154 return when not fits(image, entry, 2)
155 wanted: u64 = half_at(image, entry) & 0x7fff
156 mut at: u64 = found.definitions
157 for step in usize(0)..<256 do
158 _ = step
159 return when not fits(image, at, 20)
160 flags: u64 = half_at(image, at + 2)
161 number: u64 = half_at(image, at + 4) & 0x7fff
162 if (flags & 1) == 0 and number == wanted then
163 auxiliary: u64 = at + word_at(image, at + 12)
164 return when not fits(image, auxiliary, 8)
165 yes = string_is(image, found.strings, found.string_size,
166 word_at(image, auxiliary), version)
167 return
168 end if
169 next: u64 = word_at(image, at + 16)
170 return when next == 0
171 at += next
172 end for
173 end has_version
174
175 -- What a 64-bit little-endian ELF image's program headers say: its
176 -- first load segment's file offset, address and size in the file,
177 -- and where its dynamic table lies. `loaded` is false when the
178 -- image is no such object, its headers do not lie within it, or none
179 -- of them is a load segment.
180 segments: type = struct
181 loaded: bool
182 load_offset: u64
183 load_address: u64
184 load_size: u64
185 dynamic: u64
186 dynamic_size: u64
187 end segments
188
189 segments_of: (image: []u8) -> (found: segments) =
190 found = (loaded: false, load_offset: 0, load_address: 0,
191 load_size: 0, dynamic: 0, dynamic_size: 0)
192 return when not fits(image, 0, 64)
193 return when image[0] <> 0x7f or image[1] <> 0x45 or image[2] <> 0x4c
194 or image[3] <> 0x46 or image[4] <> 2 or image[5] <> 1
195 headers: u64 = double_at(image, 32)
196 header_size: u64 = half_at(image, 54)
197 header_count: u64 = half_at(image, 56)
198 return when header_size < 56
199 or not fits(image, headers, header_size * header_count)
200 for index in u64(0)..<header_count do
201 at: u64 = headers + index * header_size
202 kind: u64 = word_at(image, at)
203 if kind == 1 and not found.loaded then
204 found.loaded = true
205 found.load_offset = double_at(image, at + 8)
206 found.load_address = double_at(image, at + 16)
207 found.load_size = double_at(image, at + 32)
208 elsif kind == 2 then
209 found.dynamic = double_at(image, at + 8)
210 found.dynamic_size = double_at(image, at + 32)
211 end if
212 end for
213 end segments_of
214
215 --- Find the function `name` of version `version` in `image`, the
216 --- bytes of a shared object as it is mapped: whether it is there, and
217 --- its offset from the image's start. A global or weak function
218 --- defined in the image counts; where the image has no version
219 --- tables, the name alone decides, as a dynamic loader decides it.
220 --- Every offset is held to lie within the image before it is read, so
221 --- an image that is not a 64-bit little-endian ELF object, is
222 --- truncated or contradicts itself finds nothing and never traps.
223 public link(symbol: "_landin_linux_vdso_find") vdso_find:
224 (image: []u8, name: []u8, version: []u8)
225 -> (found: bool, offset: usize) =
226 found = false
227 offset = 0
228 parts: segments = segments_of(image)
229 return when not parts.loaded or parts.dynamic_size == 0
230 or not fits(image, parts.dynamic, parts.dynamic_size)
231 load_offset: u64 = parts.load_offset
232 load_address: u64 = parts.load_address
233 dynamic: u64 = parts.dynamic
234 dynamic_size: u64 = parts.dynamic_size
235 mut found_tables: tables = (hash: 0, gnu_hash: 0, strings: 0,
236 string_size: 0, symbols: 0, versions: 0,
237 definitions: 0)
238 for index in u64(0)..<dynamic_size / 16 do
239 at: u64 = dynamic + index * 16
240 tag: u64 = double_at(image, at)
241 break when tag == 0
242 place: u64 = placed(image, load_offset, load_address,
243 double_at(image, at + 8))
244 if tag == 10 then
245 found_tables.string_size = double_at(image, at + 8)
246 elsif place < u64(lenof image) then
247 if tag == 4 then found_tables.hash = place
248 elsif tag == 0x6ffffef5 then found_tables.gnu_hash = place
249 elsif tag == 5 then found_tables.strings = place
250 elsif tag == 6 then found_tables.symbols = place
251 elsif tag == 0x6ffffff0 then found_tables.versions = place
252 elsif tag == 0x6ffffffc then found_tables.definitions = place
253 end if
254 end if
255 end for
256 return when found_tables.strings == 0 or found_tables.symbols == 0
257 length: u64 = u64(lenof image)
258 if found_tables.string_size == 0
259 or not fits(image, found_tables.strings, found_tables.string_size)
260 then
261 found_tables.string_size = length - found_tables.strings
262 end if
263 versioned: bool = found_tables.versions <> 0
264 and found_tables.definitions <> 0
265 count: u64 = symbol_count(image, found_tables)
266 return when count > length / 24
267 or not fits(image, found_tables.symbols, count * 24)
268 for index in u64(0)..<count do
269 at: u64 = found_tables.symbols + index * 24
270 information: u64 = u64(image[usize(at + 4)])
271 binding: u64 = information >> 4
272 section: u64 = half_at(image, at + 6)
273 defined: bool = section <> 0 and (information & 15) == 2
274 and (binding == 1 or binding == 2)
275 if defined and string_is(image, found_tables.strings,
276 found_tables.string_size,
277 word_at(image, at), name)
278 and (not versioned
279 or has_version(image, found_tables, index, version)) then
280 place: u64 = placed(image, load_offset, load_address,
281 double_at(image, at + 8))
282 if place < length then
283 found = true
284 offset = usize(place)
285 return
286 end if
287 end if
288 end for
289 end vdso_find
290
291 -- What `vdso_image` gives when there is no vDSO.
292 no_image: [1]u8 = [0]
293
294 -- The `length` bytes the kernel mapped at `first`, which is not zero.
295 mapped: (first: usize, length: usize) -> (bytes: []u8) =
296 whole: ptr [1]u8 = ptr(first)
297 bytes = whole.val[0..<0]
298 if length > 0 then
299 unchecked begin
300 bytes = whole.val[0..<length]
301 end unchecked
302 end if
303 end mapped
304
305 --- The vDSO as the kernel mapped it: whether the auxiliary vector
306 --- names one, and its bytes, as far as its first load segment reaches
307 --- and no further than `vdso_largest`. There is none when the C
308 --- runtime started the program, whose auxiliary vector `auxiliary`
309 --- does not see, or when the image's first page does not describe a
310 --- load segment.
311 public link(symbol: "_landin_linux_vdso_image") vdso_image:
312 () -> (found: bool, image: []u8) =
313 (found: named, value: base) := auxiliary(at_sysinfo_ehdr)
314 found = false
315 image = no_image[0..<0]
316 return when not named or base == 0
317 page: []u8 = mapped(usize(base), first_page)
318 parts: segments = segments_of(page)
319 start: u64 = parts.load_offset
320 size: u64 = parts.load_size
321 largest: u64 = u64(vdso_largest)
322 return when not parts.loaded or start > largest
323 or size > largest - start or start + size < 64
324 image = mapped(usize(base), usize(start + size))
325 found = true
326 end vdso_image
327
328 --- The address of the vDSO's function `name` of version `version`,
329 --- found by `vdso_find` in `vdso_image`: whether it is there, and
330 --- where.
331 public link(symbol: "_landin_linux_vdso_symbol") vdso_symbol:
332 (name: []u8, version: []u8) -> (found: bool, address: usize) =
333 (found: present, image) := vdso_image()
334 found = false
335 address = 0
336 if present then
337 (found: named, offset) := vdso_find(image, name, version)
338 (found: _, value: base) := auxiliary(at_sysinfo_ehdr)
339 found = named
340 if named then address = usize(base) + offset end if
341 end if
342 end vdso_symbol
343end if