1-- The vDSO: a small shared object the kernel maps into every process,
2-- whose functions answer some calls, the clocks among them, without
3-- entering the kernel. The auxiliary vector names where it is mapped,
4-- AT_SYSINFO_EHDR, and its functions are found the way a dynamic loader
5-- finds any: through its ELF program headers, its dynamic table, its
6-- symbols and their versions. Nothing here trusts the image: every
7-- offset is held to lie within it before a byte is read, so an image that
8-- is malformed, truncated or of another kind finds nothing rather than
9-- trapping, and the caller makes the system call instead. A submodule of
10-- platform/linux, since only a caller that reads a clock needs it.
11
12import platform/linux
13
14compiler.assert(compiler.os == linux or compiler.os == darwin
15 or compiler.os == freebsd)
16
17fixed if compiler.os == linux then
18 --- The auxiliary-vector tag of the vDSO's address, AT_SYSINFO_EHDR.
19 public at_sysinfo_ehdr: u64 = 33
20
21 fixed if compiler.arch == x86_64 then
22 --- The name the vDSO gives `clock_gettime` on this architecture.
23 public clock_symbol: []u8 = "__vdso_clock_gettime"
24 --- The symbol version it has: x86-64's is `LINUX_2.6`.
25 public clock_version: []u8 = "LINUX_2.6"
26 elsif compiler.arch == arm64 then
27 --- The name the vDSO gives `clock_gettime` on this architecture.
28 public clock_symbol: []u8 = "__kernel_clock_gettime"
29 --- The symbol version it has: arm64's is `LINUX_2.6.39`.
30 public clock_version: []u8 = "LINUX_2.6.39"
31 else
32 --- The name the vDSO gives `clock_gettime` on this architecture.
33 public clock_symbol: []u8 = "__vdso_clock_gettime"
34 --- The symbol version it has: RV64's is `LINUX_4.15`.
35 public clock_version: []u8 = "LINUX_4.15"
36 end if
37
38 --- The vDSO's `clock_gettime`, a C function: it fills `into` with the
39 --- clock `clock`'s time and returns zero, or minus the errno, making
40 --- the system call itself for a clock it cannot read.
41 public clock_gettime: type = extern(c) (clock: i32, into: ptr mut linux.timespec) -> (result: i32)
42
43 --- The most of an image `symbol` believes: the kernel's vDSO is a
44 --- few pages, and a load segment claiming more is not one.
45 public largest: usize = 1048576
46
47 -- The least of any page size Linux uses, which the vDSO's first page,
48 -- holding its ELF header and program headers, is at least.
49 first_page: usize = 4096
50
51 -- Whether `size` bytes from `at` lie within `image`, asked without an
52 -- addition that could overflow.
53 fits: (image: []u8, at: u64, size: u64) -> (yes: bool) =
54 length: u64 = u64(lenof image)
55 yes = at <= length and size <= length - at
56 end fits
57
58 -- The little-endian integers at `at`, which the caller has held to lie
59 -- within the image.
60 half_at: (image: []u8, at: u64) -> (value: u64) =
61 first: usize = usize(at)
62 value = u64(image[first]) | (u64(image[first + 1]) << 8)
63 end half_at
64
65 word_at: (image: []u8, at: u64) -> (value: u64) =
66 value = half_at(image, at) | (half_at(image, at + 2) << 16)
67 end word_at
68
69 double_at: (image: []u8, at: u64) -> (value: u64) =
70 value = word_at(image, at) | (word_at(image, at + 4) << 32)
71 end double_at
72
73 -- Whether the NUL-terminated string `at` bytes into the string table,
74 -- which starts at `table` and holds `size` bytes, is `wanted`.
75 string_is: (image: []u8, table: u64, size: u64, at: u64, wanted: []u8)
76 -> (yes: bool) =
77 yes = false
78 length: u64 = u64(lenof wanted)
79 return when at >= size or length >= size - at
80 start: u64 = table + at
81 return when not fits(image, start, length + 1)
82 for index in usize(0)..<lenof wanted do
83 return when image[usize(start) + index] <> wanted[index]
84 end for
85 yes = image[usize(start + length)] == 0
86 end string_is
87
88 -- What the dynamic table says, as offsets into the image: zero for a
89 -- table it does not name, since no table lies at the image's start.
90 tables: type = struct
91 hash: u64
92 gnu_hash: u64
93 strings: u64
94 string_size: u64
95 symbols: u64
96 versions: u64
97 definitions: u64
98 end tables
99
100 -- Where in the image the address `address` lies, given the first load
101 -- segment's file offset and address: its offset, or the image's
102 -- length when it lies outside.
103 placed: (image: []u8, load_offset: u64, load_address: u64, address: u64)
104 -> (offset: u64) =
105 length: u64 = u64(lenof image)
106 offset = length
107 return when address < load_address
108 distance: u64 = address - load_address
109 return when distance > length or load_offset >= length - distance
110 offset = load_offset + distance
111 end placed
112
113 -- How many symbols the symbol table holds: DT_HASH's chain count, or
114 -- where DT_GNU_HASH's last chain ends. Zero when neither says.
115 symbol_count: (image: []u8, found: tables) -> (count: u64) =
116 count = 0
117 if found.hash <> 0 then
118 return when not fits(image, found.hash, 8)
119 count = word_at(image, found.hash + 4)
120 return
121 end if
122 return when found.gnu_hash == 0 or not fits(image, found.gnu_hash, 16)
123 buckets: u64 = word_at(image, found.gnu_hash)
124 first: u64 = word_at(image, found.gnu_hash + 4)
125 bloom: u64 = word_at(image, found.gnu_hash + 8)
126 start: u64 = found.gnu_hash + 16
127 length: u64 = u64(lenof image)
128 return when bloom > length / 8 or not fits(image, start, bloom * 8)
129 bucket_start: u64 = start + bloom * 8
130 return when buckets > length / 4
131 or not fits(image, bucket_start, buckets * 4)
132 mut last: u64 = 0
133 for index in u64(0)..<buckets do
134 value: u64 = word_at(image, bucket_start + index * 4)
135 if value > last then last = value end if
136 end for
137 if last < first then
138 count = first
139 return
140 end if
141 chain_start: u64 = bucket_start + buckets * 4
142 loop do
143 return when last - first > length / 4
144 at: u64 = chain_start + (last - first) * 4
145 return when not fits(image, at, 4)
146 break when (word_at(image, at) & 1) <> 0
147 inc last
148 end loop
149 count = last + 1
150 end symbol_count
151
152 -- Whether the symbol at `index` has the version named `version`: the
153 -- version definitions are walked, as a dynamic loader walks them, for
154 -- the one whose index the symbol's version entry gives. A table of
155 -- definitions that runs on is cut off after 256.
156 has_version: (image: []u8, found: tables, index: u64, version: []u8)
157 -> (yes: bool) =
158 yes = false
159 entry: u64 = found.versions + index * 2
160 return when not fits(image, entry, 2)
161 wanted: u64 = half_at(image, entry) & 0x7fff
162 mut at: u64 = found.definitions
163 for step in usize(0)..<256 do
164 _ = step
165 return when not fits(image, at, 20)
166 flags: u64 = half_at(image, at + 2)
167 number: u64 = half_at(image, at + 4) & 0x7fff
168 if (flags & 1) == 0 and number == wanted then
169 auxiliary: u64 = at + word_at(image, at + 12)
170 return when not fits(image, auxiliary, 8)
171 yes = string_is(image, found.strings, found.string_size,
172 word_at(image, auxiliary), version)
173 return
174 end if
175 next: u64 = word_at(image, at + 16)
176 return when next == 0
177 at += next
178 end for
179 end has_version
180
181 -- What a 64-bit little-endian ELF image's program headers say: its
182 -- first load segment's file offset, address and size in the file,
183 -- and where its dynamic table lies. `loaded` is false when the
184 -- image is no such object, its headers do not lie within it, or none
185 -- of them is a load segment.
186 segments: type = struct
187 loaded: bool
188 load_offset: u64
189 load_address: u64
190 load_size: u64
191 dynamic: u64
192 dynamic_size: u64
193 end segments
194
195 segments_of: (image: []u8) -> (found: segments) =
196 found = (loaded: false, load_offset: 0, load_address: 0,
197 load_size: 0, dynamic: 0, dynamic_size: 0)
198 return when not fits(image, 0, 64)
199 return when image[0] <> 0x7f or image[1] <> 0x45 or image[2] <> 0x4c
200 or image[3] <> 0x46 or image[4] <> 2 or image[5] <> 1
201 headers: u64 = double_at(image, 32)
202 header_size: u64 = half_at(image, 54)
203 header_count: u64 = half_at(image, 56)
204 return when header_size < 56
205 or not fits(image, headers, header_size * header_count)
206 for index in u64(0)..<header_count do
207 at: u64 = headers + index * header_size
208 kind: u64 = word_at(image, at)
209 if kind == 1 and not found.loaded then
210 found.loaded = true
211 found.load_offset = double_at(image, at + 8)
212 found.load_address = double_at(image, at + 16)
213 found.load_size = double_at(image, at + 32)
214 elsif kind == 2 then
215 found.dynamic = double_at(image, at + 8)
216 found.dynamic_size = double_at(image, at + 32)
217 end if
218 end for
219 end segments_of
220
221 --- Find the function `name` of version `version` in `image`, the
222 --- bytes of a shared object as it is mapped: whether it is there, and
223 --- its offset from the image's start. A global or weak function
224 --- defined in the image counts; where the image has no version
225 --- tables, the name alone decides, as a dynamic loader decides it.
226 --- Every offset is held to lie within the image before it is read, so
227 --- an image that is not a 64-bit little-endian ELF object, is
228 --- truncated or contradicts itself finds nothing and never traps.
229 public link(symbol: "_landin_linux_vdso_find") find:
230 (image: []u8, name: []u8, version: []u8)
231 -> (found: bool, offset: usize) =
232 found = false
233 offset = 0
234 parts: segments = segments_of(image)
235 return when not parts.loaded or parts.dynamic_size == 0
236 or not fits(image, parts.dynamic, parts.dynamic_size)
237 load_offset: u64 = parts.load_offset
238 load_address: u64 = parts.load_address
239 dynamic: u64 = parts.dynamic
240 dynamic_size: u64 = parts.dynamic_size
241 mut found_tables: tables = (hash: 0, gnu_hash: 0, strings: 0,
242 string_size: 0, symbols: 0, versions: 0,
243 definitions: 0)
244 for index in u64(0)..<dynamic_size / 16 do
245 at: u64 = dynamic + index * 16
246 tag: u64 = double_at(image, at)
247 break when tag == 0
248 place: u64 = placed(image, load_offset, load_address,
249 double_at(image, at + 8))
250 if tag == 10 then
251 found_tables.string_size = double_at(image, at + 8)
252 elsif place < u64(lenof image) then
253 if tag == 4 then found_tables.hash = place
254 elsif tag == 0x6ffffef5 then found_tables.gnu_hash = place
255 elsif tag == 5 then found_tables.strings = place
256 elsif tag == 6 then found_tables.symbols = place
257 elsif tag == 0x6ffffff0 then found_tables.versions = place
258 elsif tag == 0x6ffffffc then found_tables.definitions = place
259 end if
260 end if
261 end for
262 return when found_tables.strings == 0 or found_tables.symbols == 0
263 length: u64 = u64(lenof image)
264 if found_tables.string_size == 0
265 or not fits(image, found_tables.strings, found_tables.string_size)
266 then
267 found_tables.string_size = length - found_tables.strings
268 end if
269 versioned: bool = found_tables.versions <> 0
270 and found_tables.definitions <> 0
271 count: u64 = symbol_count(image, found_tables)
272 return when count > length / 24
273 or not fits(image, found_tables.symbols, count * 24)
274 for index in u64(0)..<count do
275 at: u64 = found_tables.symbols + index * 24
276 information: u64 = u64(image[usize(at + 4)])
277 binding: u64 = information >> 4
278 section: u64 = half_at(image, at + 6)
279 defined: bool = section <> 0 and (information & 15) == 2
280 and (binding == 1 or binding == 2)
281 if defined and string_is(image, found_tables.strings,
282 found_tables.string_size,
283 word_at(image, at), name)
284 and (not versioned
285 or has_version(image, found_tables, index, version)) then
286 place: u64 = placed(image, load_offset, load_address,
287 double_at(image, at + 8))
288 if place < length then
289 found = true
290 offset = usize(place)
291 return
292 end if
293 end if
294 end for
295 end find
296
297 -- What `kernel_image` gives when there is no vDSO.
298 no_image: [1]u8 = [0]
299
300 -- The `length` bytes the kernel mapped at `first`, which is not zero.
301 mapped: (first: usize, length: usize) -> (bytes: []u8) =
302 whole: ptr [1]u8 = ptr(first)
303 bytes = whole.val[0..<0]
304 if length > 0 then
305 unchecked begin
306 bytes = whole.val[0..<length]
307 end unchecked
308 end if
309 end mapped
310
311 --- The vDSO as the kernel mapped it: whether the auxiliary vector
312 --- names one, and its bytes, as far as its first load segment reaches
313 --- and no further than `largest`. There is none when `auxiliary`
314 --- found no auxiliary vector, as when a C `main` built the tables it
315 --- passed the argument initializer itself, or when the image's first
316 --- page does not describe a load segment.
317 public link(symbol: "_landin_linux_vdso_image") kernel_image:
318 () -> (found: bool, image: []u8) =
319 (found: named, value: base) := linux.auxiliary(at_sysinfo_ehdr)
320 found = false
321 image = no_image[0..<0]
322 return when not named or base == 0
323 page: []u8 = mapped(usize(base), first_page)
324 parts: segments = segments_of(page)
325 start: u64 = parts.load_offset
326 size: u64 = parts.load_size
327 limit: u64 = u64(largest)
328 return when not parts.loaded or start > limit
329 or size > limit - start or start + size < 64
330 image = mapped(usize(base), usize(start + size))
331 found = true
332 end kernel_image
333
334 --- The address of the vDSO's function `name` of version `version`,
335 --- found by `find` in `kernel_image`: whether it is there, and
336 --- where.
337 public link(symbol: "_landin_linux_vdso_symbol") symbol:
338 (name: []u8, version: []u8) -> (found: bool, address: usize) =
339 (found: present, image) := kernel_image()
340 found = false
341 address = 0
342 return when not present
343 (found: named, offset) := find(image, name, version)
344 (found: _, value: base) := linux.auxiliary(at_sysinfo_ehdr)
345 found = named
346 if named then address = usize(base) + offset end if
347 end symbol
348end if