1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
//! Linking for Universal-compiled code.

use crate::{
    get_libcall_trampoline,
    types::{
        relocation::{RelocationKind, RelocationLike, RelocationTarget},
        section::SectionIndex,
    },
    FunctionExtent,
};
use std::{
    collections::{HashMap, HashSet},
    ptr::{read_unaligned, write_unaligned},
};

use wasmer_types::{entity::PrimaryMap, LocalFunctionIndex, ModuleInfo};
use wasmer_vm::{libcalls::function_pointer, SectionBodyPtr};

#[allow(clippy::too_many_arguments)]
fn apply_relocation(
    body: usize,
    r: &impl RelocationLike,
    allocated_functions: &PrimaryMap<LocalFunctionIndex, FunctionExtent>,
    allocated_sections: &PrimaryMap<SectionIndex, SectionBodyPtr>,
    libcall_trampolines_sec_idx: SectionIndex,
    libcall_trampoline_len: usize,
    riscv_pcrel_hi20s: &mut HashMap<usize, u32>,
    get_got_address: &dyn Fn(RelocationTarget) -> Option<usize>,
) {
    let reloc_target = r.reloc_target();

    // Note: if the relocation needs GOT and its addend is not zero we will relax the
    // relocation and, instead of making it use the GOT entry, we will fixup the assembly to
    // use the final pointer directly, without any indirection. Also, see the comment in
    // compiler-llvm/src/object_file.rs:288.
    let target_func_address: usize = if r.kind().needs_got() && r.addend() == 0 {
        if let Some(got_address) = get_got_address(reloc_target) {
            got_address
        } else {
            panic!("No GOT entry for reloc target {reloc_target:?}")
        }
    } else {
        match reloc_target {
            RelocationTarget::LocalFunc(index) => *allocated_functions[index].ptr as usize,
            RelocationTarget::LibCall(libcall) => {
                // Use the direct target of the libcall if the relocation supports
                // a full 64-bit address. Otherwise use a trampoline.
                if matches!(
                    r.kind(),
                    RelocationKind::Abs8
                        | RelocationKind::X86PCRel8
                        | RelocationKind::MachoArm64RelocUnsigned
                        | RelocationKind::MachoX86_64RelocUnsigned
                ) {
                    function_pointer(libcall)
                } else {
                    get_libcall_trampoline(
                        libcall,
                        allocated_sections[libcall_trampolines_sec_idx].0 as usize,
                        libcall_trampoline_len,
                    )
                }
            }
            RelocationTarget::CustomSection(custom_section) => {
                *allocated_sections[custom_section] as usize
            }
        }
    };

    // A set of addresses at which a SUBTRACTOR relocation was applied.
    let mut macho_aarch64_subtractor_addresses = HashSet::new();

    match r.kind() {
        RelocationKind::Abs8 => unsafe {
            let (reloc_address, reloc_delta) = r.for_address(body, target_func_address as u64);
            write_unaligned(reloc_address as *mut u64, reloc_delta);
        },
        RelocationKind::X86PCRel4 => unsafe {
            let (reloc_address, reloc_delta) = r.for_address(body, target_func_address as u64);
            write_unaligned(reloc_address as *mut u32, reloc_delta as _);
        },
        RelocationKind::X86PCRel8 => unsafe {
            let (reloc_address, reloc_delta) = r.for_address(body, target_func_address as u64);
            write_unaligned(reloc_address as *mut u64, reloc_delta);
        },
        RelocationKind::X86CallPCRel4 => unsafe {
            let (reloc_address, reloc_delta) = r.for_address(body, target_func_address as u64);
            write_unaligned(reloc_address as *mut u32, reloc_delta as _);
        },
        RelocationKind::Arm64Call => unsafe {
            let (reloc_address, reloc_delta) = r.for_address(body, target_func_address as u64);
            if (reloc_delta as i64).abs() >= 0x1000_0000 {
                panic!(
                    "Relocation to big for {:?} for {:?} with {:x}, current val {:x}",
                    r.kind(),
                    r.reloc_target(),
                    reloc_delta,
                    read_unaligned(reloc_address as *mut u32)
                )
            }
            let reloc_delta = (((reloc_delta / 4) as u32) & 0x3ff_ffff)
                | (read_unaligned(reloc_address as *mut u32) & 0xfc00_0000);
            write_unaligned(reloc_address as *mut u32, reloc_delta);
        },
        RelocationKind::Arm64Movw0 => unsafe {
            let (reloc_address, reloc_delta) = r.for_address(body, target_func_address as u64);
            let reloc_delta =
                (((reloc_delta & 0xffff) as u32) << 5) | read_unaligned(reloc_address as *mut u32);
            write_unaligned(reloc_address as *mut u32, reloc_delta);
        },
        RelocationKind::Arm64Movw1 => unsafe {
            let (reloc_address, reloc_delta) = r.for_address(body, target_func_address as u64);
            let reloc_delta = ((((reloc_delta >> 16) & 0xffff) as u32) << 5)
                | read_unaligned(reloc_address as *mut u32);
            write_unaligned(reloc_address as *mut u32, reloc_delta);
        },
        RelocationKind::Arm64Movw2 => unsafe {
            let (reloc_address, reloc_delta) = r.for_address(body, target_func_address as u64);
            let reloc_delta = ((((reloc_delta >> 32) & 0xffff) as u32) << 5)
                | read_unaligned(reloc_address as *mut u32);
            write_unaligned(reloc_address as *mut u32, reloc_delta);
        },
        RelocationKind::Arm64Movw3 => unsafe {
            let (reloc_address, reloc_delta) = r.for_address(body, target_func_address as u64);
            let reloc_delta = ((((reloc_delta >> 48) & 0xffff) as u32) << 5)
                | read_unaligned(reloc_address as *mut u32);
            write_unaligned(reloc_address as *mut u32, reloc_delta);
        },
        RelocationKind::RiscvPCRelHi20 => unsafe {
            let (reloc_address, reloc_delta) = r.for_address(body, target_func_address as u64);

            // save for later reference with RiscvPCRelLo12I
            riscv_pcrel_hi20s.insert(reloc_address, reloc_delta as u32);

            let reloc_delta = ((reloc_delta.wrapping_add(0x800) & 0xfffff000) as u32)
                | read_unaligned(reloc_address as *mut u32);
            write_unaligned(reloc_address as *mut u32, reloc_delta);
        },
        RelocationKind::RiscvPCRelLo12I => unsafe {
            let (reloc_address, reloc_abs) = r.for_address(body, target_func_address as u64);
            let reloc_delta = ((riscv_pcrel_hi20s.get(&(reloc_abs as usize)).expect(
                "R_RISCV_PCREL_LO12_I relocation target must be a symbol with R_RISCV_PCREL_HI20",
            ) & 0xfff)
                << 20)
                | read_unaligned(reloc_address as *mut u32);
            write_unaligned(reloc_address as *mut u32, reloc_delta);
        },
        RelocationKind::RiscvCall => unsafe {
            let (reloc_address, reloc_delta) = r.for_address(body, target_func_address as u64);
            let reloc_delta = ((reloc_delta & 0xfff) << 52)
                | (reloc_delta.wrapping_add(0x800) & 0xfffff000)
                | read_unaligned(reloc_address as *mut u64);
            write_unaligned(reloc_address as *mut u64, reloc_delta);
        },
        RelocationKind::LArchAbsHi20 | RelocationKind::LArchPCAlaHi20 => unsafe {
            let (reloc_address, reloc_abs) = r.for_address(body, target_func_address as u64);
            let reloc_abs = ((((reloc_abs >> 12) & 0xfffff) as u32) << 5)
                | read_unaligned(reloc_address as *mut u32);
            write_unaligned(reloc_address as *mut u32, reloc_abs);
        },
        RelocationKind::LArchAbsLo12 | RelocationKind::LArchPCAlaLo12 => unsafe {
            let (reloc_address, reloc_abs) = r.for_address(body, target_func_address as u64);
            let reloc_abs =
                (((reloc_abs & 0xfff) as u32) << 10) | read_unaligned(reloc_address as *mut u32);
            write_unaligned(reloc_address as *mut u32, reloc_abs);
        },
        RelocationKind::LArchAbs64Hi12 | RelocationKind::LArchPCAla64Hi12 => unsafe {
            let (reloc_address, reloc_abs) = r.for_address(body, target_func_address as u64);
            let reloc_abs = ((((reloc_abs >> 52) & 0xfff) as u32) << 10)
                | read_unaligned(reloc_address as *mut u32);
            write_unaligned(reloc_address as *mut u32, reloc_abs);
        },
        RelocationKind::LArchAbs64Lo20 | RelocationKind::LArchPCAla64Lo20 => unsafe {
            let (reloc_address, reloc_abs) = r.for_address(body, target_func_address as u64);
            let reloc_abs = ((((reloc_abs >> 32) & 0xfffff) as u32) << 5)
                | read_unaligned(reloc_address as *mut u32);
            write_unaligned(reloc_address as *mut u32, reloc_abs);
        },
        RelocationKind::LArchCall36 => unsafe {
            let (reloc_address, reloc_delta) = r.for_address(body, target_func_address as u64);
            let reloc_delta1 = ((((reloc_delta >> 18) & 0xfffff) as u32) << 5)
                | read_unaligned(reloc_address as *mut u32);
            write_unaligned(reloc_address as *mut u32, reloc_delta1);
            let reloc_delta2 = ((((reloc_delta >> 2) & 0xffff) as u32) << 10)
                | read_unaligned((reloc_address + 4) as *mut u32);
            write_unaligned((reloc_address + 4) as *mut u32, reloc_delta2);
        },
        RelocationKind::Aarch64AdrPrelPgHi21 => unsafe {
            let (reloc_address, delta) = r.for_address(body, target_func_address as u64);

            let delta = delta as isize;
            assert!(
                ((-1 << 32)..(1 << 32)).contains(&delta),
                "can't generate page-relative relocation with ±4GB `adrp` instruction"
            );

            let op = read_unaligned(reloc_address as *mut u32);
            let delta = delta >> 12;
            let immlo = ((delta as u32) & 0b11) << 29;
            let immhi = (((delta as u32) >> 2) & 0x7ffff) << 5;
            let mask = !((0x7ffff << 5) | (0b11 << 29));
            let op = (op & mask) | immlo | immhi;

            write_unaligned(reloc_address as *mut u32, op);
        },
        RelocationKind::Aarch64AdrPrelLo21 => unsafe {
            let (reloc_address, delta) = r.for_address(body, target_func_address as u64);

            let delta = delta as isize;
            assert!(
                ((-1 << 20)..(1 << 20)).contains(&delta),
                "can't generate an ADR_PREL_LO21 relocation with an immediate larger than 20 bits"
            );

            let op = read_unaligned(reloc_address as *mut u32);
            let immlo = ((delta as u32) & 0b11) << 29;
            let immhi = (((delta as u32) >> 2) & 0x7ffff) << 5;
            let mask = !((0x7ffff << 5) | (0b11 << 29));
            let op = (op & mask) | immlo | immhi;

            write_unaligned(reloc_address as *mut u32, op);
        },
        RelocationKind::Aarch64AddAbsLo12Nc => unsafe {
            let (reloc_address, delta) = r.for_address(body, target_func_address as u64);

            let delta = delta as isize;
            let op = read_unaligned(reloc_address as *mut u32);
            let imm = ((delta as u32) & 0xfff) << 10;
            let mask = !((0xfff) << 10);
            let op = (op & mask) | imm;

            write_unaligned(reloc_address as *mut u32, op);
        },
        RelocationKind::Aarch64Ldst128AbsLo12Nc => unsafe {
            let (reloc_address, reloc_delta) = r.for_address(body, target_func_address as u64);
            let reloc_delta = ((reloc_delta as u32 & 0xfff) >> 4) << 10
                | (read_unaligned(reloc_address as *mut u32) & 0xFFC003FF);
            write_unaligned(reloc_address as *mut u32, reloc_delta);
        },
        RelocationKind::Aarch64Ldst64AbsLo12Nc => unsafe {
            let (reloc_address, reloc_delta) = r.for_address(body, target_func_address as u64);
            let reloc_delta = ((reloc_delta as u32 & 0xfff) >> 3) << 10
                | (read_unaligned(reloc_address as *mut u32) & 0xFFC003FF);
            write_unaligned(reloc_address as *mut u32, reloc_delta);
        },
        RelocationKind::MachoArm64RelocSubtractor | RelocationKind::MachoX86_64RelocSubtractor => unsafe {
            let (reloc_address, reloc_sub) = r.for_address(body, target_func_address as u64);
            macho_aarch64_subtractor_addresses.insert(reloc_address);
            write_unaligned(reloc_address as *mut u64, reloc_sub);
        },
        RelocationKind::MachoArm64RelocGotLoadPage21
        | RelocationKind::MachoArm64RelocTlvpLoadPage21 => unsafe {
            let (reloc_address, _) = r.for_address(body, target_func_address as u64);
            let target_func_page = target_func_address & !0xfff;
            let reloc_at_page = reloc_address & !0xfff;
            let pcrel = (target_func_page as isize)
                .checked_sub(reloc_at_page as isize)
                .unwrap();
            assert!(
                (-1 << 32) <= (pcrel as i64) && (pcrel as i64) < (1 << 32),
                "can't reach GOT page with ±4GB `adrp` instruction"
            );
            let val = pcrel >> 12;

            let immlo = ((val as u32) & 0b11) << 29;
            let immhi = (((val as u32) >> 2) & 0x7ffff) << 5;
            let mask = !((0x7ffff << 5) | (0b11 << 29));
            let op = read_unaligned(reloc_address as *mut u32);
            write_unaligned(reloc_address as *mut u32, (op & mask) | immlo | immhi);
        },

        RelocationKind::MachoArm64RelocPage21 => unsafe {
            let target_page: u64 =
                ((target_func_address.wrapping_add(r.addend() as _)) & !0xfff) as u64;
            let reloc_address = body.wrapping_add(r.offset() as _);
            let pc_page: u64 = (reloc_address & !0xfff) as u64;
            let page_delta = target_page - pc_page;
            let raw_instr = read_unaligned(reloc_address as *mut u32);
            assert_eq!(
                (raw_instr & 0xffffffe0),
                0x90000000,
                "raw_instr isn't an ADRP instruction"
            );

            let immlo: u32 = ((page_delta >> 12) & 0x3) as _;
            let immhi: u32 = ((page_delta >> 14) & 0x7ffff) as _;
            let fixed_instr = raw_instr | (immlo << 29) | (immhi << 5);
            write_unaligned(reloc_address as *mut u32, fixed_instr);
        },
        RelocationKind::MachoArm64RelocPageoff12 => unsafe {
            let target_offset: u64 =
                ((target_func_address.wrapping_add(r.addend() as _)) & 0xfff) as u64;

            let reloc_address = body.wrapping_add(r.offset() as _);
            let raw_instr = read_unaligned(reloc_address as *mut u32);
            let imm_shift = {
                const VEC128_MASK: u32 = 0x04800000;

                const LOAD_STORE_IMM12_MASK: u32 = 0x3b000000;
                let is_load_store_imm12 = (raw_instr & LOAD_STORE_IMM12_MASK) == 0x39000000;

                if is_load_store_imm12 {
                    let mut implicit_shift = raw_instr >> 30;

                    if implicit_shift == 0 && (raw_instr & VEC128_MASK) == VEC128_MASK {
                        implicit_shift = 4;
                    }

                    implicit_shift
                } else {
                    0
                }
            };

            assert_eq!(
                target_offset & ((1 << imm_shift) - 1),
                0,
                "PAGEOFF12 target is not aligned"
            );

            let encoded_imm: u32 = ((target_offset as u32) >> imm_shift) << 10;
            let fixed_instr: u32 = raw_instr | encoded_imm;
            write_unaligned(reloc_address as *mut u32, fixed_instr);
        },

        RelocationKind::MachoArm64RelocGotLoadPageoff12 => unsafe {
            // See comment at the top of the function. TLDR: if addend != 0 we can't really use the
            // GOT entry. We fixup this relocation to use a `add` rather than a `ldr` instruction,
            // skipping the indirection from the GOT.
            if r.addend() == 0 {
                let (reloc_address, _) = r.for_address(body, target_func_address as u64);
                assert_eq!(target_func_address & 0b111, 0);
                let val = target_func_address >> 3;
                let imm9 = ((val & 0x1ff) << 10) as u32;
                let mask = !(0x1ff << 10);
                let op = read_unaligned(reloc_address as *mut u32);
                write_unaligned(reloc_address as *mut u32, (op & mask) | imm9);
            } else {
                let fixup_ptr = body + r.offset() as usize;
                let target_address: usize = target_func_address + r.addend() as usize;

                let raw_instr = read_unaligned(fixup_ptr as *mut u32);

                assert_eq!(
                    raw_instr & 0xfffffc00, 0xf9400000,
                    "raw_instr isn't a 64-bit LDR immediate (bits: {raw_instr:032b}, hex: {raw_instr:x})"
                );

                let reg: u32 = raw_instr & 0b11111;

                let mut fixup_ldr = 0x91000000 | (reg << 5) | reg;
                fixup_ldr |= ((target_address & 0xfff) as u32) << 10;

                write_unaligned(fixup_ptr as *mut u32, fixup_ldr);
            }
        },
        RelocationKind::MachoArm64RelocUnsigned | RelocationKind::MachoX86_64RelocUnsigned => unsafe {
            let (reloc_address, mut reloc_delta) = r.for_address(body, target_func_address as u64);

            if macho_aarch64_subtractor_addresses.contains(&reloc_address) {
                reloc_delta -= read_unaligned(reloc_address as *mut u64);
            }

            write_unaligned(reloc_address as *mut u64, reloc_delta);
        },

        RelocationKind::MachoArm64RelocPointerToGot => unsafe {
            let at = body + r.offset() as usize;
            let pcrel = i32::try_from((target_func_address as isize) - (at as isize)).unwrap();
            write_unaligned(at as *mut i32, pcrel);
        },

        RelocationKind::MachoArm64RelocBranch26 => unsafe {
            let fixup_ptr = body + r.offset() as usize;
            assert_eq!(fixup_ptr & 0x3, 0, "Branch-inst is not 32-bit aligned");
            let value = i32::try_from((target_func_address as isize) - (fixup_ptr as isize))
                .unwrap()
                .wrapping_add(r.addend() as _);
            assert!(
                value & 0x3 == 0,
                "BranchPCRel26 target is not 32-bit aligned"
            );

            assert!(
                (-(1 << 27)..=((1 << 27) - 1)).contains(&value),
                "out of range BranchPCRel26 target"
            );

            let raw_instr = read_unaligned(fixup_ptr as *mut u32);

            assert_eq!(
                raw_instr & 0x7fffffff,
                0x14000000,
                "RawInstr isn't a B or BR immediate instruction"
            );
            let imm: u32 = ((value as u32) & ((1 << 28) - 1)) >> 2;
            let fixed_instr: u32 = raw_instr | imm;

            write_unaligned(fixup_ptr as *mut u32, fixed_instr);
        },
        kind => panic!("Relocation kind unsupported in the current architecture: {kind}"),
    }
}

/// Links a module, patching the allocated functions with the
/// required relocations and jump tables.
#[allow(clippy::too_many_arguments)]
pub fn link_module<'a>(
    _module: &ModuleInfo,
    allocated_functions: &PrimaryMap<LocalFunctionIndex, FunctionExtent>,
    function_relocations: impl Iterator<
        Item = (
            LocalFunctionIndex,
            impl Iterator<Item = &'a (impl RelocationLike + 'a)>,
        ),
    >,
    allocated_sections: &PrimaryMap<SectionIndex, SectionBodyPtr>,
    section_relocations: impl Iterator<
        Item = (
            SectionIndex,
            impl Iterator<Item = &'a (impl RelocationLike + 'a)>,
        ),
    >,
    libcall_trampolines: SectionIndex,
    trampoline_len: usize,
    get_got_address: &'a dyn Fn(RelocationTarget) -> Option<usize>,
) {
    let mut riscv_pcrel_hi20s: HashMap<usize, u32> = HashMap::new();

    for (i, section_relocs) in section_relocations {
        let body = *allocated_sections[i] as usize;
        for r in section_relocs {
            apply_relocation(
                body,
                r,
                allocated_functions,
                allocated_sections,
                libcall_trampolines,
                trampoline_len,
                &mut riscv_pcrel_hi20s,
                get_got_address,
            );
        }
    }
    for (i, function_relocs) in function_relocations {
        let body = *allocated_functions[i].ptr as usize;
        for r in function_relocs {
            apply_relocation(
                body,
                r,
                allocated_functions,
                allocated_sections,
                libcall_trampolines,
                trampoline_len,
                &mut riscv_pcrel_hi20s,
                get_got_address,
            );
        }
    }
}