wasmer_compiler_llvm/translator/
trampoline.rs

1// TODO: Remove
2#![allow(unused)]
3
4use crate::{
5    abi::{LLVMAbi, get_abi},
6    config::LLVM,
7    error::{err, err_nt},
8    object_file::{RkyvCompiledFunction, load_object_file},
9    translator::intrinsics::{Intrinsics, type_to_llvm},
10};
11use inkwell::{
12    AddressSpace, DLLStorageClass,
13    attributes::{Attribute, AttributeLoc},
14    context::Context,
15    module::{Linkage, Module},
16    passes::PassBuilderOptions,
17    targets::{FileType, TargetMachine},
18    types::FunctionType,
19    values::{BasicMetadataValueEnum, FunctionValue},
20};
21use std::{cmp, convert::TryInto};
22use target_lexicon::{BinaryFormat, Triple};
23use wasmer_compiler::{
24    misc::{CompiledFunctionExt, CompiledKind},
25    types::{
26        function::{CompiledFunctionBody, FunctionBody},
27        module::CompileModuleInfo,
28        relocation::{Relocation, RelocationTarget},
29        section::{CustomSection, CustomSectionProtection, SectionBody, SectionIndex},
30    },
31};
32use wasmer_types::{
33    CompileError, FunctionIndex, FunctionType as FuncType, LocalFunctionIndex, MemoryIndex,
34    entity::PrimaryMap,
35};
36use wasmer_vm::MemoryStyle;
37
38pub struct FuncTrampoline {
39    ctx: Context,
40    target_machine: TargetMachine,
41    target_triple: Triple,
42    abi: LLVMAbi,
43    binary_fmt: BinaryFormat,
44    func_section: String,
45}
46
47const FUNCTION_SECTION_ELF: &str = "__TEXT,wasmer_trmpl"; // Needs to be between 1 and 16 chars
48const FUNCTION_SECTION_MACHO: &str = "wasmer_trmpl"; // Needs to be between 1 and 16 chars
49
50fn enable_m0_optimization(compile_info: &CompileModuleInfo) -> bool {
51    compile_info
52        .memory_styles
53        .get(MemoryIndex::from_u32(0))
54        .is_some_and(|memory| matches!(memory, MemoryStyle::Static))
55}
56
57impl FuncTrampoline {
58    pub fn new(
59        target_machine: TargetMachine,
60        target_triple: Triple,
61        binary_fmt: BinaryFormat,
62    ) -> Result<Self, CompileError> {
63        let abi = get_abi(&target_machine)?;
64        Ok(Self {
65            ctx: Context::create(),
66            target_machine,
67            target_triple,
68            abi,
69            func_section: match binary_fmt {
70                BinaryFormat::Elf => FUNCTION_SECTION_ELF.to_string(),
71                BinaryFormat::Macho => FUNCTION_SECTION_MACHO.to_string(),
72                _ => {
73                    return Err(CompileError::UnsupportedTarget(format!(
74                        "Unsupported binary format: {binary_fmt:?}",
75                    )));
76                }
77            },
78            binary_fmt,
79        })
80    }
81
82    pub fn trampoline_to_module(
83        &self,
84        ty: &FuncType,
85        config: &LLVM,
86        function: &CompiledKind,
87        compile_info: &CompileModuleInfo,
88    ) -> Result<Module<'_>, CompileError> {
89        // The function type, used for the callbacks.
90        let module = self.ctx.create_module("");
91        let target_machine = &self.target_machine;
92        let target_triple = target_machine.get_triple();
93        let target_data: inkwell::targets::TargetData = target_machine.get_target_data();
94        module.set_triple(&target_triple);
95        module.set_data_layout(&target_data.get_data_layout());
96        let intrinsics = Intrinsics::declare(
97            &module,
98            &self.ctx,
99            &target_data,
100            &self.target_triple,
101            &self.binary_fmt,
102        );
103
104        let m0_is_enabled = enable_m0_optimization(compile_info);
105        let (callee_ty, callee_attrs) =
106            self.abi
107                .func_type_to_llvm(&self.ctx, &intrinsics, None, ty, m0_is_enabled)?;
108        let trampoline_ty = intrinsics.void_ty.fn_type(
109            &[
110                intrinsics.ptr_ty.into(), // vmctx ptr
111                intrinsics.ptr_ty.into(), // callee function address
112                intrinsics.ptr_ty.into(), // in/out values ptr
113            ],
114            false,
115        );
116
117        let trampoline_func = module.add_function(
118            &function.linkage_name(),
119            trampoline_ty,
120            Some(Linkage::External),
121        );
122        if !config.experimental_artifact {
123            trampoline_func
124                .as_global_value()
125                .set_section(Some(&self.func_section));
126        }
127        trampoline_func
128            .as_global_value()
129            .set_linkage(Linkage::DLLExport);
130        trampoline_func
131            .as_global_value()
132            .set_dll_storage_class(DLLStorageClass::Export);
133        // We intentionally mark this function as no-unwind; otherwise, stack unwinding could cross the
134        // trampoline boundary and cause Rust to complain about a foreign exception being thrown.
135        trampoline_func.add_attribute(AttributeLoc::Function, intrinsics.nounwind);
136        trampoline_func.add_attribute(AttributeLoc::Function, intrinsics.frame_pointer);
137        self.generate_trampoline(
138            config,
139            compile_info,
140            trampoline_func,
141            ty,
142            callee_ty,
143            &callee_attrs,
144            &self.ctx,
145            &intrinsics,
146        )?;
147
148        if let Some(ref callbacks) = config.callbacks {
149            callbacks.preopt_ir(function, &compile_info.module.hash_string(), &module);
150        }
151
152        let mut passes = vec![];
153
154        if config.enable_verifier {
155            passes.push("verify");
156        }
157
158        passes.push("instcombine");
159        module
160            .run_passes(
161                passes.join(",").as_str(),
162                target_machine,
163                PassBuilderOptions::create(),
164            )
165            .unwrap();
166
167        if let Some(ref callbacks) = config.callbacks {
168            callbacks.postopt_ir(function, &compile_info.module.hash_string(), &module);
169        }
170        Ok(module)
171    }
172
173    pub fn trampoline(
174        &self,
175        ty: &FuncType,
176        config: &LLVM,
177        function: &CompiledKind,
178        compile_info: &CompileModuleInfo,
179    ) -> Result<CompiledFunctionBody, CompileError> {
180        let module = self.trampoline_to_module(ty, config, function, compile_info)?;
181        let target_machine = &self.target_machine;
182
183        let memory_buffer = target_machine
184            .write_to_memory_buffer(&module, FileType::Object)
185            .unwrap();
186
187        if let Some(ref callbacks) = config.callbacks {
188            let module_hash = compile_info.module.hash_string();
189            callbacks.obj_memory_buffer(function, &module_hash, &memory_buffer);
190            let asm_buffer = target_machine
191                .write_to_memory_buffer(&module, FileType::Assembly)
192                .unwrap();
193            callbacks.asm_memory_buffer(function, &module_hash, &asm_buffer);
194        }
195
196        if config.experimental_artifact {
197            Ok(CompiledFunctionBody::Elf(memory_buffer.as_slice().to_vec()))
198        } else {
199            // Use a dummy function index to detect relocations against the trampoline
200            // function's address, which shouldn't exist and are not supported.
201            // Note, we just drop all custom sections, and verify that the function
202            // body itself has no relocations at all. This value should never be
203            // touched at all. However, it is set up so that if we do touch it (maybe
204            // due to someone changing the code later on), it'll explode, which is desirable!
205            let dummy_reloc_target =
206                RelocationTarget::DynamicTrampoline(FunctionIndex::from_u32(u32::MAX - 1));
207
208            // Note: we don't count .gcc_except_table here because native-to-wasm
209            // trampolines are not supposed to generate any LSDA sections. We *want* them
210            // to terminate libunwind's stack searches.
211            let RkyvCompiledFunction {
212                compiled_function,
213                custom_sections,
214                eh_frame_section_indices,
215                mut compact_unwind_section_indices,
216                ..
217            } = load_object_file(
218                memory_buffer.as_slice(),
219                &self.func_section,
220                dummy_reloc_target,
221                |name: &str| {
222                    Err(CompileError::Codegen(format!(
223                        "trampoline generation produced reference to unknown function {name}",
224                    )))
225                },
226                self.binary_fmt,
227                &self.target_triple,
228            )?;
229            let mut all_sections_are_eh_sections = true;
230            let mut unwind_section_indices = eh_frame_section_indices;
231            unwind_section_indices.append(&mut compact_unwind_section_indices);
232            if unwind_section_indices.len() != custom_sections.len() {
233                all_sections_are_eh_sections = false;
234            } else {
235                unwind_section_indices.sort_unstable();
236                for (idx, section_idx) in unwind_section_indices.iter().enumerate() {
237                    if idx as u32 != section_idx.as_u32() {
238                        all_sections_are_eh_sections = false;
239                        break;
240                    }
241                }
242            }
243            if !all_sections_are_eh_sections {
244                return Err(CompileError::Codegen(
245                    "trampoline generation produced non-eh custom sections".into(),
246                ));
247            }
248            if !compiled_function.relocations.is_empty() {
249                return Err(CompileError::Codegen(
250                    "trampoline generation produced relocations".into(),
251                ));
252            }
253            // Ignore CompiledFunctionFrameInfo. Extra frame info isn't a problem.
254
255            Ok(CompiledFunctionBody::Rkyv(FunctionBody {
256                body: compiled_function.body.body,
257                unwind_info: compiled_function.body.unwind_info,
258            }))
259        }
260    }
261
262    pub fn dynamic_trampoline_to_module(
263        &self,
264        ty: &FuncType,
265        config: &LLVM,
266        function: &CompiledKind,
267        module_hash: &Option<String>,
268    ) -> Result<Module<'_>, CompileError> {
269        // The function type, used for the callbacks
270        let module = self.ctx.create_module("");
271        let target_machine = &self.target_machine;
272        let target_data = target_machine.get_target_data();
273        let target_triple = target_machine.get_triple();
274        module.set_triple(&target_triple);
275        module.set_data_layout(&target_data.get_data_layout());
276        let intrinsics = Intrinsics::declare(
277            &module,
278            &self.ctx,
279            &target_data,
280            &self.target_triple,
281            &self.binary_fmt,
282        );
283
284        let (trampoline_ty, trampoline_attrs) =
285            self.abi
286                .func_type_to_llvm(&self.ctx, &intrinsics, None, ty, false)?;
287        let trampoline_func = module.add_function(
288            &function.linkage_name(),
289            trampoline_ty,
290            Some(Linkage::External),
291        );
292        trampoline_func.set_personality_function(intrinsics.personality);
293        trampoline_func.add_attribute(AttributeLoc::Function, intrinsics.frame_pointer);
294        for (attr, attr_loc) in trampoline_attrs {
295            trampoline_func.add_attribute(attr_loc, attr);
296        }
297        if !config.experimental_artifact {
298            trampoline_func
299                .as_global_value()
300                .set_section(Some(&self.func_section));
301        }
302        trampoline_func
303            .as_global_value()
304            .set_linkage(Linkage::DLLExport);
305        trampoline_func
306            .as_global_value()
307            .set_dll_storage_class(DLLStorageClass::Export);
308        self.generate_dynamic_trampoline(trampoline_func, ty, &self.ctx, &intrinsics)?;
309
310        if let Some(ref callbacks) = config.callbacks {
311            callbacks.preopt_ir(function, module_hash, &module);
312        }
313
314        let mut passes = vec![];
315
316        if config.enable_verifier {
317            passes.push("verify");
318        }
319
320        passes.push("early-cse");
321        module
322            .run_passes(
323                passes.join(",").as_str(),
324                target_machine,
325                PassBuilderOptions::create(),
326            )
327            .unwrap();
328
329        if let Some(ref callbacks) = config.callbacks {
330            callbacks.postopt_ir(function, module_hash, &module);
331        }
332
333        Ok(module)
334    }
335
336    #[allow(clippy::too_many_arguments)]
337    pub fn dynamic_trampoline(
338        &self,
339        ty: &FuncType,
340        config: &LLVM,
341        function: &CompiledKind,
342        dynamic_trampoline_index: u32,
343        final_module_custom_sections: &mut PrimaryMap<SectionIndex, CustomSection>,
344        eh_frame_section_bytes: &mut Vec<u8>,
345        eh_frame_section_relocations: &mut Vec<Relocation>,
346        compact_unwind_section_bytes: &mut Vec<u8>,
347        compact_unwind_section_relocations: &mut Vec<Relocation>,
348        module_hash: &Option<String>,
349    ) -> Result<CompiledFunctionBody, CompileError> {
350        let target_machine = &self.target_machine;
351
352        let module = self.dynamic_trampoline_to_module(ty, config, function, module_hash)?;
353
354        let memory_buffer = target_machine
355            .write_to_memory_buffer(&module, FileType::Object)
356            .unwrap();
357
358        if let Some(ref callbacks) = config.callbacks {
359            callbacks.obj_memory_buffer(function, module_hash, &memory_buffer);
360            let asm_buffer = target_machine
361                .write_to_memory_buffer(&module, FileType::Assembly)
362                .unwrap();
363            callbacks.asm_memory_buffer(function, module_hash, &asm_buffer)
364        }
365
366        if config.experimental_artifact {
367            Ok(CompiledFunctionBody::Elf(memory_buffer.as_slice().to_vec()))
368        } else {
369            let RkyvCompiledFunction {
370                compiled_function,
371                custom_sections,
372                eh_frame_section_indices,
373                compact_unwind_section_indices,
374                gcc_except_table_section_indices,
375                data_dw_ref_personality_section_indices,
376            } = load_object_file(
377                memory_buffer.as_slice(),
378                &self.func_section,
379                RelocationTarget::DynamicTrampoline(FunctionIndex::from_u32(
380                    dynamic_trampoline_index,
381                )),
382                |name: &str| {
383                    Err(CompileError::Codegen(format!(
384                        "trampoline generation produced reference to unknown function {name}",
385                    )))
386                },
387                self.binary_fmt,
388                &self.target_triple,
389            )?;
390
391            if !compiled_function.relocations.is_empty() {
392                return Err(CompileError::Codegen(
393                    "trampoline generation produced relocations".into(),
394                ));
395            }
396            // Ignore CompiledFunctionFrameInfo. Extra frame info isn't a problem.
397
398            // Also append EH-related sections to the final module, since we expect
399            // dynamic trampolines to participate in unwinding
400            {
401                let first_section = final_module_custom_sections.len() as u32;
402                for (section_index, mut custom_section) in custom_sections.into_iter() {
403                    for reloc in &mut custom_section.relocations {
404                        if let RelocationTarget::CustomSection(index) = reloc.reloc_target {
405                            reloc.reloc_target = RelocationTarget::CustomSection(
406                                SectionIndex::from_u32(first_section + index.as_u32()),
407                            )
408                        }
409
410                        if reloc.kind.needs_got() {
411                            return Err(CompileError::Codegen(
412                                "trampoline generation produced GOT relocation".into(),
413                            ));
414                        }
415                    }
416
417                    if eh_frame_section_indices.contains(&section_index) {
418                        let offset = eh_frame_section_bytes.len() as u32;
419                        for reloc in &mut custom_section.relocations {
420                            reloc.offset += offset;
421                        }
422                        eh_frame_section_bytes.extend_from_slice(custom_section.bytes.as_slice());
423                        // Terminate the eh_frame info with a zero-length CIE.
424                        eh_frame_section_bytes.extend_from_slice(&[0, 0, 0, 0]);
425                        eh_frame_section_relocations.extend(custom_section.relocations);
426                        // TODO: we do this to keep the count right, remove it.
427                        final_module_custom_sections.push(CustomSection {
428                            protection: CustomSectionProtection::Read,
429                            alignment: None,
430                            bytes: SectionBody::new_with_vec(vec![]),
431                            relocations: vec![],
432                        });
433                    } else if compact_unwind_section_indices.contains(&section_index) {
434                        let offset = compact_unwind_section_bytes.len() as u32;
435                        for reloc in &mut custom_section.relocations {
436                            reloc.offset += offset;
437                        }
438                        compact_unwind_section_bytes
439                            .extend_from_slice(custom_section.bytes.as_slice());
440                        compact_unwind_section_relocations.extend(custom_section.relocations);
441                        // TODO: we do this to keep the count right, remove it.
442                        final_module_custom_sections.push(CustomSection {
443                            protection: CustomSectionProtection::Read,
444                            alignment: None,
445                            bytes: SectionBody::new_with_vec(vec![]),
446                            relocations: vec![],
447                        });
448                    } else if gcc_except_table_section_indices.contains(&section_index)
449                        || data_dw_ref_personality_section_indices.contains(&section_index)
450                    {
451                        final_module_custom_sections.push(custom_section);
452                    } else {
453                        return Err(CompileError::Codegen(
454                            "trampoline generation produced non-eh custom sections".into(),
455                        ));
456                    }
457                }
458            }
459
460            Ok(CompiledFunctionBody::Rkyv(FunctionBody {
461                body: compiled_function.body.body,
462                unwind_info: compiled_function.body.unwind_info,
463            }))
464        }
465    }
466
467    #[allow(clippy::too_many_arguments)]
468    fn generate_trampoline<'ctx>(
469        &self,
470        config: &LLVM,
471        compile_info: &CompileModuleInfo,
472        trampoline_func: FunctionValue,
473        func_sig: &FuncType,
474        llvm_func_type: FunctionType,
475        func_attrs: &[(Attribute, AttributeLoc)],
476        context: &'ctx Context,
477        intrinsics: &Intrinsics<'ctx>,
478    ) -> Result<(), CompileError> {
479        let entry_block = context.append_basic_block(trampoline_func, "entry");
480        let builder = context.create_builder();
481        builder.position_at_end(entry_block);
482
483        let (callee_vmctx_ptr, func_ptr, args_rets_ptr) =
484            match *trampoline_func.get_params().as_slice() {
485                [callee_vmctx_ptr, func_ptr, args_rets_ptr] => (
486                    callee_vmctx_ptr,
487                    func_ptr.into_pointer_value(),
488                    args_rets_ptr.into_pointer_value(),
489                ),
490                _ => {
491                    return Err(CompileError::Codegen(
492                        "trampoline function unimplemented".to_string(),
493                    ));
494                }
495            };
496        func_ptr.set_name("func_ptr");
497
498        let mut args_vec = Vec::with_capacity(func_sig.params().len() + 3);
499
500        if self.abi.is_sret(func_sig)? {
501            let basic_types: Vec<_> = func_sig
502                .results()
503                .iter()
504                .map(|&ty| type_to_llvm(intrinsics, ty))
505                .collect::<Result<_, _>>()?;
506
507            let sret_ty = context.struct_type(&basic_types, false);
508            args_vec.push(err!(builder.build_alloca(sret_ty, "sret")).into());
509        }
510
511        callee_vmctx_ptr.set_name("vmctx");
512        args_vec.push(callee_vmctx_ptr.into());
513
514        if enable_m0_optimization(compile_info) {
515            let wasm_module = &compile_info.module;
516            let memory_styles = &compile_info.memory_styles;
517            let callee_vmctx_ptr_value = callee_vmctx_ptr.into_pointer_value();
518            let offsets = wasmer_vm::VMOffsets::new(8, wasm_module);
519
520            // load mem
521            let memory_index = wasmer_types::MemoryIndex::from_u32(0);
522            let memory_definition_ptr = if let Some(local_memory_index) =
523                wasm_module.local_memory_index(memory_index)
524            {
525                let offset = offsets.vmctx_vmmemory_definition(local_memory_index);
526                let offset = intrinsics.i32_ty.const_int(offset.into(), false);
527                unsafe {
528                    err!(builder.build_gep(intrinsics.i8_ty, callee_vmctx_ptr_value, &[offset], ""))
529                }
530            } else {
531                let offset = offsets.vmctx_vmmemory_import(memory_index);
532                let offset = intrinsics.i32_ty.const_int(offset.into(), false);
533                let memory_definition_ptr_ptr = unsafe {
534                    err!(builder.build_gep(intrinsics.i8_ty, callee_vmctx_ptr_value, &[offset], ""))
535                };
536                let memory_definition_ptr_ptr =
537                    err!(builder.build_bit_cast(memory_definition_ptr_ptr, intrinsics.ptr_ty, "",))
538                        .into_pointer_value();
539
540                err!(builder.build_load(intrinsics.ptr_ty, memory_definition_ptr_ptr, ""))
541                    .into_pointer_value()
542            };
543            let memory_definition_ptr =
544                err!(builder.build_bit_cast(memory_definition_ptr, intrinsics.ptr_ty, "",))
545                    .into_pointer_value();
546            let base_ptr = err!(builder.build_struct_gep(
547                intrinsics.vmmemory_definition_ty,
548                memory_definition_ptr,
549                intrinsics.vmmemory_definition_base_element,
550                "",
551            ));
552
553            let memory_style = &memory_styles[memory_index];
554            let base_ptr = if let MemoryStyle::Dynamic { .. } = memory_style {
555                base_ptr
556            } else {
557                err!(builder.build_load(intrinsics.ptr_ty, base_ptr, "")).into_pointer_value()
558            };
559
560            base_ptr.set_name("trmpl_m0_base_ptr");
561
562            args_vec.push(base_ptr.into());
563        }
564
565        for (i, param_ty) in func_sig.params().iter().enumerate() {
566            let index = intrinsics.i32_ty.const_int(i as _, false);
567            let item_pointer = unsafe {
568                err!(builder.build_in_bounds_gep(
569                    intrinsics.i128_ty,
570                    args_rets_ptr,
571                    &[index],
572                    "arg_ptr"
573                ))
574            };
575
576            let casted_type = type_to_llvm(intrinsics, *param_ty)?;
577
578            let typed_item_pointer = err!(builder.build_pointer_cast(
579                item_pointer,
580                intrinsics.ptr_ty,
581                "typed_arg_pointer"
582            ));
583
584            let arg = err!(builder.build_load(casted_type, typed_item_pointer, "arg"));
585            args_vec.push(arg.into());
586        }
587
588        let call_site = err!(builder.build_indirect_call(
589            llvm_func_type,
590            func_ptr,
591            args_vec.as_slice(),
592            "call"
593        ));
594        for (attr, attr_loc) in func_attrs {
595            call_site.add_attribute(*attr_loc, *attr);
596        }
597
598        let rets = self
599            .abi
600            .rets_from_call(&builder, intrinsics, call_site, func_sig)?;
601        for (idx, v) in rets.into_iter().enumerate() {
602            let ptr = unsafe {
603                err!(builder.build_gep(
604                    intrinsics.i128_ty,
605                    args_rets_ptr,
606                    &[intrinsics.i32_ty.const_int(idx as u64, false)],
607                    "",
608                ))
609            };
610            let ptr = err!(builder.build_pointer_cast(
611                ptr,
612                self.ctx.ptr_type(AddressSpace::default()),
613                ""
614            ));
615            err!(builder.build_store(ptr, v));
616        }
617
618        err!(builder.build_return(None));
619        Ok(())
620    }
621
622    fn generate_dynamic_trampoline<'ctx>(
623        &self,
624        trampoline_func: FunctionValue,
625        func_sig: &FuncType,
626        context: &'ctx Context,
627        intrinsics: &Intrinsics<'ctx>,
628    ) -> Result<(), CompileError> {
629        let entry_block = context.append_basic_block(trampoline_func, "entry");
630        let builder = context.create_builder();
631        builder.position_at_end(entry_block);
632
633        // Allocate stack space for the params and results.
634        let values = err!(builder.build_alloca(
635            intrinsics.i128_ty.array_type(cmp::max(
636                func_sig.params().len().try_into().unwrap(),
637                func_sig.results().len().try_into().unwrap(),
638            )),
639            "",
640        ));
641
642        // Copy params to 'values'.
643        let first_user_param = if self.abi.is_sret(func_sig)? { 2 } else { 1 };
644        for i in 0..func_sig.params().len() {
645            let ptr = unsafe {
646                err!(builder.build_in_bounds_gep(
647                    intrinsics.i128_ty,
648                    values,
649                    &[intrinsics.i32_ty.const_int(i.try_into().unwrap(), false)],
650                    "args",
651                ))
652            };
653            let ptr = err!(builder.build_bit_cast(ptr, intrinsics.ptr_ty, "")).into_pointer_value();
654            err!(
655                builder.build_store(
656                    ptr,
657                    trampoline_func
658                        .get_nth_param(i as u32 + first_user_param)
659                        .unwrap(),
660                )
661            );
662        }
663
664        let callee_ptr_ty = intrinsics.void_ty.fn_type(
665            &[
666                intrinsics.ptr_ty.into(), // vmctx ptr
667                intrinsics.ptr_ty.into(), // in/out values ptr
668            ],
669            false,
670        );
671        let vmctx = self.abi.get_vmctx_ptr_param(&trampoline_func);
672        let callee_ty =
673            err!(builder.build_bit_cast(vmctx, self.ctx.ptr_type(AddressSpace::default()), ""));
674        let callee =
675            err!(builder.build_load(intrinsics.ptr_ty, callee_ty.into_pointer_value(), ""))
676                .into_pointer_value();
677        callee.set_name("func_ptr");
678
679        let values_ptr = err!(builder.build_pointer_cast(values, intrinsics.ptr_ty, ""));
680        values_ptr.set_name("value_ptr");
681        err!(builder.build_indirect_call(
682            callee_ptr_ty,
683            callee,
684            &[vmctx.into(), values_ptr.into()],
685            "",
686        ));
687
688        if func_sig.results().is_empty() {
689            err!(builder.build_return(None));
690        } else {
691            let results = func_sig
692                .results()
693                .iter()
694                .enumerate()
695                .map(|(idx, ty)| {
696                    let ptr = unsafe {
697                        err!(builder.build_gep(
698                            intrinsics.i128_ty,
699                            values,
700                            &[intrinsics.i32_ty.const_int(idx.try_into().unwrap(), false)],
701                            "",
702                        ))
703                    };
704                    let ptr = err!(builder.build_pointer_cast(ptr, intrinsics.ptr_ty, ""));
705                    err_nt!(builder.build_load(type_to_llvm(intrinsics, *ty)?, ptr, ""))
706                })
707                .collect::<Result<Vec<_>, CompileError>>()?;
708
709            if self.abi.is_sret(func_sig)? {
710                let sret = trampoline_func
711                    .get_first_param()
712                    .unwrap()
713                    .into_pointer_value();
714
715                let basic_types: Vec<_> = func_sig
716                    .results()
717                    .iter()
718                    .map(|&ty| type_to_llvm(intrinsics, ty))
719                    .collect::<Result<_, _>>()?;
720                let mut struct_value = context.struct_type(&basic_types, false).get_undef();
721
722                for (idx, value) in results.iter().enumerate() {
723                    let value = err!(builder.build_bit_cast(
724                        *value,
725                        type_to_llvm(intrinsics, func_sig.results()[idx])?,
726                        "",
727                    ));
728                    struct_value =
729                        err!(builder.build_insert_value(struct_value, value, idx as u32, ""))
730                            .into_struct_value();
731                }
732                err!(builder.build_store(sret, struct_value));
733                err!(builder.build_return(None));
734            } else {
735                err!(
736                    builder.build_return(Some(&self.abi.pack_values_for_register_return(
737                        intrinsics,
738                        &builder,
739                        results.as_slice(),
740                        func_sig,
741                        &trampoline_func.get_type(),
742                    )?))
743                );
744            }
745        }
746
747        Ok(())
748    }
749}
wasmer_compiler_llvm/translator/trampoline.rs

wasmer_compiler_llvm/translator/
trampoline.rs
