wasmer_compiler_llvm/translator/
trampoline.rs

1// TODO: Remove
2#![allow(unused)]
3
4use crate::{
5    abi::{Abi, get_abi},
6    config::LLVM,
7    error::{err, err_nt},
8    object_file::{RkyvCompiledFunction, load_object_file},
9    translator::intrinsics::{Intrinsics, type_to_llvm},
10};
11use inkwell::{
12    AddressSpace, DLLStorageClass,
13    attributes::{Attribute, AttributeLoc},
14    context::Context,
15    module::{Linkage, Module},
16    passes::PassBuilderOptions,
17    targets::{FileType, TargetMachine},
18    types::FunctionType,
19    values::{BasicMetadataValueEnum, FunctionValue},
20};
21use std::{cmp, convert::TryInto, path::Path};
22use target_lexicon::{BinaryFormat, Triple};
23use wasmer_compiler::{
24    misc::{CompiledFunctionExt, CompiledKind},
25    types::{
26        function::{CompiledFunctionBody, FunctionBody},
27        module::CompileModuleInfo,
28        relocation::{Relocation, RelocationTarget},
29        section::{CustomSection, CustomSectionProtection, SectionBody, SectionIndex},
30    },
31};
32use wasmer_types::{
33    CompileError, FunctionIndex, FunctionType as FuncType, LocalFunctionIndex, MemoryIndex,
34    entity::PrimaryMap,
35};
36use wasmer_vm::MemoryStyle;
37
38pub struct FuncTrampoline {
39    ctx: Context,
40    target_machine: TargetMachine,
41    target_triple: Triple,
42    abi: Box<dyn Abi>,
43    binary_fmt: BinaryFormat,
44    func_section: String,
45}
46
47const FUNCTION_SECTION_ELF: &str = "__TEXT,wasmer_trmpl"; // Needs to be between 1 and 16 chars
48const FUNCTION_SECTION_MACHO: &str = "wasmer_trmpl"; // Needs to be between 1 and 16 chars
49
50fn enable_m0_optimization(compile_info: &CompileModuleInfo) -> bool {
51    compile_info
52        .memory_styles
53        .get(MemoryIndex::from_u32(0))
54        .is_some_and(|memory| matches!(memory, MemoryStyle::Static { .. }))
55}
56
57impl FuncTrampoline {
58    pub fn new(
59        target_machine: TargetMachine,
60        target_triple: Triple,
61        binary_fmt: BinaryFormat,
62    ) -> Result<Self, CompileError> {
63        let abi = get_abi(&target_machine);
64        Ok(Self {
65            ctx: Context::create(),
66            target_machine,
67            target_triple,
68            abi,
69            func_section: match binary_fmt {
70                BinaryFormat::Elf => FUNCTION_SECTION_ELF.to_string(),
71                BinaryFormat::Macho => FUNCTION_SECTION_MACHO.to_string(),
72                _ => {
73                    return Err(CompileError::UnsupportedTarget(format!(
74                        "Unsupported binary format: {binary_fmt:?}",
75                    )));
76                }
77            },
78            binary_fmt,
79        })
80    }
81
82    pub fn trampoline_to_module(
83        &self,
84        ty: &FuncType,
85        config: &LLVM,
86        function: &CompiledKind,
87        compile_info: &CompileModuleInfo,
88    ) -> Result<Module<'_>, CompileError> {
89        // The function type, used for the callbacks.
90        let module = self.ctx.create_module("");
91        let target_machine = &self.target_machine;
92        let target_triple = target_machine.get_triple();
93        let target_data: inkwell::targets::TargetData = target_machine.get_target_data();
94        module.set_triple(&target_triple);
95        module.set_data_layout(&target_data.get_data_layout());
96        let intrinsics = Intrinsics::declare(
97            &module,
98            &self.ctx,
99            &target_data,
100            &self.target_triple,
101            &self.binary_fmt,
102        );
103
104        let m0_is_enabled = enable_m0_optimization(compile_info);
105        let (callee_ty, callee_attrs) =
106            self.abi
107                .func_type_to_llvm(&self.ctx, &intrinsics, None, ty, m0_is_enabled)?;
108        let trampoline_ty = intrinsics.void_ty.fn_type(
109            &[
110                intrinsics.ptr_ty.into(), // vmctx ptr
111                intrinsics.ptr_ty.into(), // callee function address
112                intrinsics.ptr_ty.into(), // in/out values ptr
113            ],
114            false,
115        );
116
117        let trampoline_func = module.add_function(
118            &function.linkage_name(),
119            trampoline_ty,
120            Some(Linkage::External),
121        );
122        if !cfg!(feature = "experimental-artifact") {
123            trampoline_func
124                .as_global_value()
125                .set_section(Some(&self.func_section));
126        }
127        trampoline_func
128            .as_global_value()
129            .set_linkage(Linkage::DLLExport);
130        trampoline_func
131            .as_global_value()
132            .set_dll_storage_class(DLLStorageClass::Export);
133        // We intentionally mark this function as no-unwind; otherwise, stack unwinding could cross the
134        // trampoline boundary and cause Rust to complain about a foreign exception being thrown.
135        trampoline_func.add_attribute(AttributeLoc::Function, intrinsics.nounwind);
136        trampoline_func.add_attribute(AttributeLoc::Function, intrinsics.frame_pointer);
137        self.generate_trampoline(
138            config,
139            compile_info,
140            trampoline_func,
141            ty,
142            callee_ty,
143            &callee_attrs,
144            &self.ctx,
145            &intrinsics,
146        )?;
147
148        if let Some(ref callbacks) = config.callbacks {
149            callbacks.preopt_ir(function, &compile_info.module.hash_string(), &module);
150        }
151
152        let mut passes = vec![];
153
154        if config.enable_verifier {
155            passes.push("verify");
156        }
157
158        passes.push("instcombine");
159        module
160            .run_passes(
161                passes.join(",").as_str(),
162                target_machine,
163                PassBuilderOptions::create(),
164            )
165            .unwrap();
166
167        if let Some(ref callbacks) = config.callbacks {
168            callbacks.postopt_ir(function, &compile_info.module.hash_string(), &module);
169        }
170        Ok(module)
171    }
172
173    pub fn trampoline(
174        &self,
175        ty: &FuncType,
176        config: &LLVM,
177        function: &CompiledKind,
178        compile_info: &CompileModuleInfo,
179        build_directory: &Path,
180    ) -> Result<CompiledFunctionBody, CompileError> {
181        let module = self.trampoline_to_module(ty, config, function, compile_info)?;
182        let target_machine = &self.target_machine;
183
184        let memory_buffer = target_machine
185            .write_to_memory_buffer(&module, FileType::Object)
186            .unwrap();
187
188        if let Some(ref callbacks) = config.callbacks {
189            let module_hash = compile_info.module.hash_string();
190            callbacks.obj_memory_buffer(function, &module_hash, &memory_buffer);
191            let asm_buffer = target_machine
192                .write_to_memory_buffer(&module, FileType::Assembly)
193                .unwrap();
194            callbacks.asm_memory_buffer(function, &module_hash, &asm_buffer);
195        }
196
197        if cfg!(feature = "experimental-artifact") {
198            let object_path = build_directory.to_path_buf().join(function.linkage_name());
199            std::fs::write(&object_path, memory_buffer.as_slice()).map_err(|e| {
200                CompileError::Codegen(format!("Cannot save LLVM object file for trampoline: {e}"))
201            })?;
202
203            Ok(CompiledFunctionBody::Elf(object_path))
204        } else {
205            // Use a dummy function index to detect relocations against the trampoline
206            // function's address, which shouldn't exist and are not supported.
207            // Note, we just drop all custom sections, and verify that the function
208            // body itself has no relocations at all. This value should never be
209            // touched at all. However, it is set up so that if we do touch it (maybe
210            // due to someone changing the code later on), it'll explode, which is desirable!
211            let dummy_reloc_target =
212                RelocationTarget::DynamicTrampoline(FunctionIndex::from_u32(u32::MAX - 1));
213
214            // Note: we don't count .gcc_except_table here because native-to-wasm
215            // trampolines are not supposed to generate any LSDA sections. We *want* them
216            // to terminate libunwind's stack searches.
217            let RkyvCompiledFunction {
218                compiled_function,
219                custom_sections,
220                eh_frame_section_indices,
221                mut compact_unwind_section_indices,
222                ..
223            } = load_object_file(
224                memory_buffer.as_slice(),
225                &self.func_section,
226                dummy_reloc_target,
227                |name: &str| {
228                    Err(CompileError::Codegen(format!(
229                        "trampoline generation produced reference to unknown function {name}",
230                    )))
231                },
232                self.binary_fmt,
233                &self.target_triple,
234            )?;
235            let mut all_sections_are_eh_sections = true;
236            let mut unwind_section_indices = eh_frame_section_indices;
237            unwind_section_indices.append(&mut compact_unwind_section_indices);
238            if unwind_section_indices.len() != custom_sections.len() {
239                all_sections_are_eh_sections = false;
240            } else {
241                unwind_section_indices.sort_unstable();
242                for (idx, section_idx) in unwind_section_indices.iter().enumerate() {
243                    if idx as u32 != section_idx.as_u32() {
244                        all_sections_are_eh_sections = false;
245                        break;
246                    }
247                }
248            }
249            if !all_sections_are_eh_sections {
250                return Err(CompileError::Codegen(
251                    "trampoline generation produced non-eh custom sections".into(),
252                ));
253            }
254            if !compiled_function.relocations.is_empty() {
255                return Err(CompileError::Codegen(
256                    "trampoline generation produced relocations".into(),
257                ));
258            }
259            // Ignore CompiledFunctionFrameInfo. Extra frame info isn't a problem.
260
261            Ok(CompiledFunctionBody::Rkyv(FunctionBody {
262                body: compiled_function.body.body,
263                unwind_info: compiled_function.body.unwind_info,
264            }))
265        }
266    }
267
268    pub fn dynamic_trampoline_to_module(
269        &self,
270        ty: &FuncType,
271        config: &LLVM,
272        function: &CompiledKind,
273        module_hash: &Option<String>,
274    ) -> Result<Module<'_>, CompileError> {
275        // The function type, used for the callbacks
276        let module = self.ctx.create_module("");
277        let target_machine = &self.target_machine;
278        let target_data = target_machine.get_target_data();
279        let target_triple = target_machine.get_triple();
280        module.set_triple(&target_triple);
281        module.set_data_layout(&target_data.get_data_layout());
282        let intrinsics = Intrinsics::declare(
283            &module,
284            &self.ctx,
285            &target_data,
286            &self.target_triple,
287            &self.binary_fmt,
288        );
289
290        let (trampoline_ty, trampoline_attrs) =
291            self.abi
292                .func_type_to_llvm(&self.ctx, &intrinsics, None, ty, false)?;
293        let trampoline_func = module.add_function(
294            &function.linkage_name(),
295            trampoline_ty,
296            Some(Linkage::External),
297        );
298        trampoline_func.set_personality_function(intrinsics.personality);
299        trampoline_func.add_attribute(AttributeLoc::Function, intrinsics.frame_pointer);
300        for (attr, attr_loc) in trampoline_attrs {
301            trampoline_func.add_attribute(attr_loc, attr);
302        }
303        if !cfg!(feature = "experimental-artifact") {
304            trampoline_func
305                .as_global_value()
306                .set_section(Some(&self.func_section));
307        }
308        trampoline_func
309            .as_global_value()
310            .set_linkage(Linkage::DLLExport);
311        trampoline_func
312            .as_global_value()
313            .set_dll_storage_class(DLLStorageClass::Export);
314        self.generate_dynamic_trampoline(trampoline_func, ty, &self.ctx, &intrinsics)?;
315
316        if let Some(ref callbacks) = config.callbacks {
317            callbacks.preopt_ir(function, module_hash, &module);
318        }
319
320        let mut passes = vec![];
321
322        if config.enable_verifier {
323            passes.push("verify");
324        }
325
326        passes.push("early-cse");
327        module
328            .run_passes(
329                passes.join(",").as_str(),
330                target_machine,
331                PassBuilderOptions::create(),
332            )
333            .unwrap();
334
335        if let Some(ref callbacks) = config.callbacks {
336            callbacks.postopt_ir(function, module_hash, &module);
337        }
338
339        Ok(module)
340    }
341
342    #[allow(clippy::too_many_arguments)]
343    pub fn dynamic_trampoline(
344        &self,
345        ty: &FuncType,
346        config: &LLVM,
347        function: &CompiledKind,
348        dynamic_trampoline_index: u32,
349        final_module_custom_sections: &mut PrimaryMap<SectionIndex, CustomSection>,
350        eh_frame_section_bytes: &mut Vec<u8>,
351        eh_frame_section_relocations: &mut Vec<Relocation>,
352        compact_unwind_section_bytes: &mut Vec<u8>,
353        compact_unwind_section_relocations: &mut Vec<Relocation>,
354        module_hash: &Option<String>,
355        build_directory: &Path,
356    ) -> Result<CompiledFunctionBody, CompileError> {
357        let target_machine = &self.target_machine;
358
359        let module = self.dynamic_trampoline_to_module(ty, config, function, module_hash)?;
360
361        let memory_buffer = target_machine
362            .write_to_memory_buffer(&module, FileType::Object)
363            .unwrap();
364
365        if let Some(ref callbacks) = config.callbacks {
366            callbacks.obj_memory_buffer(function, module_hash, &memory_buffer);
367            let asm_buffer = target_machine
368                .write_to_memory_buffer(&module, FileType::Assembly)
369                .unwrap();
370            callbacks.asm_memory_buffer(function, module_hash, &asm_buffer)
371        }
372
373        if cfg!(feature = "experimental-artifact") {
374            let object_path = build_directory.to_path_buf().join(function.linkage_name());
375            std::fs::write(&object_path, memory_buffer.as_slice()).map_err(|e| {
376                CompileError::Codegen(format!(
377                    "Cannot save LLVM object file for dynamic trampoline: {e}"
378                ))
379            })?;
380
381            Ok(CompiledFunctionBody::Elf(object_path))
382        } else {
383            let RkyvCompiledFunction {
384                compiled_function,
385                custom_sections,
386                eh_frame_section_indices,
387                compact_unwind_section_indices,
388                gcc_except_table_section_indices,
389                data_dw_ref_personality_section_indices,
390            } = load_object_file(
391                memory_buffer.as_slice(),
392                &self.func_section,
393                RelocationTarget::DynamicTrampoline(FunctionIndex::from_u32(
394                    dynamic_trampoline_index,
395                )),
396                |name: &str| {
397                    Err(CompileError::Codegen(format!(
398                        "trampoline generation produced reference to unknown function {name}",
399                    )))
400                },
401                self.binary_fmt,
402                &self.target_triple,
403            )?;
404
405            if !compiled_function.relocations.is_empty() {
406                return Err(CompileError::Codegen(
407                    "trampoline generation produced relocations".into(),
408                ));
409            }
410            // Ignore CompiledFunctionFrameInfo. Extra frame info isn't a problem.
411
412            // Also append EH-related sections to the final module, since we expect
413            // dynamic trampolines to participate in unwinding
414            {
415                let first_section = final_module_custom_sections.len() as u32;
416                for (section_index, mut custom_section) in custom_sections.into_iter() {
417                    for reloc in &mut custom_section.relocations {
418                        if let RelocationTarget::CustomSection(index) = reloc.reloc_target {
419                            reloc.reloc_target = RelocationTarget::CustomSection(
420                                SectionIndex::from_u32(first_section + index.as_u32()),
421                            )
422                        }
423
424                        if reloc.kind.needs_got() {
425                            return Err(CompileError::Codegen(
426                                "trampoline generation produced GOT relocation".into(),
427                            ));
428                        }
429                    }
430
431                    if eh_frame_section_indices.contains(&section_index) {
432                        let offset = eh_frame_section_bytes.len() as u32;
433                        for reloc in &mut custom_section.relocations {
434                            reloc.offset += offset;
435                        }
436                        eh_frame_section_bytes.extend_from_slice(custom_section.bytes.as_slice());
437                        // Terminate the eh_frame info with a zero-length CIE.
438                        eh_frame_section_bytes.extend_from_slice(&[0, 0, 0, 0]);
439                        eh_frame_section_relocations.extend(custom_section.relocations);
440                        // TODO: we do this to keep the count right, remove it.
441                        final_module_custom_sections.push(CustomSection {
442                            protection: CustomSectionProtection::Read,
443                            alignment: None,
444                            bytes: SectionBody::new_with_vec(vec![]),
445                            relocations: vec![],
446                        });
447                    } else if compact_unwind_section_indices.contains(&section_index) {
448                        let offset = compact_unwind_section_bytes.len() as u32;
449                        for reloc in &mut custom_section.relocations {
450                            reloc.offset += offset;
451                        }
452                        compact_unwind_section_bytes
453                            .extend_from_slice(custom_section.bytes.as_slice());
454                        compact_unwind_section_relocations.extend(custom_section.relocations);
455                        // TODO: we do this to keep the count right, remove it.
456                        final_module_custom_sections.push(CustomSection {
457                            protection: CustomSectionProtection::Read,
458                            alignment: None,
459                            bytes: SectionBody::new_with_vec(vec![]),
460                            relocations: vec![],
461                        });
462                    } else if gcc_except_table_section_indices.contains(&section_index)
463                        || data_dw_ref_personality_section_indices.contains(&section_index)
464                    {
465                        final_module_custom_sections.push(custom_section);
466                    } else {
467                        return Err(CompileError::Codegen(
468                            "trampoline generation produced non-eh custom sections".into(),
469                        ));
470                    }
471                }
472            }
473
474            Ok(CompiledFunctionBody::Rkyv(FunctionBody {
475                body: compiled_function.body.body,
476                unwind_info: compiled_function.body.unwind_info,
477            }))
478        }
479    }
480
481    #[allow(clippy::too_many_arguments)]
482    fn generate_trampoline<'ctx>(
483        &self,
484        config: &LLVM,
485        compile_info: &CompileModuleInfo,
486        trampoline_func: FunctionValue,
487        func_sig: &FuncType,
488        llvm_func_type: FunctionType,
489        func_attrs: &[(Attribute, AttributeLoc)],
490        context: &'ctx Context,
491        intrinsics: &Intrinsics<'ctx>,
492    ) -> Result<(), CompileError> {
493        let entry_block = context.append_basic_block(trampoline_func, "entry");
494        let builder = context.create_builder();
495        builder.position_at_end(entry_block);
496
497        let (callee_vmctx_ptr, func_ptr, args_rets_ptr) =
498            match *trampoline_func.get_params().as_slice() {
499                [callee_vmctx_ptr, func_ptr, args_rets_ptr] => (
500                    callee_vmctx_ptr,
501                    func_ptr.into_pointer_value(),
502                    args_rets_ptr.into_pointer_value(),
503                ),
504                _ => {
505                    return Err(CompileError::Codegen(
506                        "trampoline function unimplemented".to_string(),
507                    ));
508                }
509            };
510        func_ptr.set_name("func_ptr");
511
512        let mut args_vec = Vec::with_capacity(func_sig.params().len() + 3);
513
514        if self.abi.is_sret(func_sig)? {
515            let basic_types: Vec<_> = func_sig
516                .results()
517                .iter()
518                .map(|&ty| type_to_llvm(intrinsics, ty))
519                .collect::<Result<_, _>>()?;
520
521            let sret_ty = context.struct_type(&basic_types, false);
522            args_vec.push(err!(builder.build_alloca(sret_ty, "sret")).into());
523        }
524
525        callee_vmctx_ptr.set_name("vmctx");
526        args_vec.push(callee_vmctx_ptr.into());
527
528        if enable_m0_optimization(compile_info) {
529            let wasm_module = &compile_info.module;
530            let memory_styles = &compile_info.memory_styles;
531            let callee_vmctx_ptr_value = callee_vmctx_ptr.into_pointer_value();
532            let offsets = wasmer_vm::VMOffsets::new(8, wasm_module);
533
534            // load mem
535            let memory_index = wasmer_types::MemoryIndex::from_u32(0);
536            let memory_definition_ptr = if let Some(local_memory_index) =
537                wasm_module.local_memory_index(memory_index)
538            {
539                let offset = offsets.vmctx_vmmemory_definition(local_memory_index);
540                let offset = intrinsics.i32_ty.const_int(offset.into(), false);
541                unsafe {
542                    err!(builder.build_gep(intrinsics.i8_ty, callee_vmctx_ptr_value, &[offset], ""))
543                }
544            } else {
545                let offset = offsets.vmctx_vmmemory_import(memory_index);
546                let offset = intrinsics.i32_ty.const_int(offset.into(), false);
547                let memory_definition_ptr_ptr = unsafe {
548                    err!(builder.build_gep(intrinsics.i8_ty, callee_vmctx_ptr_value, &[offset], ""))
549                };
550                let memory_definition_ptr_ptr =
551                    err!(builder.build_bit_cast(memory_definition_ptr_ptr, intrinsics.ptr_ty, "",))
552                        .into_pointer_value();
553
554                err!(builder.build_load(intrinsics.ptr_ty, memory_definition_ptr_ptr, ""))
555                    .into_pointer_value()
556            };
557            let memory_definition_ptr =
558                err!(builder.build_bit_cast(memory_definition_ptr, intrinsics.ptr_ty, "",))
559                    .into_pointer_value();
560            let base_ptr = err!(builder.build_struct_gep(
561                intrinsics.vmmemory_definition_ty,
562                memory_definition_ptr,
563                intrinsics.vmmemory_definition_base_element,
564                "",
565            ));
566
567            let memory_style = &memory_styles[memory_index];
568            let base_ptr = if let MemoryStyle::Dynamic { .. } = memory_style {
569                base_ptr
570            } else {
571                err!(builder.build_load(intrinsics.ptr_ty, base_ptr, "")).into_pointer_value()
572            };
573
574            base_ptr.set_name("trmpl_m0_base_ptr");
575
576            args_vec.push(base_ptr.into());
577        }
578
579        for (i, param_ty) in func_sig.params().iter().enumerate() {
580            let index = intrinsics.i32_ty.const_int(i as _, false);
581            let item_pointer = unsafe {
582                err!(builder.build_in_bounds_gep(
583                    intrinsics.i128_ty,
584                    args_rets_ptr,
585                    &[index],
586                    "arg_ptr"
587                ))
588            };
589
590            let casted_type = type_to_llvm(intrinsics, *param_ty)?;
591
592            let typed_item_pointer = err!(builder.build_pointer_cast(
593                item_pointer,
594                intrinsics.ptr_ty,
595                "typed_arg_pointer"
596            ));
597
598            let arg = err!(builder.build_load(casted_type, typed_item_pointer, "arg"));
599            args_vec.push(arg.into());
600        }
601
602        let call_site = err!(builder.build_indirect_call(
603            llvm_func_type,
604            func_ptr,
605            args_vec.as_slice(),
606            "call"
607        ));
608        for (attr, attr_loc) in func_attrs {
609            call_site.add_attribute(*attr_loc, *attr);
610        }
611
612        let rets = self
613            .abi
614            .rets_from_call(&builder, intrinsics, call_site, func_sig)?;
615        for (idx, v) in rets.into_iter().enumerate() {
616            let ptr = unsafe {
617                err!(builder.build_gep(
618                    intrinsics.i128_ty,
619                    args_rets_ptr,
620                    &[intrinsics.i32_ty.const_int(idx as u64, false)],
621                    "",
622                ))
623            };
624            let ptr = err!(builder.build_pointer_cast(
625                ptr,
626                self.ctx.ptr_type(AddressSpace::default()),
627                ""
628            ));
629            err!(builder.build_store(ptr, v));
630        }
631
632        err!(builder.build_return(None));
633        Ok(())
634    }
635
636    fn generate_dynamic_trampoline<'ctx>(
637        &self,
638        trampoline_func: FunctionValue,
639        func_sig: &FuncType,
640        context: &'ctx Context,
641        intrinsics: &Intrinsics<'ctx>,
642    ) -> Result<(), CompileError> {
643        let entry_block = context.append_basic_block(trampoline_func, "entry");
644        let builder = context.create_builder();
645        builder.position_at_end(entry_block);
646
647        // Allocate stack space for the params and results.
648        let values = err!(builder.build_alloca(
649            intrinsics.i128_ty.array_type(cmp::max(
650                func_sig.params().len().try_into().unwrap(),
651                func_sig.results().len().try_into().unwrap(),
652            )),
653            "",
654        ));
655
656        // Copy params to 'values'.
657        let first_user_param = if self.abi.is_sret(func_sig)? { 2 } else { 1 };
658        for i in 0..func_sig.params().len() {
659            let ptr = unsafe {
660                err!(builder.build_in_bounds_gep(
661                    intrinsics.i128_ty,
662                    values,
663                    &[intrinsics.i32_ty.const_int(i.try_into().unwrap(), false)],
664                    "args",
665                ))
666            };
667            let ptr = err!(builder.build_bit_cast(ptr, intrinsics.ptr_ty, "")).into_pointer_value();
668            err!(
669                builder.build_store(
670                    ptr,
671                    trampoline_func
672                        .get_nth_param(i as u32 + first_user_param)
673                        .unwrap(),
674                )
675            );
676        }
677
678        let callee_ptr_ty = intrinsics.void_ty.fn_type(
679            &[
680                intrinsics.ptr_ty.into(), // vmctx ptr
681                intrinsics.ptr_ty.into(), // in/out values ptr
682            ],
683            false,
684        );
685        let vmctx = self.abi.get_vmctx_ptr_param(&trampoline_func);
686        let callee_ty =
687            err!(builder.build_bit_cast(vmctx, self.ctx.ptr_type(AddressSpace::default()), ""));
688        let callee =
689            err!(builder.build_load(intrinsics.ptr_ty, callee_ty.into_pointer_value(), ""))
690                .into_pointer_value();
691        callee.set_name("func_ptr");
692
693        let values_ptr = err!(builder.build_pointer_cast(values, intrinsics.ptr_ty, ""));
694        values_ptr.set_name("value_ptr");
695        err!(builder.build_indirect_call(
696            callee_ptr_ty,
697            callee,
698            &[vmctx.into(), values_ptr.into()],
699            "",
700        ));
701
702        if func_sig.results().is_empty() {
703            err!(builder.build_return(None));
704        } else {
705            let results = func_sig
706                .results()
707                .iter()
708                .enumerate()
709                .map(|(idx, ty)| {
710                    let ptr = unsafe {
711                        err!(builder.build_gep(
712                            intrinsics.i128_ty,
713                            values,
714                            &[intrinsics.i32_ty.const_int(idx.try_into().unwrap(), false)],
715                            "",
716                        ))
717                    };
718                    let ptr = err!(builder.build_pointer_cast(ptr, intrinsics.ptr_ty, ""));
719                    err_nt!(builder.build_load(type_to_llvm(intrinsics, *ty)?, ptr, ""))
720                })
721                .collect::<Result<Vec<_>, CompileError>>()?;
722
723            if self.abi.is_sret(func_sig)? {
724                let sret = trampoline_func
725                    .get_first_param()
726                    .unwrap()
727                    .into_pointer_value();
728
729                let basic_types: Vec<_> = func_sig
730                    .results()
731                    .iter()
732                    .map(|&ty| type_to_llvm(intrinsics, ty))
733                    .collect::<Result<_, _>>()?;
734                let mut struct_value = context.struct_type(&basic_types, false).get_undef();
735
736                for (idx, value) in results.iter().enumerate() {
737                    let value = err!(builder.build_bit_cast(
738                        *value,
739                        type_to_llvm(intrinsics, func_sig.results()[idx])?,
740                        "",
741                    ));
742                    struct_value =
743                        err!(builder.build_insert_value(struct_value, value, idx as u32, ""))
744                            .into_struct_value();
745                }
746                err!(builder.build_store(sret, struct_value));
747                err!(builder.build_return(None));
748            } else {
749                err!(
750                    builder.build_return(Some(&self.abi.pack_values_for_register_return(
751                        intrinsics,
752                        &builder,
753                        results.as_slice(),
754                        &trampoline_func.get_type(),
755                    )?))
756                );
757            }
758        }
759
760        Ok(())
761    }
762}
wasmer_compiler_llvm/translator/trampoline.rs

wasmer_compiler_llvm/translator/
trampoline.rs
