wasmer_compiler_llvm/translator/
trampoline.rs

1// TODO: Remove
2#![allow(unused)]
3
4use crate::{
5    abi::{Abi, get_abi},
6    config::LLVM,
7    error::{err, err_nt},
8    object_file::{CompiledFunction, load_object_file},
9    translator::intrinsics::{Intrinsics, type_to_llvm},
10};
11use inkwell::{
12    AddressSpace, DLLStorageClass,
13    attributes::{Attribute, AttributeLoc},
14    context::Context,
15    module::{Linkage, Module},
16    passes::PassBuilderOptions,
17    targets::{FileType, TargetMachine},
18    types::FunctionType,
19    values::{BasicMetadataValueEnum, FunctionValue},
20};
21use std::{cmp, convert::TryInto};
22use target_lexicon::{BinaryFormat, Triple};
23use wasmer_compiler::{
24    misc::CompiledKind,
25    types::{
26        function::FunctionBody,
27        module::CompileModuleInfo,
28        relocation::{Relocation, RelocationTarget},
29        section::{CustomSection, CustomSectionProtection, SectionBody, SectionIndex},
30    },
31};
32use wasmer_types::{
33    CompileError, FunctionIndex, FunctionType as FuncType, LocalFunctionIndex, MemoryIndex,
34    entity::PrimaryMap,
35};
36use wasmer_vm::MemoryStyle;
37
38pub struct FuncTrampoline {
39    ctx: Context,
40    target_machine: TargetMachine,
41    target_triple: Triple,
42    abi: Box<dyn Abi>,
43    binary_fmt: BinaryFormat,
44    func_section: String,
45}
46
47const FUNCTION_SECTION_ELF: &str = "__TEXT,wasmer_trmpl"; // Needs to be between 1 and 16 chars
48const FUNCTION_SECTION_MACHO: &str = "wasmer_trmpl"; // Needs to be between 1 and 16 chars
49
50fn enable_m0_optimization(compile_info: &CompileModuleInfo) -> bool {
51    compile_info
52        .memory_styles
53        .get(MemoryIndex::from_u32(0))
54        .is_some_and(|memory| matches!(memory, MemoryStyle::Static { .. }))
55}
56
57impl FuncTrampoline {
58    pub fn new(
59        target_machine: TargetMachine,
60        target_triple: Triple,
61        binary_fmt: BinaryFormat,
62    ) -> Result<Self, CompileError> {
63        let abi = get_abi(&target_machine);
64        Ok(Self {
65            ctx: Context::create(),
66            target_machine,
67            target_triple,
68            abi,
69            func_section: match binary_fmt {
70                BinaryFormat::Elf => FUNCTION_SECTION_ELF.to_string(),
71                BinaryFormat::Macho => FUNCTION_SECTION_MACHO.to_string(),
72                _ => {
73                    return Err(CompileError::UnsupportedTarget(format!(
74                        "Unsupported binary format: {binary_fmt:?}",
75                    )));
76                }
77            },
78            binary_fmt,
79        })
80    }
81
82    pub fn trampoline_to_module(
83        &self,
84        ty: &FuncType,
85        config: &LLVM,
86        name: &str,
87        compile_info: &CompileModuleInfo,
88    ) -> Result<Module<'_>, CompileError> {
89        // The function type, used for the callbacks.
90        let function = CompiledKind::FunctionCallTrampoline(ty.clone());
91        let module = self.ctx.create_module("");
92        let target_machine = &self.target_machine;
93        let target_triple = target_machine.get_triple();
94        let target_data: inkwell::targets::TargetData = target_machine.get_target_data();
95        module.set_triple(&target_triple);
96        module.set_data_layout(&target_data.get_data_layout());
97        let intrinsics = Intrinsics::declare(
98            &module,
99            &self.ctx,
100            &target_data,
101            &self.target_triple,
102            &self.binary_fmt,
103        );
104
105        let m0_is_enabled = enable_m0_optimization(compile_info);
106        let (callee_ty, callee_attrs) =
107            self.abi
108                .func_type_to_llvm(&self.ctx, &intrinsics, None, ty, m0_is_enabled)?;
109        let trampoline_ty = intrinsics.void_ty.fn_type(
110            &[
111                intrinsics.ptr_ty.into(), // vmctx ptr
112                intrinsics.ptr_ty.into(), // callee function address
113                intrinsics.ptr_ty.into(), // in/out values ptr
114            ],
115            false,
116        );
117
118        let trampoline_func = module.add_function(name, trampoline_ty, Some(Linkage::External));
119        trampoline_func
120            .as_global_value()
121            .set_section(Some(&self.func_section));
122        trampoline_func
123            .as_global_value()
124            .set_linkage(Linkage::DLLExport);
125        trampoline_func
126            .as_global_value()
127            .set_dll_storage_class(DLLStorageClass::Export);
128        trampoline_func.add_attribute(AttributeLoc::Function, intrinsics.uwtable);
129        trampoline_func.add_attribute(AttributeLoc::Function, intrinsics.frame_pointer);
130        self.generate_trampoline(
131            config,
132            compile_info,
133            trampoline_func,
134            ty,
135            callee_ty,
136            &callee_attrs,
137            &self.ctx,
138            &intrinsics,
139        )?;
140
141        if let Some(ref callbacks) = config.callbacks {
142            callbacks.preopt_ir(&function, &compile_info.module.hash_string(), &module);
143        }
144
145        let mut passes = vec![];
146
147        if config.enable_verifier {
148            passes.push("verify");
149        }
150
151        passes.push("instcombine");
152        module
153            .run_passes(
154                passes.join(",").as_str(),
155                target_machine,
156                PassBuilderOptions::create(),
157            )
158            .unwrap();
159
160        if let Some(ref callbacks) = config.callbacks {
161            callbacks.postopt_ir(&function, &compile_info.module.hash_string(), &module);
162        }
163        Ok(module)
164    }
165
166    pub fn trampoline(
167        &self,
168        ty: &FuncType,
169        config: &LLVM,
170        name: &str,
171        compile_info: &CompileModuleInfo,
172    ) -> Result<FunctionBody, CompileError> {
173        let module = self.trampoline_to_module(ty, config, name, compile_info)?;
174        let function = CompiledKind::FunctionCallTrampoline(ty.clone());
175        let target_machine = &self.target_machine;
176
177        let memory_buffer = target_machine
178            .write_to_memory_buffer(&module, FileType::Object)
179            .unwrap();
180
181        if let Some(ref callbacks) = config.callbacks {
182            let module_hash = compile_info.module.hash_string();
183            callbacks.obj_memory_buffer(&function, &module_hash, &memory_buffer);
184            let asm_buffer = target_machine
185                .write_to_memory_buffer(&module, FileType::Assembly)
186                .unwrap();
187            callbacks.asm_memory_buffer(&function, &module_hash, &asm_buffer);
188        }
189
190        let mem_buf_slice = memory_buffer.as_slice();
191
192        // Use a dummy function index to detect relocations against the trampoline
193        // function's address, which shouldn't exist and are not supported.
194        // Note, we just drop all custom sections, and verify that the function
195        // body itself has no relocations at all. This value should never be
196        // touched at all. However, it is set up so that if we do touch it (maybe
197        // due to someone changing the code later on), it'll explode, which is desirable!
198        let dummy_reloc_target =
199            RelocationTarget::DynamicTrampoline(FunctionIndex::from_u32(u32::MAX - 1));
200
201        // Note: we don't count .gcc_except_table here because native-to-wasm
202        // trampolines are not supposed to generate any LSDA sections. We *want* them
203        // to terminate libunwind's stack searches.
204        let CompiledFunction {
205            compiled_function,
206            custom_sections,
207            eh_frame_section_indices,
208            mut compact_unwind_section_indices,
209            ..
210        } = load_object_file(
211            mem_buf_slice,
212            &self.func_section,
213            dummy_reloc_target,
214            |name: &str| {
215                Err(CompileError::Codegen(format!(
216                    "trampoline generation produced reference to unknown function {name}",
217                )))
218            },
219            self.binary_fmt,
220        )?;
221        let mut all_sections_are_eh_sections = true;
222        let mut unwind_section_indices = eh_frame_section_indices;
223        unwind_section_indices.append(&mut compact_unwind_section_indices);
224        if unwind_section_indices.len() != custom_sections.len() {
225            all_sections_are_eh_sections = false;
226        } else {
227            unwind_section_indices.sort_unstable();
228            for (idx, section_idx) in unwind_section_indices.iter().enumerate() {
229                if idx as u32 != section_idx.as_u32() {
230                    all_sections_are_eh_sections = false;
231                    break;
232                }
233            }
234        }
235        if !all_sections_are_eh_sections {
236            return Err(CompileError::Codegen(
237                "trampoline generation produced non-eh custom sections".into(),
238            ));
239        }
240        if !compiled_function.relocations.is_empty() {
241            return Err(CompileError::Codegen(
242                "trampoline generation produced relocations".into(),
243            ));
244        }
245        // Ignore CompiledFunctionFrameInfo. Extra frame info isn't a problem.
246
247        Ok(FunctionBody {
248            body: compiled_function.body.body,
249            unwind_info: compiled_function.body.unwind_info,
250        })
251    }
252
253    pub fn dynamic_trampoline_to_module(
254        &self,
255        ty: &FuncType,
256        config: &LLVM,
257        name: &str,
258        module_hash: &Option<String>,
259    ) -> Result<Module<'_>, CompileError> {
260        // The function type, used for the callbacks
261        let function = CompiledKind::DynamicFunctionTrampoline(ty.clone());
262        let module = self.ctx.create_module("");
263        let target_machine = &self.target_machine;
264        let target_data = target_machine.get_target_data();
265        let target_triple = target_machine.get_triple();
266        module.set_triple(&target_triple);
267        module.set_data_layout(&target_data.get_data_layout());
268        let intrinsics = Intrinsics::declare(
269            &module,
270            &self.ctx,
271            &target_data,
272            &self.target_triple,
273            &self.binary_fmt,
274        );
275
276        let (trampoline_ty, trampoline_attrs) =
277            self.abi
278                .func_type_to_llvm(&self.ctx, &intrinsics, None, ty, false)?;
279        let trampoline_func = module.add_function(name, trampoline_ty, Some(Linkage::External));
280        trampoline_func.set_personality_function(intrinsics.personality);
281        trampoline_func.add_attribute(AttributeLoc::Function, intrinsics.frame_pointer);
282        for (attr, attr_loc) in trampoline_attrs {
283            trampoline_func.add_attribute(attr_loc, attr);
284        }
285        trampoline_func
286            .as_global_value()
287            .set_section(Some(&self.func_section));
288        trampoline_func
289            .as_global_value()
290            .set_linkage(Linkage::DLLExport);
291        trampoline_func
292            .as_global_value()
293            .set_dll_storage_class(DLLStorageClass::Export);
294        self.generate_dynamic_trampoline(trampoline_func, ty, &self.ctx, &intrinsics)?;
295
296        if let Some(ref callbacks) = config.callbacks {
297            callbacks.preopt_ir(&function, module_hash, &module);
298        }
299
300        let mut passes = vec![];
301
302        if config.enable_verifier {
303            passes.push("verify");
304        }
305
306        passes.push("early-cse");
307        module
308            .run_passes(
309                passes.join(",").as_str(),
310                target_machine,
311                PassBuilderOptions::create(),
312            )
313            .unwrap();
314
315        if let Some(ref callbacks) = config.callbacks {
316            callbacks.postopt_ir(&function, module_hash, &module);
317        }
318
319        Ok(module)
320    }
321
322    #[allow(clippy::too_many_arguments)]
323    pub fn dynamic_trampoline(
324        &self,
325        ty: &FuncType,
326        config: &LLVM,
327        name: &str,
328        dynamic_trampoline_index: u32,
329        final_module_custom_sections: &mut PrimaryMap<SectionIndex, CustomSection>,
330        eh_frame_section_bytes: &mut Vec<u8>,
331        eh_frame_section_relocations: &mut Vec<Relocation>,
332        compact_unwind_section_bytes: &mut Vec<u8>,
333        compact_unwind_section_relocations: &mut Vec<Relocation>,
334        module_hash: &Option<String>,
335    ) -> Result<FunctionBody, CompileError> {
336        let function = CompiledKind::DynamicFunctionTrampoline(ty.clone());
337        let target_machine = &self.target_machine;
338
339        let module = self.dynamic_trampoline_to_module(ty, config, name, module_hash)?;
340
341        let memory_buffer = target_machine
342            .write_to_memory_buffer(&module, FileType::Object)
343            .unwrap();
344
345        if let Some(ref callbacks) = config.callbacks {
346            callbacks.obj_memory_buffer(&function, module_hash, &memory_buffer);
347            let asm_buffer = target_machine
348                .write_to_memory_buffer(&module, FileType::Assembly)
349                .unwrap();
350            callbacks.asm_memory_buffer(&function, module_hash, &asm_buffer)
351        }
352
353        let mem_buf_slice = memory_buffer.as_slice();
354        let CompiledFunction {
355            compiled_function,
356            custom_sections,
357            eh_frame_section_indices,
358            compact_unwind_section_indices,
359            gcc_except_table_section_indices,
360            data_dw_ref_personality_section_indices,
361        } = load_object_file(
362            mem_buf_slice,
363            &self.func_section,
364            RelocationTarget::DynamicTrampoline(FunctionIndex::from_u32(dynamic_trampoline_index)),
365            |name: &str| {
366                Err(CompileError::Codegen(format!(
367                    "trampoline generation produced reference to unknown function {name}",
368                )))
369            },
370            self.binary_fmt,
371        )?;
372
373        if !compiled_function.relocations.is_empty() {
374            return Err(CompileError::Codegen(
375                "trampoline generation produced relocations".into(),
376            ));
377        }
378        // Ignore CompiledFunctionFrameInfo. Extra frame info isn't a problem.
379
380        // Also append EH-related sections to the final module, since we expect
381        // dynamic trampolines to participate in unwinding
382        {
383            let first_section = final_module_custom_sections.len() as u32;
384            for (section_index, mut custom_section) in custom_sections.into_iter() {
385                for reloc in &mut custom_section.relocations {
386                    if let RelocationTarget::CustomSection(index) = reloc.reloc_target {
387                        reloc.reloc_target = RelocationTarget::CustomSection(
388                            SectionIndex::from_u32(first_section + index.as_u32()),
389                        )
390                    }
391
392                    if reloc.kind.needs_got() {
393                        return Err(CompileError::Codegen(
394                            "trampoline generation produced GOT relocation".into(),
395                        ));
396                    }
397                }
398
399                if eh_frame_section_indices.contains(&section_index) {
400                    let offset = eh_frame_section_bytes.len() as u32;
401                    for reloc in &mut custom_section.relocations {
402                        reloc.offset += offset;
403                    }
404                    eh_frame_section_bytes.extend_from_slice(custom_section.bytes.as_slice());
405                    // Terminate the eh_frame info with a zero-length CIE.
406                    eh_frame_section_bytes.extend_from_slice(&[0, 0, 0, 0]);
407                    eh_frame_section_relocations.extend(custom_section.relocations);
408                    // TODO: we do this to keep the count right, remove it.
409                    final_module_custom_sections.push(CustomSection {
410                        protection: CustomSectionProtection::Read,
411                        alignment: None,
412                        bytes: SectionBody::new_with_vec(vec![]),
413                        relocations: vec![],
414                    });
415                } else if compact_unwind_section_indices.contains(&section_index) {
416                    let offset = compact_unwind_section_bytes.len() as u32;
417                    for reloc in &mut custom_section.relocations {
418                        reloc.offset += offset;
419                    }
420                    compact_unwind_section_bytes.extend_from_slice(custom_section.bytes.as_slice());
421                    compact_unwind_section_relocations.extend(custom_section.relocations);
422                    // TODO: we do this to keep the count right, remove it.
423                    final_module_custom_sections.push(CustomSection {
424                        protection: CustomSectionProtection::Read,
425                        alignment: None,
426                        bytes: SectionBody::new_with_vec(vec![]),
427                        relocations: vec![],
428                    });
429                } else if gcc_except_table_section_indices.contains(&section_index)
430                    || data_dw_ref_personality_section_indices.contains(&section_index)
431                {
432                    final_module_custom_sections.push(custom_section);
433                } else {
434                    return Err(CompileError::Codegen(
435                        "trampoline generation produced non-eh custom sections".into(),
436                    ));
437                }
438            }
439        }
440
441        Ok(FunctionBody {
442            body: compiled_function.body.body,
443            unwind_info: compiled_function.body.unwind_info,
444        })
445    }
446
447    #[allow(clippy::too_many_arguments)]
448    fn generate_trampoline<'ctx>(
449        &self,
450        config: &LLVM,
451        compile_info: &CompileModuleInfo,
452        trampoline_func: FunctionValue,
453        func_sig: &FuncType,
454        llvm_func_type: FunctionType,
455        func_attrs: &[(Attribute, AttributeLoc)],
456        context: &'ctx Context,
457        intrinsics: &Intrinsics<'ctx>,
458    ) -> Result<(), CompileError> {
459        let entry_block = context.append_basic_block(trampoline_func, "entry");
460        let builder = context.create_builder();
461        builder.position_at_end(entry_block);
462
463        let (callee_vmctx_ptr, func_ptr, args_rets_ptr) =
464            match *trampoline_func.get_params().as_slice() {
465                [callee_vmctx_ptr, func_ptr, args_rets_ptr] => (
466                    callee_vmctx_ptr,
467                    func_ptr.into_pointer_value(),
468                    args_rets_ptr.into_pointer_value(),
469                ),
470                _ => {
471                    return Err(CompileError::Codegen(
472                        "trampoline function unimplemented".to_string(),
473                    ));
474                }
475            };
476        func_ptr.set_name("func_ptr");
477
478        let mut args_vec = Vec::with_capacity(func_sig.params().len() + 3);
479
480        if self.abi.is_sret(func_sig)? {
481            let basic_types: Vec<_> = func_sig
482                .results()
483                .iter()
484                .map(|&ty| type_to_llvm(intrinsics, ty))
485                .collect::<Result<_, _>>()?;
486
487            let sret_ty = context.struct_type(&basic_types, false);
488            args_vec.push(err!(builder.build_alloca(sret_ty, "sret")).into());
489        }
490
491        callee_vmctx_ptr.set_name("vmctx");
492        args_vec.push(callee_vmctx_ptr.into());
493
494        if enable_m0_optimization(compile_info) {
495            let wasm_module = &compile_info.module;
496            let memory_styles = &compile_info.memory_styles;
497            let callee_vmctx_ptr_value = callee_vmctx_ptr.into_pointer_value();
498            let offsets = wasmer_vm::VMOffsets::new(8, wasm_module);
499
500            // load mem
501            let memory_index = wasmer_types::MemoryIndex::from_u32(0);
502            let memory_definition_ptr = if let Some(local_memory_index) =
503                wasm_module.local_memory_index(memory_index)
504            {
505                let offset = offsets.vmctx_vmmemory_definition(local_memory_index);
506                let offset = intrinsics.i32_ty.const_int(offset.into(), false);
507                unsafe {
508                    err!(builder.build_gep(intrinsics.i8_ty, callee_vmctx_ptr_value, &[offset], ""))
509                }
510            } else {
511                let offset = offsets.vmctx_vmmemory_import(memory_index);
512                let offset = intrinsics.i32_ty.const_int(offset.into(), false);
513                let memory_definition_ptr_ptr = unsafe {
514                    err!(builder.build_gep(intrinsics.i8_ty, callee_vmctx_ptr_value, &[offset], ""))
515                };
516                let memory_definition_ptr_ptr =
517                    err!(builder.build_bit_cast(memory_definition_ptr_ptr, intrinsics.ptr_ty, "",))
518                        .into_pointer_value();
519
520                err!(builder.build_load(intrinsics.ptr_ty, memory_definition_ptr_ptr, ""))
521                    .into_pointer_value()
522            };
523            let memory_definition_ptr =
524                err!(builder.build_bit_cast(memory_definition_ptr, intrinsics.ptr_ty, "",))
525                    .into_pointer_value();
526            let base_ptr = err!(builder.build_struct_gep(
527                intrinsics.vmmemory_definition_ty,
528                memory_definition_ptr,
529                intrinsics.vmmemory_definition_base_element,
530                "",
531            ));
532
533            let memory_style = &memory_styles[memory_index];
534            let base_ptr = if let MemoryStyle::Dynamic { .. } = memory_style {
535                base_ptr
536            } else {
537                err!(builder.build_load(intrinsics.ptr_ty, base_ptr, "")).into_pointer_value()
538            };
539
540            base_ptr.set_name("trmpl_m0_base_ptr");
541
542            args_vec.push(base_ptr.into());
543        }
544
545        for (i, param_ty) in func_sig.params().iter().enumerate() {
546            let index = intrinsics.i32_ty.const_int(i as _, false);
547            let item_pointer = unsafe {
548                err!(builder.build_in_bounds_gep(
549                    intrinsics.i128_ty,
550                    args_rets_ptr,
551                    &[index],
552                    "arg_ptr"
553                ))
554            };
555
556            let casted_type = type_to_llvm(intrinsics, *param_ty)?;
557
558            let typed_item_pointer = err!(builder.build_pointer_cast(
559                item_pointer,
560                intrinsics.ptr_ty,
561                "typed_arg_pointer"
562            ));
563
564            let arg = err!(builder.build_load(casted_type, typed_item_pointer, "arg"));
565            args_vec.push(arg.into());
566        }
567
568        let call_site = err!(builder.build_indirect_call(
569            llvm_func_type,
570            func_ptr,
571            args_vec.as_slice(),
572            "call"
573        ));
574        for (attr, attr_loc) in func_attrs {
575            call_site.add_attribute(*attr_loc, *attr);
576        }
577
578        let rets = self
579            .abi
580            .rets_from_call(&builder, intrinsics, call_site, func_sig)?;
581        for (idx, v) in rets.into_iter().enumerate() {
582            let ptr = unsafe {
583                err!(builder.build_gep(
584                    intrinsics.i128_ty,
585                    args_rets_ptr,
586                    &[intrinsics.i32_ty.const_int(idx as u64, false)],
587                    "",
588                ))
589            };
590            let ptr = err!(builder.build_pointer_cast(
591                ptr,
592                self.ctx.ptr_type(AddressSpace::default()),
593                ""
594            ));
595            err!(builder.build_store(ptr, v));
596        }
597
598        err!(builder.build_return(None));
599        Ok(())
600    }
601
602    fn generate_dynamic_trampoline<'ctx>(
603        &self,
604        trampoline_func: FunctionValue,
605        func_sig: &FuncType,
606        context: &'ctx Context,
607        intrinsics: &Intrinsics<'ctx>,
608    ) -> Result<(), CompileError> {
609        let entry_block = context.append_basic_block(trampoline_func, "entry");
610        let builder = context.create_builder();
611        builder.position_at_end(entry_block);
612
613        // Allocate stack space for the params and results.
614        let values = err!(builder.build_alloca(
615            intrinsics.i128_ty.array_type(cmp::max(
616                func_sig.params().len().try_into().unwrap(),
617                func_sig.results().len().try_into().unwrap(),
618            )),
619            "",
620        ));
621
622        // Copy params to 'values'.
623        let first_user_param = if self.abi.is_sret(func_sig)? { 2 } else { 1 };
624        for i in 0..func_sig.params().len() {
625            let ptr = unsafe {
626                err!(builder.build_in_bounds_gep(
627                    intrinsics.i128_ty,
628                    values,
629                    &[intrinsics.i32_ty.const_int(i.try_into().unwrap(), false)],
630                    "args",
631                ))
632            };
633            let ptr = err!(builder.build_bit_cast(ptr, intrinsics.ptr_ty, "")).into_pointer_value();
634            err!(
635                builder.build_store(
636                    ptr,
637                    trampoline_func
638                        .get_nth_param(i as u32 + first_user_param)
639                        .unwrap(),
640                )
641            );
642        }
643
644        let callee_ptr_ty = intrinsics.void_ty.fn_type(
645            &[
646                intrinsics.ptr_ty.into(), // vmctx ptr
647                intrinsics.ptr_ty.into(), // in/out values ptr
648            ],
649            false,
650        );
651        let vmctx = self.abi.get_vmctx_ptr_param(&trampoline_func);
652        let callee_ty =
653            err!(builder.build_bit_cast(vmctx, self.ctx.ptr_type(AddressSpace::default()), ""));
654        let callee =
655            err!(builder.build_load(intrinsics.ptr_ty, callee_ty.into_pointer_value(), ""))
656                .into_pointer_value();
657        callee.set_name("func_ptr");
658
659        let values_ptr = err!(builder.build_pointer_cast(values, intrinsics.ptr_ty, ""));
660        values_ptr.set_name("value_ptr");
661        err!(builder.build_indirect_call(
662            callee_ptr_ty,
663            callee,
664            &[vmctx.into(), values_ptr.into()],
665            "",
666        ));
667
668        if func_sig.results().is_empty() {
669            err!(builder.build_return(None));
670        } else {
671            let results = func_sig
672                .results()
673                .iter()
674                .enumerate()
675                .map(|(idx, ty)| {
676                    let ptr = unsafe {
677                        err!(builder.build_gep(
678                            intrinsics.i128_ty,
679                            values,
680                            &[intrinsics.i32_ty.const_int(idx.try_into().unwrap(), false)],
681                            "",
682                        ))
683                    };
684                    let ptr = err!(builder.build_pointer_cast(ptr, intrinsics.ptr_ty, ""));
685                    err_nt!(builder.build_load(type_to_llvm(intrinsics, *ty)?, ptr, ""))
686                })
687                .collect::<Result<Vec<_>, CompileError>>()?;
688
689            if self.abi.is_sret(func_sig)? {
690                let sret = trampoline_func
691                    .get_first_param()
692                    .unwrap()
693                    .into_pointer_value();
694
695                let basic_types: Vec<_> = func_sig
696                    .results()
697                    .iter()
698                    .map(|&ty| type_to_llvm(intrinsics, ty))
699                    .collect::<Result<_, _>>()?;
700                let mut struct_value = context.struct_type(&basic_types, false).get_undef();
701
702                for (idx, value) in results.iter().enumerate() {
703                    let value = err!(builder.build_bit_cast(
704                        *value,
705                        type_to_llvm(intrinsics, func_sig.results()[idx])?,
706                        "",
707                    ));
708                    struct_value =
709                        err!(builder.build_insert_value(struct_value, value, idx as u32, ""))
710                            .into_struct_value();
711                }
712                err!(builder.build_store(sret, struct_value));
713                err!(builder.build_return(None));
714            } else {
715                err!(
716                    builder.build_return(Some(&self.abi.pack_values_for_register_return(
717                        intrinsics,
718                        &builder,
719                        results.as_slice(),
720                        &trampoline_func.get_type(),
721                    )?))
722                );
723            }
724        }
725
726        Ok(())
727    }
728}
wasmer_compiler_llvm/translator/trampoline.rs

wasmer_compiler_llvm/translator/
trampoline.rs
