wasmer_compiler_llvm/translator/
trampoline.rs

1// TODO: Remove
2#![allow(unused)]
3
4use crate::{
5    abi::{LLVMAbi, get_abi},
6    config::LLVM,
7    error::{err, err_nt},
8    object_file::{RkyvCompiledFunction, load_object_file},
9    translator::{
10        code::enable_m0_optimization,
11        intrinsics::{Intrinsics, type_to_llvm},
12    },
13};
14use inkwell::{
15    AddressSpace, DLLStorageClass,
16    attributes::{Attribute, AttributeLoc},
17    context::Context,
18    module::{Linkage, Module},
19    passes::PassBuilderOptions,
20    targets::{FileType, TargetMachine},
21    types::FunctionType,
22    values::{BasicMetadataValueEnum, FunctionValue},
23};
24use std::{cmp, convert::TryInto};
25use target_lexicon::{BinaryFormat, Triple};
26use wasmer_compiler::{
27    misc::{CompiledFunctionExt, CompiledKind},
28    types::{
29        function::{CompiledFunctionBody, FunctionBody},
30        module::CompileModuleInfo,
31        relocation::{Relocation, RelocationTarget},
32        section::{CustomSection, CustomSectionProtection, SectionBody, SectionIndex},
33    },
34};
35use wasmer_types::{
36    CompileError, FunctionIndex, FunctionType as FuncType, LocalFunctionIndex, entity::PrimaryMap,
37};
38use wasmer_vm::MemoryStyle;
39
40pub struct FuncTrampoline {
41    ctx: Context,
42    target_machine: TargetMachine,
43    target_triple: Triple,
44    abi: LLVMAbi,
45    binary_fmt: BinaryFormat,
46    func_section: String,
47}
48
49const FUNCTION_SECTION_ELF: &str = "__TEXT,wasmer_trmpl"; // Needs to be between 1 and 16 chars
50const FUNCTION_SECTION_MACHO: &str = "wasmer_trmpl"; // Needs to be between 1 and 16 chars
51
52impl FuncTrampoline {
53    pub fn new(
54        target_machine: TargetMachine,
55        target_triple: Triple,
56        binary_fmt: BinaryFormat,
57    ) -> Result<Self, CompileError> {
58        let abi = get_abi(&target_machine)?;
59        Ok(Self {
60            ctx: Context::create(),
61            target_machine,
62            target_triple,
63            abi,
64            func_section: match binary_fmt {
65                BinaryFormat::Elf => FUNCTION_SECTION_ELF.to_string(),
66                BinaryFormat::Macho => FUNCTION_SECTION_MACHO.to_string(),
67                _ => {
68                    return Err(CompileError::UnsupportedTarget(format!(
69                        "Unsupported binary format: {binary_fmt:?}",
70                    )));
71                }
72            },
73            binary_fmt,
74        })
75    }
76
77    pub fn trampoline_to_module(
78        &self,
79        ty: &FuncType,
80        config: &LLVM,
81        function: &CompiledKind,
82        compile_info: &CompileModuleInfo,
83    ) -> Result<Module<'_>, CompileError> {
84        // The function type, used for the callbacks.
85        let module = self.ctx.create_module("");
86        let target_machine = &self.target_machine;
87        let target_triple = target_machine.get_triple();
88        let target_data: inkwell::targets::TargetData = target_machine.get_target_data();
89        module.set_triple(&target_triple);
90        module.set_data_layout(&target_data.get_data_layout());
91        let intrinsics = Intrinsics::declare(
92            &module,
93            &self.ctx,
94            &target_data,
95            &self.target_triple,
96            &self.binary_fmt,
97        );
98
99        let m0_is_enabled = enable_m0_optimization(config, &compile_info.memory_styles);
100        let (callee_ty, callee_attrs) =
101            self.abi
102                .func_type_to_llvm(&self.ctx, &intrinsics, None, ty, m0_is_enabled)?;
103        let trampoline_ty = intrinsics.void_ty.fn_type(
104            &[
105                intrinsics.ptr_ty.into(), // vmctx ptr
106                intrinsics.ptr_ty.into(), // callee function address
107                intrinsics.ptr_ty.into(), // in/out values ptr
108            ],
109            false,
110        );
111
112        let trampoline_func = module.add_function(
113            &function.linkage_name(),
114            trampoline_ty,
115            Some(Linkage::External),
116        );
117        if !config.experimental_artifact {
118            trampoline_func
119                .as_global_value()
120                .set_section(Some(&self.func_section));
121        }
122        trampoline_func
123            .as_global_value()
124            .set_linkage(Linkage::DLLExport);
125        trampoline_func
126            .as_global_value()
127            .set_dll_storage_class(DLLStorageClass::Export);
128        // We intentionally mark this function as no-unwind; otherwise, stack unwinding could cross the
129        // trampoline boundary and cause Rust to complain about a foreign exception being thrown.
130        trampoline_func.add_attribute(AttributeLoc::Function, intrinsics.nounwind);
131        trampoline_func.add_attribute(AttributeLoc::Function, intrinsics.frame_pointer);
132        self.generate_trampoline(
133            config,
134            compile_info,
135            trampoline_func,
136            ty,
137            callee_ty,
138            &callee_attrs,
139            &self.ctx,
140            &intrinsics,
141        )?;
142
143        if let Some(ref callbacks) = config.callbacks {
144            callbacks.preopt_ir(function, &compile_info.module.hash_string(), &module);
145        }
146
147        // Always verify the LLVM IR.
148        err!(module.verify());
149
150        err!(module.run_passes("instcombine", target_machine, PassBuilderOptions::create(),));
151
152        if let Some(ref callbacks) = config.callbacks {
153            callbacks.postopt_ir(function, &compile_info.module.hash_string(), &module);
154        }
155        Ok(module)
156    }
157
158    pub fn trampoline(
159        &self,
160        ty: &FuncType,
161        config: &LLVM,
162        function: &CompiledKind,
163        compile_info: &CompileModuleInfo,
164    ) -> Result<CompiledFunctionBody, CompileError> {
165        let module = self.trampoline_to_module(ty, config, function, compile_info)?;
166        let target_machine = &self.target_machine;
167
168        let memory_buffer = target_machine
169            .write_to_memory_buffer(&module, FileType::Object)
170            .unwrap();
171
172        if let Some(ref callbacks) = config.callbacks {
173            let module_hash = compile_info.module.hash_string();
174            callbacks.obj_memory_buffer(function, &module_hash, &memory_buffer);
175            let asm_buffer = target_machine
176                .write_to_memory_buffer(&module, FileType::Assembly)
177                .unwrap();
178            callbacks.asm_memory_buffer(function, &module_hash, &asm_buffer);
179        }
180
181        if config.experimental_artifact {
182            Ok(CompiledFunctionBody::Elf(memory_buffer.as_slice().to_vec()))
183        } else {
184            // Use a dummy function index to detect relocations against the trampoline
185            // function's address, which shouldn't exist and are not supported.
186            // Note, we just drop all custom sections, and verify that the function
187            // body itself has no relocations at all. This value should never be
188            // touched at all. However, it is set up so that if we do touch it (maybe
189            // due to someone changing the code later on), it'll explode, which is desirable!
190            let dummy_reloc_target =
191                RelocationTarget::DynamicTrampoline(FunctionIndex::from_u32(u32::MAX - 1));
192
193            // Note: we don't count .gcc_except_table here because native-to-wasm
194            // trampolines are not supposed to generate any LSDA sections. We *want* them
195            // to terminate libunwind's stack searches.
196            let RkyvCompiledFunction {
197                compiled_function,
198                custom_sections,
199                eh_frame_section_indices,
200                mut compact_unwind_section_indices,
201                ..
202            } = load_object_file(
203                memory_buffer.as_slice(),
204                &self.func_section,
205                dummy_reloc_target,
206                |name: &str| {
207                    Err(CompileError::Codegen(format!(
208                        "trampoline generation produced reference to unknown function {name}",
209                    )))
210                },
211                self.binary_fmt,
212                &self.target_triple,
213            )?;
214            let mut all_sections_are_eh_sections = true;
215            let mut unwind_section_indices = eh_frame_section_indices;
216            unwind_section_indices.append(&mut compact_unwind_section_indices);
217            if unwind_section_indices.len() != custom_sections.len() {
218                all_sections_are_eh_sections = false;
219            } else {
220                unwind_section_indices.sort_unstable();
221                for (idx, section_idx) in unwind_section_indices.iter().enumerate() {
222                    if idx as u32 != section_idx.as_u32() {
223                        all_sections_are_eh_sections = false;
224                        break;
225                    }
226                }
227            }
228            if !all_sections_are_eh_sections {
229                return Err(CompileError::Codegen(
230                    "trampoline generation produced non-eh custom sections".into(),
231                ));
232            }
233            if !compiled_function.relocations.is_empty() {
234                return Err(CompileError::Codegen(
235                    "trampoline generation produced relocations".into(),
236                ));
237            }
238            // Ignore CompiledFunctionFrameInfo. Extra frame info isn't a problem.
239
240            Ok(CompiledFunctionBody::Rkyv(FunctionBody {
241                body: compiled_function.body.body,
242                unwind_info: compiled_function.body.unwind_info,
243            }))
244        }
245    }
246
247    pub fn dynamic_trampoline_to_module(
248        &self,
249        ty: &FuncType,
250        config: &LLVM,
251        function: &CompiledKind,
252        module_hash: &Option<String>,
253    ) -> Result<Module<'_>, CompileError> {
254        // The function type, used for the callbacks
255        let module = self.ctx.create_module("");
256        let target_machine = &self.target_machine;
257        let target_data = target_machine.get_target_data();
258        let target_triple = target_machine.get_triple();
259        module.set_triple(&target_triple);
260        module.set_data_layout(&target_data.get_data_layout());
261        let intrinsics = Intrinsics::declare(
262            &module,
263            &self.ctx,
264            &target_data,
265            &self.target_triple,
266            &self.binary_fmt,
267        );
268
269        let (trampoline_ty, trampoline_attrs) =
270            self.abi
271                .func_type_to_llvm(&self.ctx, &intrinsics, None, ty, false)?;
272        let trampoline_func = module.add_function(
273            &function.linkage_name(),
274            trampoline_ty,
275            Some(Linkage::External),
276        );
277        trampoline_func.set_personality_function(intrinsics.personality);
278        trampoline_func.add_attribute(AttributeLoc::Function, intrinsics.frame_pointer);
279        for (attr, attr_loc) in trampoline_attrs {
280            trampoline_func.add_attribute(attr_loc, attr);
281        }
282        if !config.experimental_artifact {
283            trampoline_func
284                .as_global_value()
285                .set_section(Some(&self.func_section));
286        }
287        trampoline_func
288            .as_global_value()
289            .set_linkage(Linkage::DLLExport);
290        trampoline_func
291            .as_global_value()
292            .set_dll_storage_class(DLLStorageClass::Export);
293        self.generate_dynamic_trampoline(trampoline_func, ty, &self.ctx, &intrinsics)?;
294
295        if let Some(ref callbacks) = config.callbacks {
296            callbacks.preopt_ir(function, module_hash, &module);
297        }
298
299        // Always verify the LLVM IR.
300        err!(module.verify());
301
302        err!(module.run_passes("early-cse", target_machine, PassBuilderOptions::create(),));
303
304        if let Some(ref callbacks) = config.callbacks {
305            callbacks.postopt_ir(function, module_hash, &module);
306        }
307
308        Ok(module)
309    }
310
311    #[allow(clippy::too_many_arguments)]
312    pub fn dynamic_trampoline(
313        &self,
314        ty: &FuncType,
315        config: &LLVM,
316        function: &CompiledKind,
317        dynamic_trampoline_index: u32,
318        final_module_custom_sections: &mut PrimaryMap<SectionIndex, CustomSection>,
319        eh_frame_section_bytes: &mut Vec<u8>,
320        eh_frame_section_relocations: &mut Vec<Relocation>,
321        compact_unwind_section_bytes: &mut Vec<u8>,
322        compact_unwind_section_relocations: &mut Vec<Relocation>,
323        module_hash: &Option<String>,
324    ) -> Result<CompiledFunctionBody, CompileError> {
325        let target_machine = &self.target_machine;
326
327        let module = self.dynamic_trampoline_to_module(ty, config, function, module_hash)?;
328
329        let memory_buffer = target_machine
330            .write_to_memory_buffer(&module, FileType::Object)
331            .unwrap();
332
333        if let Some(ref callbacks) = config.callbacks {
334            callbacks.obj_memory_buffer(function, module_hash, &memory_buffer);
335            let asm_buffer = target_machine
336                .write_to_memory_buffer(&module, FileType::Assembly)
337                .unwrap();
338            callbacks.asm_memory_buffer(function, module_hash, &asm_buffer)
339        }
340
341        if config.experimental_artifact {
342            Ok(CompiledFunctionBody::Elf(memory_buffer.as_slice().to_vec()))
343        } else {
344            let RkyvCompiledFunction {
345                compiled_function,
346                custom_sections,
347                eh_frame_section_indices,
348                compact_unwind_section_indices,
349                gcc_except_table_section_indices,
350                data_dw_ref_personality_section_indices,
351            } = load_object_file(
352                memory_buffer.as_slice(),
353                &self.func_section,
354                RelocationTarget::DynamicTrampoline(FunctionIndex::from_u32(
355                    dynamic_trampoline_index,
356                )),
357                |name: &str| {
358                    Err(CompileError::Codegen(format!(
359                        "trampoline generation produced reference to unknown function {name}",
360                    )))
361                },
362                self.binary_fmt,
363                &self.target_triple,
364            )?;
365
366            if !compiled_function.relocations.is_empty() {
367                return Err(CompileError::Codegen(
368                    "trampoline generation produced relocations".into(),
369                ));
370            }
371            // Ignore CompiledFunctionFrameInfo. Extra frame info isn't a problem.
372
373            // Also append EH-related sections to the final module, since we expect
374            // dynamic trampolines to participate in unwinding
375            {
376                let first_section = final_module_custom_sections.len() as u32;
377                for (section_index, mut custom_section) in custom_sections.into_iter() {
378                    for reloc in &mut custom_section.relocations {
379                        if let RelocationTarget::CustomSection(index) = reloc.reloc_target {
380                            reloc.reloc_target = RelocationTarget::CustomSection(
381                                SectionIndex::from_u32(first_section + index.as_u32()),
382                            )
383                        }
384
385                        if reloc.kind.needs_got() {
386                            return Err(CompileError::Codegen(
387                                "trampoline generation produced GOT relocation".into(),
388                            ));
389                        }
390                    }
391
392                    if eh_frame_section_indices.contains(&section_index) {
393                        let offset = eh_frame_section_bytes.len() as u32;
394                        for reloc in &mut custom_section.relocations {
395                            reloc.offset += offset;
396                        }
397                        eh_frame_section_bytes.extend_from_slice(custom_section.bytes.as_slice());
398                        // Terminate the eh_frame info with a zero-length CIE.
399                        eh_frame_section_bytes.extend_from_slice(&[0, 0, 0, 0]);
400                        eh_frame_section_relocations.extend(custom_section.relocations);
401                        // TODO: we do this to keep the count right, remove it.
402                        final_module_custom_sections.push(CustomSection {
403                            protection: CustomSectionProtection::Read,
404                            alignment: None,
405                            bytes: SectionBody::new_with_vec(vec![]),
406                            relocations: vec![],
407                        });
408                    } else if compact_unwind_section_indices.contains(&section_index) {
409                        let offset = compact_unwind_section_bytes.len() as u32;
410                        for reloc in &mut custom_section.relocations {
411                            reloc.offset += offset;
412                        }
413                        compact_unwind_section_bytes
414                            .extend_from_slice(custom_section.bytes.as_slice());
415                        compact_unwind_section_relocations.extend(custom_section.relocations);
416                        // TODO: we do this to keep the count right, remove it.
417                        final_module_custom_sections.push(CustomSection {
418                            protection: CustomSectionProtection::Read,
419                            alignment: None,
420                            bytes: SectionBody::new_with_vec(vec![]),
421                            relocations: vec![],
422                        });
423                    } else if gcc_except_table_section_indices.contains(&section_index)
424                        || data_dw_ref_personality_section_indices.contains(&section_index)
425                    {
426                        final_module_custom_sections.push(custom_section);
427                    } else {
428                        return Err(CompileError::Codegen(
429                            "trampoline generation produced non-eh custom sections".into(),
430                        ));
431                    }
432                }
433            }
434
435            Ok(CompiledFunctionBody::Rkyv(FunctionBody {
436                body: compiled_function.body.body,
437                unwind_info: compiled_function.body.unwind_info,
438            }))
439        }
440    }
441
442    #[allow(clippy::too_many_arguments)]
443    fn generate_trampoline<'ctx>(
444        &self,
445        config: &LLVM,
446        compile_info: &CompileModuleInfo,
447        trampoline_func: FunctionValue,
448        func_sig: &FuncType,
449        llvm_func_type: FunctionType,
450        func_attrs: &[(Attribute, AttributeLoc)],
451        context: &'ctx Context,
452        intrinsics: &Intrinsics<'ctx>,
453    ) -> Result<(), CompileError> {
454        let entry_block = context.append_basic_block(trampoline_func, "entry");
455        let builder = context.create_builder();
456        builder.position_at_end(entry_block);
457
458        let (callee_vmctx_ptr, func_ptr, args_rets_ptr) =
459            match *trampoline_func.get_params().as_slice() {
460                [callee_vmctx_ptr, func_ptr, args_rets_ptr] => (
461                    callee_vmctx_ptr,
462                    func_ptr.into_pointer_value(),
463                    args_rets_ptr.into_pointer_value(),
464                ),
465                _ => {
466                    return Err(CompileError::Codegen(
467                        "trampoline function unimplemented".to_string(),
468                    ));
469                }
470            };
471        func_ptr.set_name("func_ptr");
472
473        let mut args_vec = Vec::with_capacity(func_sig.params().len() + 3);
474
475        if self.abi.is_sret(func_sig)? {
476            let basic_types: Vec<_> = func_sig
477                .results()
478                .iter()
479                .map(|&ty| type_to_llvm(intrinsics, ty))
480                .collect::<Result<_, _>>()?;
481
482            let sret_ty = context.struct_type(&basic_types, false);
483            args_vec.push(err!(builder.build_alloca(sret_ty, "sret")).into());
484        }
485
486        callee_vmctx_ptr.set_name("vmctx");
487        args_vec.push(callee_vmctx_ptr.into());
488
489        if enable_m0_optimization(config, &compile_info.memory_styles) {
490            let wasm_module = &compile_info.module;
491            let memory_styles = &compile_info.memory_styles;
492            let callee_vmctx_ptr_value = callee_vmctx_ptr.into_pointer_value();
493            let offsets =
494                wasmer_vm::VMOffsets::try_new(8, wasm_module).map_err(CompileError::Resource)?;
495
496            // load mem
497            let memory_index = wasmer_types::MemoryIndex::from_u32(0);
498            let memory_definition_ptr = if let Some(local_memory_index) =
499                wasm_module.local_memory_index(memory_index)
500            {
501                let offset = offsets.vmctx_vmmemory_definition(local_memory_index);
502                let offset = intrinsics.i32_ty.const_int(offset.into(), false);
503                unsafe {
504                    err!(builder.build_gep(intrinsics.i8_ty, callee_vmctx_ptr_value, &[offset], ""))
505                }
506            } else {
507                let offset = offsets.vmctx_vmmemory_import(memory_index);
508                let offset = intrinsics.i32_ty.const_int(offset.into(), false);
509                let memory_definition_ptr_ptr = unsafe {
510                    err!(builder.build_gep(intrinsics.i8_ty, callee_vmctx_ptr_value, &[offset], ""))
511                };
512                let memory_definition_ptr_ptr =
513                    err!(builder.build_bit_cast(memory_definition_ptr_ptr, intrinsics.ptr_ty, "",))
514                        .into_pointer_value();
515
516                err!(builder.build_load(intrinsics.ptr_ty, memory_definition_ptr_ptr, ""))
517                    .into_pointer_value()
518            };
519            let memory_definition_ptr =
520                err!(builder.build_bit_cast(memory_definition_ptr, intrinsics.ptr_ty, "",))
521                    .into_pointer_value();
522            let base_ptr = err!(builder.build_struct_gep(
523                intrinsics.vmmemory_definition_ty,
524                memory_definition_ptr,
525                intrinsics.vmmemory_definition_base_element,
526                "",
527            ));
528
529            let memory_style = &memory_styles[memory_index];
530            let base_ptr = if let MemoryStyle::Dynamic { .. } = memory_style {
531                base_ptr
532            } else {
533                err!(builder.build_load(intrinsics.ptr_ty, base_ptr, "")).into_pointer_value()
534            };
535
536            base_ptr.set_name("trmpl_m0_base_ptr");
537
538            args_vec.push(base_ptr.into());
539        }
540
541        for (i, param_ty) in func_sig.params().iter().enumerate() {
542            let index = intrinsics.i32_ty.const_int(i as _, false);
543            let item_pointer = unsafe {
544                err!(builder.build_in_bounds_gep(
545                    intrinsics.i128_ty,
546                    args_rets_ptr,
547                    &[index],
548                    "arg_ptr"
549                ))
550            };
551
552            let casted_type = type_to_llvm(intrinsics, *param_ty)?;
553
554            let typed_item_pointer = err!(builder.build_pointer_cast(
555                item_pointer,
556                intrinsics.ptr_ty,
557                "typed_arg_pointer"
558            ));
559
560            let arg = err!(builder.build_load(casted_type, typed_item_pointer, "arg"));
561            args_vec.push(arg.into());
562        }
563
564        let call_site = err!(builder.build_indirect_call(
565            llvm_func_type,
566            func_ptr,
567            args_vec.as_slice(),
568            "call"
569        ));
570        for (attr, attr_loc) in func_attrs {
571            call_site.add_attribute(*attr_loc, *attr);
572        }
573
574        let rets = self
575            .abi
576            .rets_from_call(&builder, intrinsics, call_site, func_sig)?;
577        for (idx, v) in rets.into_iter().enumerate() {
578            let ptr = unsafe {
579                err!(builder.build_gep(
580                    intrinsics.i128_ty,
581                    args_rets_ptr,
582                    &[intrinsics.i32_ty.const_int(idx as u64, false)],
583                    "",
584                ))
585            };
586            let ptr = err!(builder.build_pointer_cast(
587                ptr,
588                self.ctx.ptr_type(AddressSpace::default()),
589                ""
590            ));
591            err!(builder.build_store(ptr, v));
592        }
593
594        err!(builder.build_return(None));
595        Ok(())
596    }
597
598    fn generate_dynamic_trampoline<'ctx>(
599        &self,
600        trampoline_func: FunctionValue,
601        func_sig: &FuncType,
602        context: &'ctx Context,
603        intrinsics: &Intrinsics<'ctx>,
604    ) -> Result<(), CompileError> {
605        let entry_block = context.append_basic_block(trampoline_func, "entry");
606        let builder = context.create_builder();
607        builder.position_at_end(entry_block);
608
609        // Allocate stack space for the params and results.
610        let values = err!(builder.build_alloca(
611            intrinsics.i128_ty.array_type(cmp::max(
612                func_sig.params().len().try_into().unwrap(),
613                func_sig.results().len().try_into().unwrap(),
614            )),
615            "",
616        ));
617
618        // Copy params to 'values'.
619        let first_user_param = if self.abi.is_sret(func_sig)? { 2 } else { 1 };
620        for i in 0..func_sig.params().len() {
621            let ptr = unsafe {
622                err!(builder.build_in_bounds_gep(
623                    intrinsics.i128_ty,
624                    values,
625                    &[intrinsics.i32_ty.const_int(i.try_into().unwrap(), false)],
626                    "args",
627                ))
628            };
629            let ptr = err!(builder.build_bit_cast(ptr, intrinsics.ptr_ty, "")).into_pointer_value();
630            err!(
631                builder.build_store(
632                    ptr,
633                    trampoline_func
634                        .get_nth_param(i as u32 + first_user_param)
635                        .unwrap(),
636                )
637            );
638        }
639
640        let callee_ptr_ty = intrinsics.void_ty.fn_type(
641            &[
642                intrinsics.ptr_ty.into(), // vmctx ptr
643                intrinsics.ptr_ty.into(), // in/out values ptr
644            ],
645            false,
646        );
647        let vmctx = self.abi.get_vmctx_ptr_param(&trampoline_func);
648        let callee_ty =
649            err!(builder.build_bit_cast(vmctx, self.ctx.ptr_type(AddressSpace::default()), ""));
650        let callee =
651            err!(builder.build_load(intrinsics.ptr_ty, callee_ty.into_pointer_value(), ""))
652                .into_pointer_value();
653        callee.set_name("func_ptr");
654
655        let values_ptr = err!(builder.build_pointer_cast(values, intrinsics.ptr_ty, ""));
656        values_ptr.set_name("value_ptr");
657        err!(builder.build_indirect_call(
658            callee_ptr_ty,
659            callee,
660            &[vmctx.into(), values_ptr.into()],
661            "",
662        ));
663
664        if func_sig.results().is_empty() {
665            err!(builder.build_return(None));
666        } else {
667            let results = func_sig
668                .results()
669                .iter()
670                .enumerate()
671                .map(|(idx, ty)| {
672                    let ptr = unsafe {
673                        err!(builder.build_gep(
674                            intrinsics.i128_ty,
675                            values,
676                            &[intrinsics.i32_ty.const_int(idx.try_into().unwrap(), false)],
677                            "",
678                        ))
679                    };
680                    let ptr = err!(builder.build_pointer_cast(ptr, intrinsics.ptr_ty, ""));
681                    err_nt!(builder.build_load(type_to_llvm(intrinsics, *ty)?, ptr, ""))
682                })
683                .collect::<Result<Vec<_>, CompileError>>()?;
684
685            if self.abi.is_sret(func_sig)? {
686                let sret = trampoline_func
687                    .get_first_param()
688                    .unwrap()
689                    .into_pointer_value();
690
691                let basic_types: Vec<_> = func_sig
692                    .results()
693                    .iter()
694                    .map(|&ty| type_to_llvm(intrinsics, ty))
695                    .collect::<Result<_, _>>()?;
696                let mut struct_value = context.struct_type(&basic_types, false).get_undef();
697
698                for (idx, value) in results.iter().enumerate() {
699                    let value = err!(builder.build_bit_cast(
700                        *value,
701                        type_to_llvm(intrinsics, func_sig.results()[idx])?,
702                        "",
703                    ));
704                    struct_value =
705                        err!(builder.build_insert_value(struct_value, value, idx as u32, ""))
706                            .into_struct_value();
707                }
708                err!(builder.build_store(sret, struct_value));
709                err!(builder.build_return(None));
710            } else {
711                err!(
712                    builder.build_return(Some(&self.abi.pack_values_for_register_return(
713                        intrinsics,
714                        &builder,
715                        results.as_slice(),
716                        func_sig,
717                        &trampoline_func.get_type(),
718                    )?))
719                );
720            }
721        }
722
723        Ok(())
724    }
725}
wasmer_compiler_llvm/translator/trampoline.rs

wasmer_compiler_llvm/translator/
trampoline.rs
