wasmer_compiler_llvm/
abi.rs

1// LLVM implements part of the ABI lowering internally, but also requires that
2// the user pack and unpack values themselves sometimes.
3
4#![deny(missing_docs)]
5
6use crate::error::{err, err_nt};
7use crate::translator::intrinsics::{Intrinsics, type_to_llvm};
8use inkwell::{
9    AddressSpace,
10    attributes::{Attribute, AttributeLoc},
11    builder::Builder,
12    context::Context,
13    targets::TargetMachine,
14    types::{AnyType, BasicMetadataTypeEnum, BasicType, BasicTypeEnum, FunctionType, StructType},
15    values::{
16        BasicValue, BasicValueEnum, CallSiteValue, FloatValue, FunctionValue, IntValue,
17        PointerValue, VectorValue,
18    },
19};
20use itertools::Itertools;
21use wasmer_compiler::abi::{
22    PairSlot, ReturnAbi, ReturnSlot, classify_return_type_aarch64,
23    classify_return_type_loongarch64, classify_return_type_riscv64, classify_return_type_x86_64,
24};
25use wasmer_types::{CompileError, FunctionType as FuncSig, Type};
26use wasmer_vm::VMOffsets;
27
28/// Target-specific return-value classification.
29pub(crate) trait Architecture {
30    /// Classifies a WebAssembly function's return values.
31    fn classify_return_type(&self, types: &[Type]) -> ReturnAbi;
32
33    /// Whether i32 parameters require RISC-V's sign-extension attributes.
34    fn sign_extend_i32_params(&self) -> bool {
35        false
36    }
37}
38
39/// Architectures supported by the LLVM backend.
40pub(crate) enum TargetArchitecture {
41    X86_64,
42    Aarch64,
43    LoongArch64,
44    Riscv64,
45}
46
47impl Architecture for TargetArchitecture {
48    fn classify_return_type(&self, types: &[Type]) -> ReturnAbi {
49        match self {
50            Self::X86_64 => classify_return_type_x86_64(types),
51            Self::Aarch64 => classify_return_type_aarch64(types),
52            Self::LoongArch64 => classify_return_type_loongarch64(types),
53            Self::Riscv64 => classify_return_type_riscv64(types),
54        }
55    }
56
57    fn sign_extend_i32_params(&self) -> bool {
58        matches!(self, Self::Riscv64)
59    }
60}
61
62/// LLVM ABI lowering shared by all supported architectures.
63pub(crate) struct LLVMAbi<A: Architecture = TargetArchitecture> {
64    pub(crate) architecture: A,
65}
66
67/// Selects ABI lowering for an LLVM target machine.
68pub(crate) fn get_abi(
69    target_machine: &TargetMachine,
70) -> Result<LLVMAbi<TargetArchitecture>, CompileError> {
71    let target_name = target_machine.get_triple();
72    let target_name = target_name.as_str().to_string_lossy();
73    let architecture = if target_name.starts_with("aarch64") {
74        TargetArchitecture::Aarch64
75    } else if target_name.starts_with("loongarch64") {
76        TargetArchitecture::LoongArch64
77    } else if target_name.starts_with("riscv64") {
78        TargetArchitecture::Riscv64
79    } else if target_name.starts_with("x86_64") {
80        TargetArchitecture::X86_64
81    } else {
82        return Err(CompileError::UnsupportedTarget(target_name.to_string()));
83    };
84    Ok(LLVMAbi { architecture })
85}
86
87/// We need to produce different LLVM IR for different platforms. (Contrary to
88/// popular knowledge LLVM IR is not intended to be portable in that way.) This
89/// trait deals with differences between function signatures on different
90/// targets.
91impl<A: Architecture> LLVMAbi<A> {
92    /// Given a function definition, retrieve the parameter that is the vmctx pointer.
93    pub(crate) fn get_vmctx_ptr_param<'ctx>(
94        &self,
95        func_value: &FunctionValue<'ctx>,
96    ) -> PointerValue<'ctx> {
97        let param = func_value
98            .get_nth_param(u32::from(
99                func_value
100                    .get_enum_attribute(
101                        AttributeLoc::Param(0),
102                        Attribute::get_named_enum_kind_id("sret"),
103                    )
104                    .is_some(),
105            ))
106            .unwrap();
107        param.set_name("vmctx");
108
109        param.into_pointer_value()
110    }
111
112    /// Given a function definition, retrieve the parameter that is the pointer to the first --
113    /// number 0 -- local memory.
114    pub(crate) fn get_m0_ptr_param<'ctx>(
115        &self,
116        func_value: &FunctionValue<'ctx>,
117    ) -> PointerValue<'ctx> {
118        let vmctx_idx = u32::from(
119            func_value
120                .get_enum_attribute(
121                    AttributeLoc::Param(0),
122                    Attribute::get_named_enum_kind_id("sret"),
123                )
124                .is_some(),
125        );
126
127        let param = func_value.get_nth_param(vmctx_idx + 1).unwrap();
128        param.set_name("m0_base_ptr");
129
130        param.into_pointer_value()
131    }
132
133    /// Marshall wasm stack values into function parameters.
134    #[allow(clippy::too_many_arguments)]
135    pub(crate) fn args_to_call<'ctx>(
136        &self,
137        alloca_builder: &Builder<'ctx>,
138        func_sig: &FuncSig,
139        llvm_fn_ty: &FunctionType<'ctx>,
140        ctx_ptr: PointerValue<'ctx>,
141        values: &[BasicValueEnum<'ctx>],
142        intrinsics: &Intrinsics<'ctx>,
143        m0: Option<PointerValue<'ctx>>,
144        sret_ptr: Option<PointerValue<'ctx>>,
145    ) -> Result<Vec<BasicValueEnum<'ctx>>, CompileError> {
146        // If it's an sret, allocate the return space.
147        let sret = if self.llvm_fn_uses_sret(llvm_fn_ty, func_sig) {
148            let llvm_params: Vec<_> = func_sig
149                .results()
150                .iter()
151                .map(|x| type_to_llvm(intrinsics, *x).unwrap())
152                .collect();
153            let llvm_params = llvm_fn_ty
154                .get_context()
155                .struct_type(llvm_params.as_slice(), false);
156            // If return_call is used, we pass existing sret pointer instead a newly created one.
157            Some(match sret_ptr {
158                Some(sret_ptr) => sret_ptr,
159                None => err!(alloca_builder.build_alloca(llvm_params, "sret")),
160            })
161        } else {
162            None
163        };
164
165        let mut args = vec![ctx_ptr.as_basic_value_enum()];
166
167        if let Some(m0) = m0 {
168            args.push(m0.into());
169        }
170
171        let args = args.into_iter().chain(values.iter().copied());
172
173        let ret = if let Some(sret) = sret {
174            std::iter::once(sret.as_basic_value_enum())
175                .chain(args)
176                .collect()
177        } else {
178            args.collect()
179        };
180
181        Ok(ret)
182    }
183
184    /// Whether a concrete LLVM function type uses an `sret` parameter for the given wasm signature.
185    pub(crate) fn llvm_fn_uses_sret<'ctx>(
186        &self,
187        llvm_fn_ty: &FunctionType<'ctx>,
188        func_sig: &FuncSig,
189    ) -> bool {
190        llvm_fn_ty.get_return_type().is_none() && func_sig.results().len() > 1
191    }
192
193    /// Whether the native function uses an `sret` parameter.
194    pub(crate) fn is_sret(&self, func_sig: &FuncSig) -> Result<bool, CompileError> {
195        Ok(matches!(
196            self.architecture.classify_return_type(func_sig.results()),
197            ReturnAbi::Sret(_)
198        ))
199    }
200}
201
202impl<A: Architecture> LLVMAbi<A> {
203    // Given a wasm function type, produce an llvm function declaration.
204    pub(crate) fn func_type_to_llvm<'ctx>(
205        &self,
206        context: &'ctx Context,
207        intrinsics: &Intrinsics<'ctx>,
208        offsets: Option<&VMOffsets>,
209        sig: &FuncSig,
210        include_m0_param: bool,
211    ) -> Result<(FunctionType<'ctx>, Vec<(Attribute, AttributeLoc)>), CompileError> {
212        // The LLVM type carrying a single-register return slot.
213        let slot_llvm_type = |slot| match slot {
214            ReturnSlot::Natural(t) => type_to_llvm(intrinsics, t),
215            ReturnSlot::Raw(Type::F32) => Ok(intrinsics.i32_ty.as_basic_type_enum()),
216            ReturnSlot::Raw(Type::F64) => Ok(intrinsics.i64_ty.as_basic_type_enum()),
217            ReturnSlot::Raw(t) => type_to_llvm(intrinsics, t),
218        };
219
220        // The LLVM type carrying two 32-bit values sharing one register.
221        let pair_llvm_type = |pair| match pair {
222            PairSlot::F32Vector(_, _) => intrinsics.f32_ty.vec_type(2).as_basic_type_enum(),
223            PairSlot::Raw(_, _) => intrinsics.i64_ty.as_basic_type_enum(),
224        };
225
226        let return_abi = self.architecture.classify_return_type(sig.results());
227        let return_llvm_type: Option<BasicTypeEnum<'ctx>> = match &return_abi {
228            ReturnAbi::Void | ReturnAbi::Sret(_) => None,
229            ReturnAbi::Single(single_value) => Some(type_to_llvm(intrinsics, *single_value)?),
230            ReturnAbi::Pair(s0, s1) => Some(
231                context
232                    .struct_type(&[slot_llvm_type(*s0)?, slot_llvm_type(*s1)?], false)
233                    .as_basic_type_enum(),
234            ),
235            ReturnAbi::Unpacked(types) => Some(
236                context
237                    .struct_type(
238                        &types
239                            .iter()
240                            .map(|ty| type_to_llvm(intrinsics, *ty))
241                            .collect::<Result<Vec<_>, _>>()?,
242                        false,
243                    )
244                    .as_basic_type_enum(),
245            ),
246            ReturnAbi::PackedPair(pair) => Some(pair_llvm_type(*pair)),
247            ReturnAbi::PackedFirst(pair, slot) => Some(
248                context
249                    .struct_type(&[pair_llvm_type(*pair), slot_llvm_type(*slot)?], false)
250                    .as_basic_type_enum(),
251            ),
252            ReturnAbi::PackedLast(slot, pair) => Some(
253                context
254                    .struct_type(&[slot_llvm_type(*slot)?, pair_llvm_type(*pair)], false)
255                    .as_basic_type_enum(),
256            ),
257            ReturnAbi::PackedQuads(p0, p1) => Some(
258                context
259                    .struct_type(&[pair_llvm_type(*p0), pair_llvm_type(*p1)], false)
260                    .as_basic_type_enum(),
261            ),
262        };
263
264        let user_param_types = sig.params().iter().map(|&ty| type_to_llvm(intrinsics, ty));
265        let mut param_types = vec![Ok(intrinsics.ptr_ty.as_basic_type_enum())];
266        if include_m0_param {
267            param_types.push(Ok(intrinsics.ptr_ty.as_basic_type_enum()));
268        }
269        let param_llvm_types = param_types
270            .into_iter()
271            .chain(user_param_types)
272            .map(|v| v.map(Into::into))
273            .collect::<Result<Vec<BasicMetadataTypeEnum>, _>>()?;
274
275        // TODO: figure out how many bytes long vmctx is, and mark it dereferenceable. (no need to mark it nonnull once we do this.)
276        let vmctx_attributes = |i: u32| {
277            vec![
278                (
279                    context.create_enum_attribute(Attribute::get_named_enum_kind_id("nofree"), 0),
280                    AttributeLoc::Param(i),
281                ),
282                (
283                    if let Some(offsets) = offsets {
284                        context.create_enum_attribute(
285                            Attribute::get_named_enum_kind_id("dereferenceable"),
286                            offsets.size_of_vmctx().into(),
287                        )
288                    } else {
289                        context
290                            .create_enum_attribute(Attribute::get_named_enum_kind_id("nonnull"), 0)
291                    },
292                    AttributeLoc::Param(i),
293                ),
294                (
295                    context.create_enum_attribute(
296                        Attribute::get_named_enum_kind_id("align"),
297                        std::mem::align_of::<wasmer_vm::VMContext>()
298                            .try_into()
299                            .unwrap(),
300                    ),
301                    AttributeLoc::Param(i),
302                ),
303            ]
304        };
305
306        let (function_type, mut attributes, sret_param) =
307            if let ReturnAbi::Sret(types) = &return_abi {
308                let basic_types = types
309                    .iter()
310                    .map(|&ty| type_to_llvm(intrinsics, ty))
311                    .collect::<Result<Vec<_>, _>>()?;
312                let sret = context.struct_type(&basic_types, false);
313                let sret_ptr = context.ptr_type(AddressSpace::default());
314                let sret_param_llvm_types =
315                    std::iter::once(BasicMetadataTypeEnum::from(sret_ptr.as_basic_type_enum()))
316                        .chain(param_llvm_types.iter().copied())
317                        .collect_vec();
318
319                let mut attributes = vec![(
320                    context.create_type_attribute(
321                        Attribute::get_named_enum_kind_id("sret"),
322                        sret.as_any_type_enum(),
323                    ),
324                    AttributeLoc::Param(0),
325                )];
326                attributes.append(&mut vmctx_attributes(1));
327                (
328                    intrinsics
329                        .void_ty
330                        .fn_type(sret_param_llvm_types.as_slice(), false),
331                    attributes,
332                    1,
333                )
334            } else {
335                let function_type = match return_llvm_type {
336                    Some(return_type) => return_type.fn_type(param_llvm_types.as_slice(), false),
337                    None => intrinsics
338                        .void_ty
339                        .fn_type(param_llvm_types.as_slice(), false),
340                };
341                (function_type, vmctx_attributes(0), 0)
342            };
343
344        if self.architecture.sign_extend_i32_params() {
345            let extra_params = 1 + usize::from(include_m0_param) + sret_param;
346            for (index, ty) in sig.params().iter().enumerate() {
347                if *ty == Type::I32 {
348                    for name in ["signext", "noundef"] {
349                        attributes.push((
350                            context
351                                .create_enum_attribute(Attribute::get_named_enum_kind_id(name), 0),
352                            AttributeLoc::Param((index + extra_params) as u32),
353                        ));
354                    }
355                }
356            }
357        }
358
359        Ok((function_type, attributes))
360    }
361
362    // Given a CallSite, extract the returned values and return them in a Vec.
363    pub(crate) fn rets_from_call<'ctx>(
364        &self,
365        builder: &Builder<'ctx>,
366        intrinsics: &Intrinsics<'ctx>,
367        call_site: CallSiteValue<'ctx>,
368        func_sig: &FuncSig,
369    ) -> Result<Vec<BasicValueEnum<'ctx>>, CompileError> {
370        let split_i64 =
371            |value: IntValue<'ctx>| -> Result<(IntValue<'ctx>, IntValue<'ctx>), CompileError> {
372                assert!(value.get_type() == intrinsics.i64_ty);
373                let low = err!(builder.build_int_truncate(value, intrinsics.i32_ty, ""));
374                let lshr = err!(builder.build_right_shift(
375                    value,
376                    intrinsics.i64_ty.const_int(32, false),
377                    false,
378                    ""
379                ));
380                let high = err!(builder.build_int_truncate(lshr, intrinsics.i32_ty, ""));
381                Ok((low, high))
382            };
383
384        let f32x2_ty = intrinsics.f32_ty.vec_type(2).as_basic_type_enum();
385        let extract_f32x2 = |value: VectorValue<'ctx>| -> Result<(FloatValue<'ctx>, FloatValue<'ctx>), CompileError> {
386            assert!(value.get_type() == f32x2_ty.into_vector_type());
387            let ret0 = err!(builder
388                .build_extract_element(value, intrinsics.i32_ty.const_int(0, false), ""))
389                .into_float_value();
390            let ret1 = err!(builder
391                .build_extract_element(value, intrinsics.i32_ty.const_int(1, false), ""))
392                .into_float_value();
393            Ok((ret0, ret1))
394        };
395
396        let casted =
397            |value: BasicValueEnum<'ctx>, ty: Type| -> Result<BasicValueEnum<'ctx>, CompileError> {
398                match ty {
399                    Type::I32 | Type::ExceptionRef => {
400                        assert!(
401                            value.get_type() == intrinsics.i32_ty.as_basic_type_enum()
402                                || value.get_type() == intrinsics.f32_ty.as_basic_type_enum()
403                        );
404                        err_nt!(builder.build_bit_cast(value, intrinsics.i32_ty, ""))
405                    }
406                    Type::F32 => {
407                        assert!(
408                            value.get_type() == intrinsics.i32_ty.as_basic_type_enum()
409                                || value.get_type() == intrinsics.f32_ty.as_basic_type_enum()
410                        );
411                        err_nt!(builder.build_bit_cast(value, intrinsics.f32_ty, ""))
412                    }
413                    _ => panic!("should only be called to repack 32-bit values"),
414                }
415            };
416
417        // Restore a single-register return slot to its natural wasm type.
418        let restore_slot = |value, slot| match slot {
419            ReturnSlot::Natural(_) => Ok(value),
420            ReturnSlot::Raw(Type::F32) => {
421                err_nt!(builder.build_bit_cast(value, intrinsics.f32_ty, ""))
422            }
423            ReturnSlot::Raw(Type::F64) => {
424                err_nt!(builder.build_bit_cast(value, intrinsics.f64_ty, ""))
425            }
426            ReturnSlot::Raw(_) => Ok(value),
427        };
428
429        // Split a packed-pair register back into its two wasm values.
430        let unpack_pair =
431            |value: BasicValueEnum<'ctx>,
432             pair: PairSlot|
433             -> Result<(BasicValueEnum<'ctx>, BasicValueEnum<'ctx>), CompileError> {
434                match pair {
435                    PairSlot::F32Vector(_, _) => {
436                        let (v0, v1) = extract_f32x2(value.into_vector_value())?;
437                        Ok((v0.into(), v1.into()))
438                    }
439                    PairSlot::Raw(t0, t1) => {
440                        let (low, high) = split_i64(value.into_int_value())?;
441                        Ok((casted(low.into(), t0)?, casted(high.into(), t1)?))
442                    }
443                }
444            };
445
446        if let Some(basic_value) = call_site.try_as_basic_value().basic() {
447            if func_sig.results().len() > 1 {
448                if basic_value.get_type() == intrinsics.i64_ty.as_basic_type_enum() {
449                    assert!(func_sig.results().len() == 2);
450                    let value = basic_value.into_int_value();
451                    let (low, high) = split_i64(value)?;
452                    let low = casted(low.into(), func_sig.results()[0])?;
453                    let high = casted(high.into(), func_sig.results()[1])?;
454                    return Ok(vec![low, high]);
455                }
456                if basic_value.get_type() == f32x2_ty {
457                    assert!(func_sig.results().len() == 2);
458                    let (ret0, ret1) = extract_f32x2(basic_value.into_vector_value())?;
459                    return Ok(vec![ret0.into(), ret1.into()]);
460                }
461                let struct_value = basic_value.into_struct_value();
462                let rets = (0..struct_value.get_type().count_fields())
463                    .map(|i| builder.build_extract_value(struct_value, i, "").unwrap())
464                    .collect_vec();
465                let ret = match self.architecture.classify_return_type(func_sig.results()) {
466                    ReturnAbi::Unpacked(_) => rets,
467                    ReturnAbi::Pair(s0, s1) => {
468                        vec![restore_slot(rets[0], s0)?, restore_slot(rets[1], s1)?]
469                    }
470                    ReturnAbi::PackedFirst(pair, slot) => {
471                        assert!(func_sig.results().len() == 3);
472                        let (low, high) = unpack_pair(rets[0], pair)?;
473                        vec![low, high, restore_slot(rets[1], slot)?]
474                    }
475                    ReturnAbi::PackedLast(slot, pair) => {
476                        assert!(func_sig.results().len() == 3);
477                        let (low, high) = unpack_pair(rets[1], pair)?;
478                        vec![restore_slot(rets[0], slot)?, low, high]
479                    }
480                    ReturnAbi::PackedQuads(p0, p1) => {
481                        assert!(func_sig.results().len() == 4);
482                        let (low0, high0) = unpack_pair(rets[0], p0)?;
483                        let (low1, high1) = unpack_pair(rets[1], p1)?;
484                        vec![low0, high0, low1, high1]
485                    }
486                    ReturnAbi::Void
487                    | ReturnAbi::Single(_)
488                    | ReturnAbi::PackedPair(_)
489                    | ReturnAbi::Sret(_) => {
490                        unreachable!("expected an sret for this type")
491                    }
492                };
493
494                Ok(ret)
495            } else {
496                assert!(func_sig.results().len() == 1);
497                Ok(vec![basic_value])
498            }
499        } else {
500            assert!(call_site.count_arguments() > 0); // Either sret or vmctx.
501            if call_site
502                .get_enum_attribute(
503                    AttributeLoc::Param(0),
504                    Attribute::get_named_enum_kind_id("sret"),
505                )
506                .is_some()
507            {
508                let sret_ty = call_site
509                    .try_as_basic_value()
510                    .unwrap_instruction()
511                    .get_operand(0)
512                    .unwrap()
513                    .unwrap_value();
514                let sret = sret_ty.into_pointer_value();
515                // re-build the llvm-type struct holding the return values
516                let llvm_results: Vec<_> = func_sig
517                    .results()
518                    .iter()
519                    .map(|x| type_to_llvm(intrinsics, *x).unwrap())
520                    .collect();
521                let struct_type = intrinsics
522                    .i32_ty
523                    .get_context()
524                    .struct_type(llvm_results.as_slice(), false);
525
526                let struct_value =
527                    err!(builder.build_load(struct_type, sret, "")).into_struct_value();
528                let mut rets: Vec<_> = Vec::new();
529                for i in 0..struct_value.get_type().count_fields() {
530                    let value = builder.build_extract_value(struct_value, i, "").unwrap();
531                    rets.push(value);
532                }
533                assert!(func_sig.results().len() == rets.len());
534                Ok(rets)
535            } else {
536                assert!(func_sig.results().is_empty());
537                Ok(vec![])
538            }
539        }
540    }
541
542    pub(crate) fn pack_values_for_register_return<'ctx>(
543        &self,
544        intrinsics: &Intrinsics<'ctx>,
545        builder: &Builder<'ctx>,
546        values: &[BasicValueEnum<'ctx>],
547        func_sig: &FuncSig,
548        func_type: &FunctionType<'ctx>,
549    ) -> Result<BasicValueEnum<'ctx>, CompileError> {
550        let pack_i32s = |low: BasicValueEnum<'ctx>, high: BasicValueEnum<'ctx>| {
551            assert!(low.get_type() == intrinsics.i32_ty.as_basic_type_enum());
552            assert!(high.get_type() == intrinsics.i32_ty.as_basic_type_enum());
553            let (low, high) = (low.into_int_value(), high.into_int_value());
554            let low = err!(builder.build_int_z_extend(low, intrinsics.i64_ty, ""));
555            let high = err!(builder.build_int_z_extend(high, intrinsics.i64_ty, ""));
556            let high =
557                err!(builder.build_left_shift(high, intrinsics.i64_ty.const_int(32, false), ""));
558            err_nt!(
559                builder
560                    .build_or(low, high, "")
561                    .map(|v| v.as_basic_value_enum())
562            )
563        };
564
565        let pack_f32s = |first: BasicValueEnum<'ctx>,
566                         second: BasicValueEnum<'ctx>|
567         -> Result<BasicValueEnum<'ctx>, CompileError> {
568            assert!(first.get_type() == intrinsics.f32_ty.as_basic_type_enum());
569            assert!(second.get_type() == intrinsics.f32_ty.as_basic_type_enum());
570            let (first, second) = (first.into_float_value(), second.into_float_value());
571            let vec_ty = intrinsics.f32_ty.vec_type(2);
572            let vec = err!(builder.build_insert_element(
573                vec_ty.get_undef(),
574                first,
575                intrinsics.i32_zero,
576                ""
577            ));
578            err_nt!(
579                builder
580                    .build_insert_element(vec, second, intrinsics.i32_ty.const_int(1, false), "")
581                    .map(|v| v.as_basic_value_enum())
582            )
583        };
584
585        let build_struct = |ty: StructType<'ctx>, values: &[BasicValueEnum<'ctx>]| {
586            let mut struct_value = ty.get_undef();
587            for (i, v) in values.iter().enumerate() {
588                struct_value = builder
589                    .build_insert_value(struct_value, *v, i as u32, "")
590                    .unwrap()
591                    .into_struct_value();
592            }
593            struct_value.as_basic_value_enum()
594        };
595
596        let pack_slot = |value, slot| match slot {
597            ReturnSlot::Natural(_) => Ok(value),
598            ReturnSlot::Raw(Type::F32) => {
599                err_nt!(builder.build_bit_cast(value, intrinsics.i32_ty, ""))
600            }
601            ReturnSlot::Raw(Type::F64) => {
602                err_nt!(builder.build_bit_cast(value, intrinsics.i64_ty, ""))
603            }
604            ReturnSlot::Raw(_) => Ok(value),
605        };
606
607        // Pack two 32-bit values into the single register their `PairSlot` calls for.
608        let pack_pair = |first, second, pair| match pair {
609            PairSlot::F32Vector(_, _) => pack_f32s(first, second),
610            PairSlot::Raw(_, _) => {
611                let v1 = err!(builder.build_bit_cast(first, intrinsics.i32_ty, ""));
612                let v2 = err!(builder.build_bit_cast(second, intrinsics.i32_ty, ""));
613                pack_i32s(v1, v2)
614            }
615        };
616
617        let return_abi = self.architecture.classify_return_type(func_sig.results());
618        let struct_ty = || func_type.get_return_type().unwrap().into_struct_type();
619
620        Ok(match return_abi {
621            ReturnAbi::Single(_) => values[0],
622            ReturnAbi::PackedPair(pair) => pack_pair(values[0], values[1], pair)?,
623            ReturnAbi::Unpacked(_) => build_struct(struct_ty(), values),
624            ReturnAbi::Pair(s0, s1) => build_struct(
625                struct_ty(),
626                &[pack_slot(values[0], s0)?, pack_slot(values[1], s1)?],
627            ),
628            ReturnAbi::PackedFirst(pair, slot) => build_struct(
629                struct_ty(),
630                &[
631                    pack_pair(values[0], values[1], pair)?,
632                    pack_slot(values[2], slot)?,
633                ],
634            ),
635            ReturnAbi::PackedLast(slot, pair) => build_struct(
636                struct_ty(),
637                &[
638                    pack_slot(values[0], slot)?,
639                    pack_pair(values[1], values[2], pair)?,
640                ],
641            ),
642            ReturnAbi::PackedQuads(p0, p1) => build_struct(
643                struct_ty(),
644                &[
645                    pack_pair(values[0], values[1], p0)?,
646                    pack_pair(values[2], values[3], p1)?,
647                ],
648            ),
649            ReturnAbi::Void | ReturnAbi::Sret(_) => {
650                unreachable!("called to perform register return on struct return or void function")
651            }
652        })
653    }
654}
wasmer_compiler_llvm/abi.rs

wasmer_compiler_llvm/
abi.rs
