wasmer_compiler_llvm/abi/
aarch64_systemv.rs

1use crate::abi::Abi;
2use crate::error::{err, err_nt};
3use crate::translator::intrinsics::{Intrinsics, type_to_llvm};
4use inkwell::values::BasicValue;
5use inkwell::{
6    AddressSpace,
7    attributes::{Attribute, AttributeLoc},
8    builder::Builder,
9    context::Context,
10    types::{AnyType, BasicMetadataTypeEnum, BasicType, FunctionType, StructType},
11    values::{BasicValueEnum, CallSiteValue, IntValue},
12};
13use wasmer_types::CompileError;
14use wasmer_types::{FunctionType as FuncSig, Type};
15use wasmer_vm::VMOffsets;
16
17use std::convert::TryInto;
18
19/// Implementation of the [`Abi`] trait for the Aarch64 ABI on Linux.
20pub struct Aarch64SystemV {}
21
22impl Abi for Aarch64SystemV {
23    // Given a wasm function type, produce an llvm function declaration.
24    fn func_type_to_llvm<'ctx>(
25        &self,
26        context: &'ctx Context,
27        intrinsics: &Intrinsics<'ctx>,
28        offsets: Option<&VMOffsets>,
29        sig: &FuncSig,
30        include_m0_param: bool,
31    ) -> Result<(FunctionType<'ctx>, Vec<(Attribute, AttributeLoc)>), CompileError> {
32        let user_param_types = sig.params().iter().map(|&ty| type_to_llvm(intrinsics, ty));
33
34        let mut param_types = vec![Ok(intrinsics.ptr_ty.as_basic_type_enum())];
35        if include_m0_param {
36            param_types.push(Ok(intrinsics.ptr_ty.as_basic_type_enum()));
37        }
38
39        let param_types = param_types.into_iter().chain(user_param_types);
40
41        let vmctx_attributes = |i: u32| {
42            vec![
43                (
44                    context.create_enum_attribute(Attribute::get_named_enum_kind_id("nofree"), 0),
45                    AttributeLoc::Param(i),
46                ),
47                (
48                    if let Some(offsets) = offsets {
49                        context.create_enum_attribute(
50                            Attribute::get_named_enum_kind_id("dereferenceable"),
51                            offsets.size_of_vmctx().into(),
52                        )
53                    } else {
54                        context
55                            .create_enum_attribute(Attribute::get_named_enum_kind_id("nonnull"), 0)
56                    },
57                    AttributeLoc::Param(i),
58                ),
59                (
60                    context.create_enum_attribute(
61                        Attribute::get_named_enum_kind_id("align"),
62                        std::mem::align_of::<wasmer_vm::VMContext>()
63                            .try_into()
64                            .unwrap(),
65                    ),
66                    AttributeLoc::Param(i),
67                ),
68            ]
69        };
70
71        Ok(match sig.results() {
72            [] => (
73                intrinsics.void_ty.fn_type(
74                    param_types
75                        .map(|v| v.map(Into::into))
76                        .collect::<Result<Vec<BasicMetadataTypeEnum>, _>>()?
77                        .as_slice(),
78                    false,
79                ),
80                vmctx_attributes(0),
81            ),
82            [_] => {
83                let single_value = sig.results()[0];
84                (
85                    type_to_llvm(intrinsics, single_value)?.fn_type(
86                        param_types
87                            .map(|v| v.map(Into::into))
88                            .collect::<Result<Vec<BasicMetadataTypeEnum>, _>>()?
89                            .as_slice(),
90                        false,
91                    ),
92                    vmctx_attributes(0),
93                )
94            }
95            [Type::F32, Type::F32] => {
96                let f32_ty = intrinsics.f32_ty.as_basic_type_enum();
97                (
98                    context.struct_type(&[f32_ty, f32_ty], false).fn_type(
99                        param_types
100                            .map(|v| v.map(Into::into))
101                            .collect::<Result<Vec<BasicMetadataTypeEnum>, _>>()?
102                            .as_slice(),
103                        false,
104                    ),
105                    vmctx_attributes(0),
106                )
107            }
108            [Type::F64, Type::F64] => {
109                let f64_ty = intrinsics.f64_ty.as_basic_type_enum();
110                (
111                    context.struct_type(&[f64_ty, f64_ty], false).fn_type(
112                        param_types
113                            .map(|v| v.map(Into::into))
114                            .collect::<Result<Vec<BasicMetadataTypeEnum>, _>>()?
115                            .as_slice(),
116                        false,
117                    ),
118                    vmctx_attributes(0),
119                )
120            }
121            [Type::F32, Type::F32, Type::F32] => {
122                let f32_ty = intrinsics.f32_ty.as_basic_type_enum();
123                (
124                    context
125                        .struct_type(&[f32_ty, f32_ty, f32_ty], false)
126                        .fn_type(
127                            param_types
128                                .map(|v| v.map(Into::into))
129                                .collect::<Result<Vec<BasicMetadataTypeEnum>, _>>()?
130                                .as_slice(),
131                            false,
132                        ),
133                    vmctx_attributes(0),
134                )
135            }
136            [Type::F32, Type::F32, Type::F32, Type::F32] => {
137                let f32_ty = intrinsics.f32_ty.as_basic_type_enum();
138                (
139                    context
140                        .struct_type(&[f32_ty, f32_ty, f32_ty, f32_ty], false)
141                        .fn_type(
142                            param_types
143                                .map(|v| v.map(Into::into))
144                                .collect::<Result<Vec<BasicMetadataTypeEnum>, _>>()?
145                                .as_slice(),
146                            false,
147                        ),
148                    vmctx_attributes(0),
149                )
150            }
151            [t1, t2]
152                if matches!(
153                    t1,
154                    Type::I32 | Type::I64 | Type::F32 | Type::F64 | Type::ExceptionRef
155                ) && matches!(t2, Type::FuncRef | Type::ExternRef) =>
156            {
157                let t1 = type_to_llvm(intrinsics, *t1).unwrap();
158                let t2 = type_to_llvm(intrinsics, *t2).unwrap();
159                (
160                    context
161                        .struct_type(&[t1.as_basic_type_enum(), t2.as_basic_type_enum()], false)
162                        .fn_type(
163                            param_types
164                                .map(|v| v.map(Into::into))
165                                .collect::<Result<Vec<BasicMetadataTypeEnum>, _>>()?
166                                .as_slice(),
167                            false,
168                        ),
169                    vmctx_attributes(0),
170                )
171            }
172            _ => {
173                let sig_returns_bitwidths = sig
174                    .results()
175                    .iter()
176                    .map(|ty| match ty {
177                        Type::I32 | Type::F32 | Type::ExceptionRef => 32,
178                        Type::I64 | Type::F64 => 64,
179                        Type::V128 => 128,
180                        Type::ExternRef | Type::FuncRef => 64, /* pointer */
181                    })
182                    .collect::<Vec<i32>>();
183                match sig_returns_bitwidths.as_slice() {
184                    [32, 32] => (
185                        intrinsics.i64_ty.fn_type(
186                            param_types
187                                .map(|v| v.map(Into::into))
188                                .collect::<Result<Vec<BasicMetadataTypeEnum>, _>>()?
189                                .as_slice(),
190                            false,
191                        ),
192                        vmctx_attributes(0),
193                    ),
194                    [32, 64]
195                    | [64, 32]
196                    | [64, 64]
197                    | [32, 32, 32]
198                    | [64, 32, 32]
199                    | [32, 32, 64]
200                    | [32, 32, 32, 32] => (
201                        intrinsics.i64_ty.array_type(2).fn_type(
202                            param_types
203                                .map(|v| v.map(Into::into))
204                                .collect::<Result<Vec<BasicMetadataTypeEnum>, _>>()?
205                                .as_slice(),
206                            false,
207                        ),
208                        vmctx_attributes(0),
209                    ),
210                    _ => {
211                        let basic_types: Vec<_> = sig
212                            .results()
213                            .iter()
214                            .map(|&ty| type_to_llvm(intrinsics, ty))
215                            .collect::<Result<_, _>>()?;
216
217                        let sret = context.struct_type(&basic_types, false);
218                        let sret_ptr = context.ptr_type(AddressSpace::default());
219
220                        let param_types =
221                            std::iter::once(Ok(sret_ptr.as_basic_type_enum())).chain(param_types);
222
223                        let mut attributes = vec![(
224                            context.create_type_attribute(
225                                Attribute::get_named_enum_kind_id("sret"),
226                                sret.as_any_type_enum(),
227                            ),
228                            AttributeLoc::Param(0),
229                        )];
230                        attributes.append(&mut vmctx_attributes(1));
231
232                        (
233                            intrinsics.void_ty.fn_type(
234                                param_types
235                                    .map(|v| v.map(Into::into))
236                                    .collect::<Result<Vec<BasicMetadataTypeEnum>, _>>()?
237                                    .as_slice(),
238                                false,
239                            ),
240                            attributes,
241                        )
242                    }
243                }
244            }
245        })
246    }
247
248    // Given a CallSite, extract the returned values and return them in a Vec.
249    fn rets_from_call<'ctx>(
250        &self,
251        builder: &Builder<'ctx>,
252        intrinsics: &Intrinsics<'ctx>,
253        call_site: CallSiteValue<'ctx>,
254        func_sig: &FuncSig,
255    ) -> Result<Vec<BasicValueEnum<'ctx>>, CompileError> {
256        let split_i64 =
257            |value: IntValue<'ctx>| -> Result<(IntValue<'ctx>, IntValue<'ctx>), CompileError> {
258                assert!(value.get_type() == intrinsics.i64_ty);
259                let low = err!(builder.build_int_truncate(value, intrinsics.i32_ty, ""));
260                let lshr = err!(builder.build_right_shift(
261                    value,
262                    intrinsics.i64_ty.const_int(32, false),
263                    false,
264                    "",
265                ));
266                let high = err!(builder.build_int_truncate(lshr, intrinsics.i32_ty, ""));
267                Ok((low, high))
268            };
269
270        let casted =
271            |value: BasicValueEnum<'ctx>, ty: Type| -> Result<BasicValueEnum<'ctx>, CompileError> {
272                match ty {
273                    Type::I32 | Type::ExceptionRef => {
274                        assert!(
275                            value.get_type() == intrinsics.i32_ty.as_basic_type_enum()
276                                || value.get_type() == intrinsics.f32_ty.as_basic_type_enum()
277                        );
278                        err_nt!(builder.build_bit_cast(value, intrinsics.i32_ty, ""))
279                    }
280                    Type::F32 => {
281                        assert!(
282                            value.get_type() == intrinsics.i32_ty.as_basic_type_enum()
283                                || value.get_type() == intrinsics.f32_ty.as_basic_type_enum()
284                        );
285                        err_nt!(builder.build_bit_cast(value, intrinsics.f32_ty, ""))
286                    }
287                    Type::I64 => {
288                        assert!(
289                            value.get_type() == intrinsics.i64_ty.as_basic_type_enum()
290                                || value.get_type() == intrinsics.f64_ty.as_basic_type_enum()
291                        );
292                        err_nt!(builder.build_bit_cast(value, intrinsics.i64_ty, ""))
293                    }
294                    Type::F64 => {
295                        assert!(
296                            value.get_type() == intrinsics.i64_ty.as_basic_type_enum()
297                                || value.get_type() == intrinsics.f64_ty.as_basic_type_enum()
298                        );
299                        err_nt!(builder.build_bit_cast(value, intrinsics.f64_ty, ""))
300                    }
301                    Type::V128 => {
302                        assert!(value.get_type() == intrinsics.i128_ty.as_basic_type_enum());
303                        Ok(value)
304                    }
305                    Type::ExternRef | Type::FuncRef => {
306                        assert!(value.get_type() == intrinsics.i64_ty.as_basic_type_enum());
307                        err_nt!(
308                            builder
309                                .build_int_to_ptr(value.into_int_value(), intrinsics.ptr_ty, "")
310                                .map(|ptr| ptr.as_basic_value_enum())
311                        )
312                    }
313                }
314            };
315
316        if let Some(basic_value) = call_site.try_as_basic_value().basic() {
317            if func_sig.results().len() > 1 {
318                if basic_value.get_type() == intrinsics.i64_ty.as_basic_type_enum() {
319                    assert!(func_sig.results().len() == 2);
320                    let value = basic_value.into_int_value();
321                    let (low, high) = split_i64(value)?;
322                    let low = casted(low.into(), func_sig.results()[0])?;
323                    let high = casted(high.into(), func_sig.results()[1])?;
324                    return Ok(vec![low, high]);
325                }
326                if basic_value.is_struct_value() {
327                    let struct_value = basic_value.into_struct_value();
328                    return Ok((0..struct_value.get_type().count_fields())
329                        .map(|i| builder.build_extract_value(struct_value, i, "").unwrap())
330                        .collect::<Vec<_>>());
331                }
332                let array_value = basic_value.into_array_value();
333                let low = builder
334                    .build_extract_value(array_value, 0, "")
335                    .unwrap()
336                    .into_int_value();
337                let high = builder
338                    .build_extract_value(array_value, 1, "")
339                    .unwrap()
340                    .into_int_value();
341                let func_sig_returns_bitwidths = func_sig
342                    .results()
343                    .iter()
344                    .map(|ty| match ty {
345                        Type::I32 | Type::F32 | Type::ExceptionRef => 32,
346                        Type::I64 | Type::F64 => 64,
347                        Type::V128 => 128,
348                        Type::ExternRef | Type::FuncRef => 64, /* pointer */
349                    })
350                    .collect::<Vec<i32>>();
351
352                match func_sig_returns_bitwidths.as_slice() {
353                    [32, 64] => {
354                        let (low, _) = split_i64(low)?;
355                        let low = casted(low.into(), func_sig.results()[0])?;
356                        let high = casted(high.into(), func_sig.results()[1])?;
357                        Ok(vec![low, high])
358                    }
359                    [64, 32] => {
360                        let (high, _) = split_i64(high)?;
361                        let low = casted(low.into(), func_sig.results()[0])?;
362                        let high = casted(high.into(), func_sig.results()[1])?;
363                        Ok(vec![low, high])
364                    }
365                    [64, 64] => {
366                        let low = casted(low.into(), func_sig.results()[0])?;
367                        let high = casted(high.into(), func_sig.results()[1])?;
368                        Ok(vec![low, high])
369                    }
370                    [32, 32, 32] => {
371                        let (v1, v2) = split_i64(low)?;
372                        let (v3, _) = split_i64(high)?;
373                        let v1 = casted(v1.into(), func_sig.results()[0])?;
374                        let v2 = casted(v2.into(), func_sig.results()[1])?;
375                        let v3 = casted(v3.into(), func_sig.results()[2])?;
376                        Ok(vec![v1, v2, v3])
377                    }
378                    [32, 32, 64] => {
379                        let (v1, v2) = split_i64(low)?;
380                        let v1 = casted(v1.into(), func_sig.results()[0])?;
381                        let v2 = casted(v2.into(), func_sig.results()[1])?;
382                        let v3 = casted(high.into(), func_sig.results()[2])?;
383                        Ok(vec![v1, v2, v3])
384                    }
385                    [64, 32, 32] => {
386                        let v1 = casted(low.into(), func_sig.results()[0])?;
387                        let (v2, v3) = split_i64(high)?;
388                        let v2 = casted(v2.into(), func_sig.results()[1])?;
389                        let v3 = casted(v3.into(), func_sig.results()[2])?;
390                        Ok(vec![v1, v2, v3])
391                    }
392                    [32, 32, 32, 32] => {
393                        let (v1, v2) = split_i64(low)?;
394                        let (v3, v4) = split_i64(high)?;
395                        let v1 = casted(v1.into(), func_sig.results()[0])?;
396                        let v2 = casted(v2.into(), func_sig.results()[1])?;
397                        let v3 = casted(v3.into(), func_sig.results()[2])?;
398                        let v4 = casted(v4.into(), func_sig.results()[3])?;
399                        Ok(vec![v1, v2, v3, v4])
400                    }
401                    _ => unreachable!("expected an sret for this type"),
402                }
403            } else {
404                assert!(func_sig.results().len() == 1);
405                Ok(vec![basic_value])
406            }
407        } else {
408            assert!(call_site.count_arguments() > 0); // Either sret or vmctx.
409            if call_site
410                .get_enum_attribute(
411                    AttributeLoc::Param(0),
412                    Attribute::get_named_enum_kind_id("sret"),
413                )
414                .is_some()
415            {
416                let sret_ty = call_site
417                    .try_as_basic_value()
418                    .unwrap_instruction()
419                    .get_operand(0)
420                    .unwrap()
421                    .unwrap_value();
422                let sret = sret_ty.into_pointer_value();
423                // re-build the llvm-type struct holding the return values
424                let llvm_results: Vec<_> = func_sig
425                    .results()
426                    .iter()
427                    .map(|x| type_to_llvm(intrinsics, *x).unwrap())
428                    .collect();
429                let struct_type = intrinsics
430                    .i32_ty
431                    .get_context()
432                    .struct_type(llvm_results.as_slice(), false);
433
434                let struct_value =
435                    err!(builder.build_load(struct_type, sret, "")).into_struct_value();
436                let mut rets: Vec<_> = Vec::new();
437                for i in 0..struct_value.get_type().count_fields() {
438                    let value = err!(builder.build_extract_value(struct_value, i, ""));
439                    rets.push(value);
440                }
441                assert!(func_sig.results().len() == rets.len());
442                Ok(rets)
443            } else {
444                assert!(func_sig.results().is_empty());
445                Ok(vec![])
446            }
447        }
448    }
449
450    fn pack_values_for_register_return<'ctx>(
451        &self,
452        intrinsics: &Intrinsics<'ctx>,
453        builder: &Builder<'ctx>,
454        values: &[BasicValueEnum<'ctx>],
455        func_type: &FunctionType<'ctx>,
456    ) -> Result<BasicValueEnum<'ctx>, CompileError> {
457        let is_32 = |value: BasicValueEnum| {
458            (value.is_int_value() && value.into_int_value().get_type() == intrinsics.i32_ty)
459                || (value.is_float_value()
460                    && value.into_float_value().get_type() == intrinsics.f32_ty)
461        };
462        let is_64 = |value: BasicValueEnum| {
463            value.is_pointer_value()
464                || (value.is_int_value() && value.into_int_value().get_type() == intrinsics.i64_ty)
465                || (value.is_float_value()
466                    && value.into_float_value().get_type() == intrinsics.f64_ty)
467        };
468
469        let pack_i32s = |low: BasicValueEnum<'ctx>, high: BasicValueEnum<'ctx>| {
470            assert!(low.get_type() == intrinsics.i32_ty.as_basic_type_enum());
471            assert!(high.get_type() == intrinsics.i32_ty.as_basic_type_enum());
472            let (low, high) = (low.into_int_value(), high.into_int_value());
473            let low = err!(builder.build_int_z_extend(low, intrinsics.i64_ty, ""));
474            let high = err!(builder.build_int_z_extend(high, intrinsics.i64_ty, ""));
475            let high =
476                err!(builder.build_left_shift(high, intrinsics.i64_ty.const_int(32, false), ""));
477            err_nt!(
478                builder
479                    .build_or(low, high, "")
480                    .map(|v| v.as_basic_value_enum())
481            )
482        };
483
484        let to_i64 = |v: BasicValueEnum<'ctx>| -> Result<BasicValueEnum<'_>, CompileError> {
485            if v.is_float_value() {
486                let v = v.into_float_value();
487                if v.get_type() == intrinsics.f32_ty {
488                    let v = err!(builder.build_bit_cast(v, intrinsics.i32_ty, "")).into_int_value();
489                    let v = err!(builder.build_int_z_extend(v, intrinsics.i64_ty, ""));
490                    Ok(v.as_basic_value_enum())
491                } else {
492                    debug_assert!(v.get_type() == intrinsics.f64_ty);
493                    let v = err!(builder.build_bit_cast(v, intrinsics.i64_ty, ""));
494                    Ok(v.as_basic_value_enum())
495                }
496            } else if v.is_pointer_value() {
497                let v =
498                    err!(builder.build_ptr_to_int(v.into_pointer_value(), intrinsics.i64_ty, "",));
499                Ok(v.as_basic_value_enum())
500            } else {
501                let v = v.into_int_value();
502                if v.get_type() == intrinsics.i32_ty {
503                    let v = err!(builder.build_int_z_extend(v, intrinsics.i64_ty, ""));
504                    Ok(v.as_basic_value_enum())
505                } else {
506                    debug_assert!(v.get_type() == intrinsics.i64_ty);
507                    Ok(v.as_basic_value_enum())
508                }
509            }
510        };
511
512        let build_struct = |ty: StructType<'ctx>,
513                            values: &[BasicValueEnum<'ctx>]|
514         -> Result<BasicValueEnum<'_>, CompileError> {
515            let mut struct_value = ty.get_undef();
516            for (i, v) in values.iter().enumerate() {
517                struct_value = err!(builder.build_insert_value(struct_value, *v, i as u32, ""))
518                    .into_struct_value();
519            }
520            Ok(struct_value.as_basic_value_enum())
521        };
522
523        let build_2xi64 = |low: BasicValueEnum<'ctx>,
524                           high: BasicValueEnum<'ctx>|
525         -> Result<BasicValueEnum<'_>, CompileError> {
526            let low = to_i64(low)?;
527            let high = to_i64(high)?;
528            let value = intrinsics.i64_ty.array_type(2).get_undef();
529            let value = err!(builder.build_insert_value(value, low, 0, ""));
530            let value = err!(builder.build_insert_value(value, high, 1, ""));
531            Ok(value.as_basic_value_enum())
532        };
533
534        Ok(match *values {
535            [one_value] => one_value,
536            [v1, v2]
537                if v1.is_float_value()
538                    && v2.is_float_value()
539                    && v1.into_float_value().get_type() == v2.into_float_value().get_type() =>
540            {
541                build_struct(
542                    func_type.get_return_type().unwrap().into_struct_type(),
543                    &[v1, v2],
544                )?
545            }
546            [v1, v2] if is_32(v1) && is_32(v2) => {
547                let v1 = err!(builder.build_bit_cast(v1, intrinsics.i32_ty, ""));
548                let v2 = err!(builder.build_bit_cast(v2, intrinsics.i32_ty, ""));
549                pack_i32s(v1, v2)?
550            }
551            [v1, v2] => build_2xi64(v1, v2)?,
552            [v1, v2, v3]
553                if is_32(v1)
554                    && is_32(v2)
555                    && is_32(v3)
556                    && v1.is_float_value()
557                    && v2.is_float_value()
558                    && v3.is_float_value() =>
559            {
560                build_struct(
561                    func_type.get_return_type().unwrap().into_struct_type(),
562                    &[v1, v2, v3],
563                )?
564            }
565            [v1, v2, v3] if is_32(v1) && is_32(v2) => {
566                let v1 = err!(builder.build_bit_cast(v1, intrinsics.i32_ty, ""));
567                let v2 = err!(builder.build_bit_cast(v2, intrinsics.i32_ty, ""));
568                let v1v2_pack = pack_i32s(v1, v2)?;
569                build_2xi64(v1v2_pack, v3)?
570            }
571            [v1, v2, v3] if is_64(v1) && is_32(v2) && is_32(v3) => {
572                let v2 = err!(builder.build_bit_cast(v2, intrinsics.i32_ty, ""));
573                let v3 = err!(builder.build_bit_cast(v3, intrinsics.i32_ty, ""));
574                let v2v3_pack = pack_i32s(v2, v3)?;
575                build_2xi64(v1, v2v3_pack)?
576            }
577            [v1, v2, v3, v4]
578                if is_32(v1)
579                    && is_32(v2)
580                    && is_32(v3)
581                    && is_32(v4)
582                    && v1.is_float_value()
583                    && v2.is_float_value()
584                    && v3.is_float_value()
585                    && v4.is_float_value() =>
586            {
587                build_struct(
588                    func_type.get_return_type().unwrap().into_struct_type(),
589                    &[v1, v2, v3, v4],
590                )?
591            }
592            [v1, v2, v3, v4] if is_32(v1) && is_32(v2) && is_32(v3) && is_32(v4) => {
593                let v1 = err!(builder.build_bit_cast(v1, intrinsics.i32_ty, ""));
594                let v2 = err!(builder.build_bit_cast(v2, intrinsics.i32_ty, ""));
595                let v1v2_pack = pack_i32s(v1, v2)?;
596                let v3 = err!(builder.build_bit_cast(v3, intrinsics.i32_ty, ""));
597                let v4 = err!(builder.build_bit_cast(v4, intrinsics.i32_ty, ""));
598                let v3v4_pack = pack_i32s(v3, v4)?;
599                build_2xi64(v1v2_pack, v3v4_pack)?
600            }
601            _ => {
602                unreachable!("called to perform register return on struct return or void function")
603            }
604        })
605    }
606}