1use crate::abi::Abi;
2use crate::error::{err, err_nt};
3use crate::translator::intrinsics::{Intrinsics, type_to_llvm};
4use inkwell::values::BasicValue;
5use inkwell::{
6 AddressSpace,
7 attributes::{Attribute, AttributeLoc},
8 builder::Builder,
9 context::Context,
10 types::{AnyType, BasicMetadataTypeEnum, BasicType, FunctionType, StructType},
11 values::{BasicValueEnum, CallSiteValue, IntValue},
12};
13use wasmer_types::CompileError;
14use wasmer_types::{FunctionType as FuncSig, Type};
15use wasmer_vm::VMOffsets;
16
17use std::convert::TryInto;
18
19pub struct Aarch64SystemV {}
21
22impl Abi for Aarch64SystemV {
23 fn func_type_to_llvm<'ctx>(
25 &self,
26 context: &'ctx Context,
27 intrinsics: &Intrinsics<'ctx>,
28 offsets: Option<&VMOffsets>,
29 sig: &FuncSig,
30 include_m0_param: bool,
31 ) -> Result<(FunctionType<'ctx>, Vec<(Attribute, AttributeLoc)>), CompileError> {
32 let user_param_types = sig.params().iter().map(|&ty| type_to_llvm(intrinsics, ty));
33
34 let mut param_types = vec![Ok(intrinsics.ptr_ty.as_basic_type_enum())];
35 if include_m0_param {
36 param_types.push(Ok(intrinsics.ptr_ty.as_basic_type_enum()));
37 }
38
39 let param_types = param_types.into_iter().chain(user_param_types);
40
41 let vmctx_attributes = |i: u32| {
42 vec![
43 (
44 context.create_enum_attribute(Attribute::get_named_enum_kind_id("nofree"), 0),
45 AttributeLoc::Param(i),
46 ),
47 (
48 if let Some(offsets) = offsets {
49 context.create_enum_attribute(
50 Attribute::get_named_enum_kind_id("dereferenceable"),
51 offsets.size_of_vmctx().into(),
52 )
53 } else {
54 context
55 .create_enum_attribute(Attribute::get_named_enum_kind_id("nonnull"), 0)
56 },
57 AttributeLoc::Param(i),
58 ),
59 (
60 context.create_enum_attribute(
61 Attribute::get_named_enum_kind_id("align"),
62 std::mem::align_of::<wasmer_vm::VMContext>()
63 .try_into()
64 .unwrap(),
65 ),
66 AttributeLoc::Param(i),
67 ),
68 ]
69 };
70
71 Ok(match sig.results() {
72 [] => (
73 intrinsics.void_ty.fn_type(
74 param_types
75 .map(|v| v.map(Into::into))
76 .collect::<Result<Vec<BasicMetadataTypeEnum>, _>>()?
77 .as_slice(),
78 false,
79 ),
80 vmctx_attributes(0),
81 ),
82 [_] => {
83 let single_value = sig.results()[0];
84 (
85 type_to_llvm(intrinsics, single_value)?.fn_type(
86 param_types
87 .map(|v| v.map(Into::into))
88 .collect::<Result<Vec<BasicMetadataTypeEnum>, _>>()?
89 .as_slice(),
90 false,
91 ),
92 vmctx_attributes(0),
93 )
94 }
95 [Type::F32, Type::F32] => {
96 let f32_ty = intrinsics.f32_ty.as_basic_type_enum();
97 (
98 context.struct_type(&[f32_ty, f32_ty], false).fn_type(
99 param_types
100 .map(|v| v.map(Into::into))
101 .collect::<Result<Vec<BasicMetadataTypeEnum>, _>>()?
102 .as_slice(),
103 false,
104 ),
105 vmctx_attributes(0),
106 )
107 }
108 [Type::F64, Type::F64] => {
109 let f64_ty = intrinsics.f64_ty.as_basic_type_enum();
110 (
111 context.struct_type(&[f64_ty, f64_ty], false).fn_type(
112 param_types
113 .map(|v| v.map(Into::into))
114 .collect::<Result<Vec<BasicMetadataTypeEnum>, _>>()?
115 .as_slice(),
116 false,
117 ),
118 vmctx_attributes(0),
119 )
120 }
121 [Type::F32, Type::F32, Type::F32] => {
122 let f32_ty = intrinsics.f32_ty.as_basic_type_enum();
123 (
124 context
125 .struct_type(&[f32_ty, f32_ty, f32_ty], false)
126 .fn_type(
127 param_types
128 .map(|v| v.map(Into::into))
129 .collect::<Result<Vec<BasicMetadataTypeEnum>, _>>()?
130 .as_slice(),
131 false,
132 ),
133 vmctx_attributes(0),
134 )
135 }
136 [Type::F32, Type::F32, Type::F32, Type::F32] => {
137 let f32_ty = intrinsics.f32_ty.as_basic_type_enum();
138 (
139 context
140 .struct_type(&[f32_ty, f32_ty, f32_ty, f32_ty], false)
141 .fn_type(
142 param_types
143 .map(|v| v.map(Into::into))
144 .collect::<Result<Vec<BasicMetadataTypeEnum>, _>>()?
145 .as_slice(),
146 false,
147 ),
148 vmctx_attributes(0),
149 )
150 }
151 [t1, t2]
152 if matches!(
153 t1,
154 Type::I32 | Type::I64 | Type::F32 | Type::F64 | Type::ExceptionRef
155 ) && matches!(t2, Type::FuncRef | Type::ExternRef) =>
156 {
157 let t1 = type_to_llvm(intrinsics, *t1).unwrap();
158 let t2 = type_to_llvm(intrinsics, *t2).unwrap();
159 (
160 context
161 .struct_type(&[t1.as_basic_type_enum(), t2.as_basic_type_enum()], false)
162 .fn_type(
163 param_types
164 .map(|v| v.map(Into::into))
165 .collect::<Result<Vec<BasicMetadataTypeEnum>, _>>()?
166 .as_slice(),
167 false,
168 ),
169 vmctx_attributes(0),
170 )
171 }
172 _ => {
173 let sig_returns_bitwidths = sig
174 .results()
175 .iter()
176 .map(|ty| match ty {
177 Type::I32 | Type::F32 | Type::ExceptionRef => 32,
178 Type::I64 | Type::F64 => 64,
179 Type::V128 => 128,
180 Type::ExternRef | Type::FuncRef => 64, })
182 .collect::<Vec<i32>>();
183 match sig_returns_bitwidths.as_slice() {
184 [32, 32] => (
185 intrinsics.i64_ty.fn_type(
186 param_types
187 .map(|v| v.map(Into::into))
188 .collect::<Result<Vec<BasicMetadataTypeEnum>, _>>()?
189 .as_slice(),
190 false,
191 ),
192 vmctx_attributes(0),
193 ),
194 [32, 64]
195 | [64, 32]
196 | [64, 64]
197 | [32, 32, 32]
198 | [64, 32, 32]
199 | [32, 32, 64]
200 | [32, 32, 32, 32] => (
201 intrinsics.i64_ty.array_type(2).fn_type(
202 param_types
203 .map(|v| v.map(Into::into))
204 .collect::<Result<Vec<BasicMetadataTypeEnum>, _>>()?
205 .as_slice(),
206 false,
207 ),
208 vmctx_attributes(0),
209 ),
210 _ => {
211 let basic_types: Vec<_> = sig
212 .results()
213 .iter()
214 .map(|&ty| type_to_llvm(intrinsics, ty))
215 .collect::<Result<_, _>>()?;
216
217 let sret = context.struct_type(&basic_types, false);
218 let sret_ptr = context.ptr_type(AddressSpace::default());
219
220 let param_types =
221 std::iter::once(Ok(sret_ptr.as_basic_type_enum())).chain(param_types);
222
223 let mut attributes = vec![(
224 context.create_type_attribute(
225 Attribute::get_named_enum_kind_id("sret"),
226 sret.as_any_type_enum(),
227 ),
228 AttributeLoc::Param(0),
229 )];
230 attributes.append(&mut vmctx_attributes(1));
231
232 (
233 intrinsics.void_ty.fn_type(
234 param_types
235 .map(|v| v.map(Into::into))
236 .collect::<Result<Vec<BasicMetadataTypeEnum>, _>>()?
237 .as_slice(),
238 false,
239 ),
240 attributes,
241 )
242 }
243 }
244 }
245 })
246 }
247
248 fn rets_from_call<'ctx>(
250 &self,
251 builder: &Builder<'ctx>,
252 intrinsics: &Intrinsics<'ctx>,
253 call_site: CallSiteValue<'ctx>,
254 func_sig: &FuncSig,
255 ) -> Result<Vec<BasicValueEnum<'ctx>>, CompileError> {
256 let split_i64 =
257 |value: IntValue<'ctx>| -> Result<(IntValue<'ctx>, IntValue<'ctx>), CompileError> {
258 assert!(value.get_type() == intrinsics.i64_ty);
259 let low = err!(builder.build_int_truncate(value, intrinsics.i32_ty, ""));
260 let lshr = err!(builder.build_right_shift(
261 value,
262 intrinsics.i64_ty.const_int(32, false),
263 false,
264 "",
265 ));
266 let high = err!(builder.build_int_truncate(lshr, intrinsics.i32_ty, ""));
267 Ok((low, high))
268 };
269
270 let casted =
271 |value: BasicValueEnum<'ctx>, ty: Type| -> Result<BasicValueEnum<'ctx>, CompileError> {
272 match ty {
273 Type::I32 | Type::ExceptionRef => {
274 assert!(
275 value.get_type() == intrinsics.i32_ty.as_basic_type_enum()
276 || value.get_type() == intrinsics.f32_ty.as_basic_type_enum()
277 );
278 err_nt!(builder.build_bit_cast(value, intrinsics.i32_ty, ""))
279 }
280 Type::F32 => {
281 assert!(
282 value.get_type() == intrinsics.i32_ty.as_basic_type_enum()
283 || value.get_type() == intrinsics.f32_ty.as_basic_type_enum()
284 );
285 err_nt!(builder.build_bit_cast(value, intrinsics.f32_ty, ""))
286 }
287 Type::I64 => {
288 assert!(
289 value.get_type() == intrinsics.i64_ty.as_basic_type_enum()
290 || value.get_type() == intrinsics.f64_ty.as_basic_type_enum()
291 );
292 err_nt!(builder.build_bit_cast(value, intrinsics.i64_ty, ""))
293 }
294 Type::F64 => {
295 assert!(
296 value.get_type() == intrinsics.i64_ty.as_basic_type_enum()
297 || value.get_type() == intrinsics.f64_ty.as_basic_type_enum()
298 );
299 err_nt!(builder.build_bit_cast(value, intrinsics.f64_ty, ""))
300 }
301 Type::V128 => {
302 assert!(value.get_type() == intrinsics.i128_ty.as_basic_type_enum());
303 Ok(value)
304 }
305 Type::ExternRef | Type::FuncRef => {
306 assert!(value.get_type() == intrinsics.i64_ty.as_basic_type_enum());
307 err_nt!(
308 builder
309 .build_int_to_ptr(value.into_int_value(), intrinsics.ptr_ty, "")
310 .map(|ptr| ptr.as_basic_value_enum())
311 )
312 }
313 }
314 };
315
316 if let Some(basic_value) = call_site.try_as_basic_value().basic() {
317 if func_sig.results().len() > 1 {
318 if basic_value.get_type() == intrinsics.i64_ty.as_basic_type_enum() {
319 assert!(func_sig.results().len() == 2);
320 let value = basic_value.into_int_value();
321 let (low, high) = split_i64(value)?;
322 let low = casted(low.into(), func_sig.results()[0])?;
323 let high = casted(high.into(), func_sig.results()[1])?;
324 return Ok(vec![low, high]);
325 }
326 if basic_value.is_struct_value() {
327 let struct_value = basic_value.into_struct_value();
328 return Ok((0..struct_value.get_type().count_fields())
329 .map(|i| builder.build_extract_value(struct_value, i, "").unwrap())
330 .collect::<Vec<_>>());
331 }
332 let array_value = basic_value.into_array_value();
333 let low = builder
334 .build_extract_value(array_value, 0, "")
335 .unwrap()
336 .into_int_value();
337 let high = builder
338 .build_extract_value(array_value, 1, "")
339 .unwrap()
340 .into_int_value();
341 let func_sig_returns_bitwidths = func_sig
342 .results()
343 .iter()
344 .map(|ty| match ty {
345 Type::I32 | Type::F32 | Type::ExceptionRef => 32,
346 Type::I64 | Type::F64 => 64,
347 Type::V128 => 128,
348 Type::ExternRef | Type::FuncRef => 64, })
350 .collect::<Vec<i32>>();
351
352 match func_sig_returns_bitwidths.as_slice() {
353 [32, 64] => {
354 let (low, _) = split_i64(low)?;
355 let low = casted(low.into(), func_sig.results()[0])?;
356 let high = casted(high.into(), func_sig.results()[1])?;
357 Ok(vec![low, high])
358 }
359 [64, 32] => {
360 let (high, _) = split_i64(high)?;
361 let low = casted(low.into(), func_sig.results()[0])?;
362 let high = casted(high.into(), func_sig.results()[1])?;
363 Ok(vec![low, high])
364 }
365 [64, 64] => {
366 let low = casted(low.into(), func_sig.results()[0])?;
367 let high = casted(high.into(), func_sig.results()[1])?;
368 Ok(vec![low, high])
369 }
370 [32, 32, 32] => {
371 let (v1, v2) = split_i64(low)?;
372 let (v3, _) = split_i64(high)?;
373 let v1 = casted(v1.into(), func_sig.results()[0])?;
374 let v2 = casted(v2.into(), func_sig.results()[1])?;
375 let v3 = casted(v3.into(), func_sig.results()[2])?;
376 Ok(vec![v1, v2, v3])
377 }
378 [32, 32, 64] => {
379 let (v1, v2) = split_i64(low)?;
380 let v1 = casted(v1.into(), func_sig.results()[0])?;
381 let v2 = casted(v2.into(), func_sig.results()[1])?;
382 let v3 = casted(high.into(), func_sig.results()[2])?;
383 Ok(vec![v1, v2, v3])
384 }
385 [64, 32, 32] => {
386 let v1 = casted(low.into(), func_sig.results()[0])?;
387 let (v2, v3) = split_i64(high)?;
388 let v2 = casted(v2.into(), func_sig.results()[1])?;
389 let v3 = casted(v3.into(), func_sig.results()[2])?;
390 Ok(vec![v1, v2, v3])
391 }
392 [32, 32, 32, 32] => {
393 let (v1, v2) = split_i64(low)?;
394 let (v3, v4) = split_i64(high)?;
395 let v1 = casted(v1.into(), func_sig.results()[0])?;
396 let v2 = casted(v2.into(), func_sig.results()[1])?;
397 let v3 = casted(v3.into(), func_sig.results()[2])?;
398 let v4 = casted(v4.into(), func_sig.results()[3])?;
399 Ok(vec![v1, v2, v3, v4])
400 }
401 _ => unreachable!("expected an sret for this type"),
402 }
403 } else {
404 assert!(func_sig.results().len() == 1);
405 Ok(vec![basic_value])
406 }
407 } else {
408 assert!(call_site.count_arguments() > 0); if call_site
410 .get_enum_attribute(
411 AttributeLoc::Param(0),
412 Attribute::get_named_enum_kind_id("sret"),
413 )
414 .is_some()
415 {
416 let sret_ty = call_site
417 .try_as_basic_value()
418 .unwrap_instruction()
419 .get_operand(0)
420 .unwrap()
421 .unwrap_value();
422 let sret = sret_ty.into_pointer_value();
423 let llvm_results: Vec<_> = func_sig
425 .results()
426 .iter()
427 .map(|x| type_to_llvm(intrinsics, *x).unwrap())
428 .collect();
429 let struct_type = intrinsics
430 .i32_ty
431 .get_context()
432 .struct_type(llvm_results.as_slice(), false);
433
434 let struct_value =
435 err!(builder.build_load(struct_type, sret, "")).into_struct_value();
436 let mut rets: Vec<_> = Vec::new();
437 for i in 0..struct_value.get_type().count_fields() {
438 let value = err!(builder.build_extract_value(struct_value, i, ""));
439 rets.push(value);
440 }
441 assert!(func_sig.results().len() == rets.len());
442 Ok(rets)
443 } else {
444 assert!(func_sig.results().is_empty());
445 Ok(vec![])
446 }
447 }
448 }
449
450 fn pack_values_for_register_return<'ctx>(
451 &self,
452 intrinsics: &Intrinsics<'ctx>,
453 builder: &Builder<'ctx>,
454 values: &[BasicValueEnum<'ctx>],
455 func_type: &FunctionType<'ctx>,
456 ) -> Result<BasicValueEnum<'ctx>, CompileError> {
457 let is_32 = |value: BasicValueEnum| {
458 (value.is_int_value() && value.into_int_value().get_type() == intrinsics.i32_ty)
459 || (value.is_float_value()
460 && value.into_float_value().get_type() == intrinsics.f32_ty)
461 };
462 let is_64 = |value: BasicValueEnum| {
463 value.is_pointer_value()
464 || (value.is_int_value() && value.into_int_value().get_type() == intrinsics.i64_ty)
465 || (value.is_float_value()
466 && value.into_float_value().get_type() == intrinsics.f64_ty)
467 };
468
469 let pack_i32s = |low: BasicValueEnum<'ctx>, high: BasicValueEnum<'ctx>| {
470 assert!(low.get_type() == intrinsics.i32_ty.as_basic_type_enum());
471 assert!(high.get_type() == intrinsics.i32_ty.as_basic_type_enum());
472 let (low, high) = (low.into_int_value(), high.into_int_value());
473 let low = err!(builder.build_int_z_extend(low, intrinsics.i64_ty, ""));
474 let high = err!(builder.build_int_z_extend(high, intrinsics.i64_ty, ""));
475 let high =
476 err!(builder.build_left_shift(high, intrinsics.i64_ty.const_int(32, false), ""));
477 err_nt!(
478 builder
479 .build_or(low, high, "")
480 .map(|v| v.as_basic_value_enum())
481 )
482 };
483
484 let to_i64 = |v: BasicValueEnum<'ctx>| -> Result<BasicValueEnum<'_>, CompileError> {
485 if v.is_float_value() {
486 let v = v.into_float_value();
487 if v.get_type() == intrinsics.f32_ty {
488 let v = err!(builder.build_bit_cast(v, intrinsics.i32_ty, "")).into_int_value();
489 let v = err!(builder.build_int_z_extend(v, intrinsics.i64_ty, ""));
490 Ok(v.as_basic_value_enum())
491 } else {
492 debug_assert!(v.get_type() == intrinsics.f64_ty);
493 let v = err!(builder.build_bit_cast(v, intrinsics.i64_ty, ""));
494 Ok(v.as_basic_value_enum())
495 }
496 } else if v.is_pointer_value() {
497 let v =
498 err!(builder.build_ptr_to_int(v.into_pointer_value(), intrinsics.i64_ty, "",));
499 Ok(v.as_basic_value_enum())
500 } else {
501 let v = v.into_int_value();
502 if v.get_type() == intrinsics.i32_ty {
503 let v = err!(builder.build_int_z_extend(v, intrinsics.i64_ty, ""));
504 Ok(v.as_basic_value_enum())
505 } else {
506 debug_assert!(v.get_type() == intrinsics.i64_ty);
507 Ok(v.as_basic_value_enum())
508 }
509 }
510 };
511
512 let build_struct = |ty: StructType<'ctx>,
513 values: &[BasicValueEnum<'ctx>]|
514 -> Result<BasicValueEnum<'_>, CompileError> {
515 let mut struct_value = ty.get_undef();
516 for (i, v) in values.iter().enumerate() {
517 struct_value = err!(builder.build_insert_value(struct_value, *v, i as u32, ""))
518 .into_struct_value();
519 }
520 Ok(struct_value.as_basic_value_enum())
521 };
522
523 let build_2xi64 = |low: BasicValueEnum<'ctx>,
524 high: BasicValueEnum<'ctx>|
525 -> Result<BasicValueEnum<'_>, CompileError> {
526 let low = to_i64(low)?;
527 let high = to_i64(high)?;
528 let value = intrinsics.i64_ty.array_type(2).get_undef();
529 let value = err!(builder.build_insert_value(value, low, 0, ""));
530 let value = err!(builder.build_insert_value(value, high, 1, ""));
531 Ok(value.as_basic_value_enum())
532 };
533
534 Ok(match *values {
535 [one_value] => one_value,
536 [v1, v2]
537 if v1.is_float_value()
538 && v2.is_float_value()
539 && v1.into_float_value().get_type() == v2.into_float_value().get_type() =>
540 {
541 build_struct(
542 func_type.get_return_type().unwrap().into_struct_type(),
543 &[v1, v2],
544 )?
545 }
546 [v1, v2] if is_32(v1) && is_32(v2) => {
547 let v1 = err!(builder.build_bit_cast(v1, intrinsics.i32_ty, ""));
548 let v2 = err!(builder.build_bit_cast(v2, intrinsics.i32_ty, ""));
549 pack_i32s(v1, v2)?
550 }
551 [v1, v2] => build_2xi64(v1, v2)?,
552 [v1, v2, v3]
553 if is_32(v1)
554 && is_32(v2)
555 && is_32(v3)
556 && v1.is_float_value()
557 && v2.is_float_value()
558 && v3.is_float_value() =>
559 {
560 build_struct(
561 func_type.get_return_type().unwrap().into_struct_type(),
562 &[v1, v2, v3],
563 )?
564 }
565 [v1, v2, v3] if is_32(v1) && is_32(v2) => {
566 let v1 = err!(builder.build_bit_cast(v1, intrinsics.i32_ty, ""));
567 let v2 = err!(builder.build_bit_cast(v2, intrinsics.i32_ty, ""));
568 let v1v2_pack = pack_i32s(v1, v2)?;
569 build_2xi64(v1v2_pack, v3)?
570 }
571 [v1, v2, v3] if is_64(v1) && is_32(v2) && is_32(v3) => {
572 let v2 = err!(builder.build_bit_cast(v2, intrinsics.i32_ty, ""));
573 let v3 = err!(builder.build_bit_cast(v3, intrinsics.i32_ty, ""));
574 let v2v3_pack = pack_i32s(v2, v3)?;
575 build_2xi64(v1, v2v3_pack)?
576 }
577 [v1, v2, v3, v4]
578 if is_32(v1)
579 && is_32(v2)
580 && is_32(v3)
581 && is_32(v4)
582 && v1.is_float_value()
583 && v2.is_float_value()
584 && v3.is_float_value()
585 && v4.is_float_value() =>
586 {
587 build_struct(
588 func_type.get_return_type().unwrap().into_struct_type(),
589 &[v1, v2, v3, v4],
590 )?
591 }
592 [v1, v2, v3, v4] if is_32(v1) && is_32(v2) && is_32(v3) && is_32(v4) => {
593 let v1 = err!(builder.build_bit_cast(v1, intrinsics.i32_ty, ""));
594 let v2 = err!(builder.build_bit_cast(v2, intrinsics.i32_ty, ""));
595 let v1v2_pack = pack_i32s(v1, v2)?;
596 let v3 = err!(builder.build_bit_cast(v3, intrinsics.i32_ty, ""));
597 let v4 = err!(builder.build_bit_cast(v4, intrinsics.i32_ty, ""));
598 let v3v4_pack = pack_i32s(v3, v4)?;
599 build_2xi64(v1v2_pack, v3v4_pack)?
600 }
601 _ => {
602 unreachable!("called to perform register return on struct return or void function")
603 }
604 })
605 }
606}