wasmer_compiler_cranelift/
abi.rs

1//! Native return-value ABI lowering for Cranelift.
2
3use cranelift_codegen::{
4    ir::{
5        self, AbiParam, ArgumentPurpose, InstBuilder, MemFlagsData, StackSlotData, StackSlotKind,
6    },
7    isa::TargetFrontendConfig,
8};
9use cranelift_frontend::FunctionBuilder;
10use smallvec::{SmallVec, smallvec};
11use target_lexicon::Architecture;
12use wasmer_compiler::abi::{
13    PairSlot, ReturnAbi, ReturnSlot, classify_return_type_aarch64, classify_return_type_riscv64,
14    classify_return_type_x86_64,
15};
16use wasmer_types::{FunctionType, Type};
17
18use crate::translator::type_to_irtype;
19
20/// Classify return values for a target architecture.
21pub(crate) fn classify_returns(arch: Architecture, types: &[Type]) -> ReturnAbi {
22    match arch {
23        Architecture::X86_64 => classify_return_type_x86_64(types),
24        Architecture::Aarch64(_) => classify_return_type_aarch64(types),
25        Architecture::Riscv64(_) => classify_return_type_riscv64(types),
26        _ => unreachable!("unexpected architecture: {arch}"),
27    }
28}
29
30fn natural_type(ty: Type, config: TargetFrontendConfig) -> ir::Type {
31    type_to_irtype(ty, config).expect("supported WebAssembly signature type")
32}
33
34fn slot_type(slot: ReturnSlot, config: TargetFrontendConfig) -> ir::Type {
35    match slot {
36        ReturnSlot::Natural(ty) => natural_type(ty, config),
37        ReturnSlot::Raw(Type::F32) => ir::types::I32,
38        ReturnSlot::Raw(Type::F64) => ir::types::I64,
39        ReturnSlot::Raw(ty) => natural_type(ty, config),
40    }
41}
42
43fn pair_type(pair: PairSlot) -> ir::Type {
44    match pair {
45        PairSlot::F32Vector(_, _) => ir::types::F32X2,
46        PairSlot::Raw(_, _) => ir::types::I64,
47    }
48}
49
50/// Lower a WebAssembly signature to the native signature described by `ReturnAbi`.
51pub(crate) fn signature_to_ir(
52    signature: &FunctionType,
53    config: TargetFrontendConfig,
54    arch: Architecture,
55) -> ir::Signature {
56    let return_abi = classify_returns(arch, signature.results());
57    let mut sig = ir::Signature::new(config.default_call_conv);
58
59    if matches!(return_abi, ReturnAbi::Sret(_)) {
60        sig.params.push(AbiParam::special(
61            config.pointer_type(),
62            ArgumentPurpose::StructReturn,
63        ));
64    }
65    sig.params.push(AbiParam::special(
66        config.pointer_type(),
67        ArgumentPurpose::VMContext,
68    ));
69    sig.params.extend(
70        signature
71            .params()
72            .iter()
73            .map(|&ty| AbiParam::new(natural_type(ty, config))),
74    );
75
76    match return_abi {
77        ReturnAbi::Void => {}
78        ReturnAbi::Single(ty) => sig.returns.push(AbiParam::new(natural_type(ty, config))),
79        ReturnAbi::Pair(a, b) => {
80            sig.returns.push(AbiParam::new(slot_type(a, config)));
81            sig.returns.push(AbiParam::new(slot_type(b, config)));
82        }
83        ReturnAbi::PackedPair(pair) => sig.returns.push(AbiParam::new(pair_type(pair))),
84        ReturnAbi::PackedFirst(pair, slot) => {
85            sig.returns.push(AbiParam::new(pair_type(pair)));
86            sig.returns.push(AbiParam::new(slot_type(slot, config)));
87        }
88        ReturnAbi::PackedLast(slot, pair) => {
89            sig.returns.push(AbiParam::new(slot_type(slot, config)));
90            sig.returns.push(AbiParam::new(pair_type(pair)));
91        }
92        ReturnAbi::PackedQuads(a, b) => {
93            sig.returns.push(AbiParam::new(pair_type(a)));
94            sig.returns.push(AbiParam::new(pair_type(b)));
95        }
96        ReturnAbi::Unpacked(types) => sig.returns.extend(
97            types
98                .into_iter()
99                .map(|ty| AbiParam::new(natural_type(ty, config))),
100        ),
101        // sret got already added as the very first argument
102        ReturnAbi::Sret(_) => {}
103    }
104    sig
105}
106
107fn bitcast(builder: &mut FunctionBuilder, ty: ir::Type, value: ir::Value) -> ir::Value {
108    if builder.func.dfg.value_type(value) == ty {
109        value
110    } else {
111        let mut flags = MemFlagsData::new();
112        flags.set_endianness(ir::Endianness::Little);
113        builder.ins().bitcast(ty, flags, value)
114    }
115}
116
117fn pack_slot(builder: &mut FunctionBuilder, value: ir::Value, slot: ReturnSlot) -> ir::Value {
118    match slot {
119        ReturnSlot::Natural(Type::V128) => bitcast(builder, ir::types::I8X16, value),
120        ReturnSlot::Natural(_) => value,
121        ReturnSlot::Raw(Type::F32) => bitcast(builder, ir::types::I32, value),
122        ReturnSlot::Raw(Type::F64) => bitcast(builder, ir::types::I64, value),
123        ReturnSlot::Raw(_) => value,
124    }
125}
126
127fn unpack_slot(builder: &mut FunctionBuilder, value: ir::Value, slot: ReturnSlot) -> ir::Value {
128    match slot {
129        ReturnSlot::Natural(_) => value,
130        ReturnSlot::Raw(Type::F32) => bitcast(builder, ir::types::F32, value),
131        ReturnSlot::Raw(Type::F64) => bitcast(builder, ir::types::F64, value),
132        ReturnSlot::Raw(_) => value,
133    }
134}
135
136fn pack_pair(
137    builder: &mut FunctionBuilder,
138    first: ir::Value,
139    second: ir::Value,
140    pair: PairSlot,
141) -> ir::Value {
142    match pair {
143        PairSlot::Raw(_, _) => {
144            let low = bitcast(builder, ir::types::I32, first);
145            let high = bitcast(builder, ir::types::I32, second);
146            let low = builder.ins().uextend(ir::types::I64, low);
147            let high = builder.ins().uextend(ir::types::I64, high);
148            let high = builder.ins().ishl_imm_u(high, 32);
149            builder.ins().bor(low, high)
150        }
151        PairSlot::F32Vector(_, _) => {
152            let low = bitcast(builder, ir::types::I32, first);
153            let high = bitcast(builder, ir::types::I32, second);
154            let low = builder.ins().uextend(ir::types::I64, low);
155            let high = builder.ins().uextend(ir::types::I64, high);
156            let high = builder.ins().ishl_imm_u(high, 32);
157            let bits = builder.ins().bor(low, high);
158            bitcast(builder, ir::types::F32X2, bits)
159        }
160    }
161}
162
163/// Layout of the explicit structure-return area.
164#[derive(Clone, Debug)]
165pub(crate) struct ReturnAreaLayout {
166    /// Byte offset of each result.
167    pub(crate) offsets: Vec<i32>,
168    /// Total allocation size.
169    pub(crate) size: u32,
170    /// Base-two logarithm of the allocation alignment.
171    pub(crate) align_shift: u8,
172}
173
174fn type_size(ty: Type) -> u32 {
175    match ty {
176        Type::I32 | Type::F32 | Type::ExceptionRef => 4,
177        Type::I64 | Type::F64 => 8,
178        // Only 64-bit architectures are supported by Cranelift.
179        Type::ExternRef | Type::FuncRef => 8,
180        Type::V128 => 16,
181    }
182}
183
184/// Compute the natural struct layout used for an explicit return area.
185pub(crate) fn return_area_layout(types: &[Type]) -> ReturnAreaLayout {
186    let mut offset = 0u32;
187    let mut align = 1u32;
188    let mut offsets = Vec::with_capacity(types.len());
189    for &ty in types {
190        let size = type_size(ty);
191        align = align.max(size);
192        offset = offset.next_multiple_of(size);
193        offsets.push(i32::try_from(offset).unwrap());
194        offset += size;
195    }
196    let size = offset.next_multiple_of(align);
197    ReturnAreaLayout {
198        offsets,
199        size,
200        align_shift: align.trailing_zeros() as u8,
201    }
202}
203
204/// Allocate an explicit return area and return its address and layout.
205pub(crate) fn allocate_return_area(
206    builder: &mut FunctionBuilder,
207    types: &[Type],
208    config: TargetFrontendConfig,
209) -> (ir::Value, ReturnAreaLayout) {
210    let layout = return_area_layout(types);
211    let slot = builder.create_sized_stack_slot(StackSlotData::new(
212        StackSlotKind::ExplicitSlot,
213        layout.size,
214        layout.align_shift,
215    ));
216    let ptr = builder.ins().stack_addr(config.pointer_type(), slot, 0);
217    (ptr, layout)
218}
219
220/// Store natural Wasm return values in an explicit return area.
221pub(crate) fn store_sret(
222    builder: &mut FunctionBuilder,
223    ptr: ir::Value,
224    layout: &ReturnAreaLayout,
225    values: &[ir::Value],
226) {
227    let flags = MemFlagsData::trusted();
228    for (&value, &offset) in values.iter().zip(&layout.offsets) {
229        builder.ins().store(flags, value, ptr, offset);
230    }
231}
232
233/// Load natural Wasm return values from an explicit return area.
234pub(crate) fn load_sret(
235    builder: &mut FunctionBuilder,
236    ptr: ir::Value,
237    layout: &ReturnAreaLayout,
238    types: &[Type],
239    config: TargetFrontendConfig,
240) -> SmallVec<[ir::Value; 4]> {
241    let flags = MemFlagsData::trusted();
242    types
243        .iter()
244        .zip(&layout.offsets)
245        .map(|(&ty, &offset)| {
246            builder
247                .ins()
248                .load(natural_type(ty, config), flags, ptr, offset)
249        })
250        .collect()
251}
252
253/// Pack natural Wasm values into native register-return carriers.
254pub(crate) fn pack_register_returns(
255    builder: &mut FunctionBuilder,
256    abi: &ReturnAbi,
257    values: &[ir::Value],
258) -> SmallVec<[ir::Value; 4]> {
259    match abi {
260        ReturnAbi::Void => SmallVec::new(),
261        ReturnAbi::Single(ty) => smallvec![if *ty == Type::V128 {
262            bitcast(builder, ir::types::I8X16, values[0])
263        } else {
264            values[0]
265        }],
266        ReturnAbi::Pair(a, b) => smallvec![
267            pack_slot(builder, values[0], *a),
268            pack_slot(builder, values[1], *b)
269        ],
270        ReturnAbi::PackedPair(pair) => {
271            smallvec![pack_pair(builder, values[0], values[1], *pair)]
272        }
273        ReturnAbi::PackedFirst(pair, slot) => smallvec![
274            pack_pair(builder, values[0], values[1], *pair),
275            pack_slot(builder, values[2], *slot)
276        ],
277        ReturnAbi::PackedLast(slot, pair) => smallvec![
278            pack_slot(builder, values[0], *slot),
279            pack_pair(builder, values[1], values[2], *pair)
280        ],
281        ReturnAbi::PackedQuads(a, b) => smallvec![
282            pack_pair(builder, values[0], values[1], *a),
283            pack_pair(builder, values[2], values[3], *b)
284        ],
285        ReturnAbi::Unpacked(_) => values.iter().copied().collect(),
286        ReturnAbi::Sret(_) => panic!("sret values must be stored, not packed"),
287    }
288}
289
290/// Unpack native register-return carriers into natural Wasm values.
291pub(crate) fn unpack_register_returns(
292    builder: &mut FunctionBuilder,
293    abi: &ReturnAbi,
294    values: &[ir::Value],
295    config: TargetFrontendConfig,
296) -> SmallVec<[ir::Value; 4]> {
297    let unpack_pair_with_config = |builder: &mut FunctionBuilder, value, pair| match pair {
298        PairSlot::Raw(first, second) => {
299            let low = builder.ins().ireduce(ir::types::I32, value);
300            let high = builder.ins().ushr_imm_u(value, 32);
301            let high = builder.ins().ireduce(ir::types::I32, high);
302            (
303                bitcast(builder, natural_type(first, config), low),
304                bitcast(builder, natural_type(second, config), high),
305            )
306        }
307        PairSlot::F32Vector(_, _) => {
308            let bits = bitcast(builder, ir::types::I64, value);
309            let low = builder.ins().ireduce(ir::types::I32, bits);
310            let high = builder.ins().ushr_imm_u(bits, 32);
311            let high = builder.ins().ireduce(ir::types::I32, high);
312            (
313                bitcast(builder, ir::types::F32, low),
314                bitcast(builder, ir::types::F32, high),
315            )
316        }
317    };
318    match abi {
319        ReturnAbi::Void => SmallVec::new(),
320        ReturnAbi::Single(_) => smallvec![values[0]],
321        ReturnAbi::Pair(a, b) => smallvec![
322            unpack_slot(builder, values[0], *a),
323            unpack_slot(builder, values[1], *b)
324        ],
325        ReturnAbi::PackedPair(pair) => {
326            let (a, b) = unpack_pair_with_config(builder, values[0], *pair);
327            smallvec![a, b]
328        }
329        ReturnAbi::PackedFirst(pair, slot) => {
330            let (a, b) = unpack_pair_with_config(builder, values[0], *pair);
331            smallvec![a, b, unpack_slot(builder, values[1], *slot)]
332        }
333        ReturnAbi::PackedLast(slot, pair) => {
334            let (a, b) = unpack_pair_with_config(builder, values[1], *pair);
335            smallvec![unpack_slot(builder, values[0], *slot), a, b]
336        }
337        ReturnAbi::PackedQuads(a, b) => {
338            let (a0, a1) = unpack_pair_with_config(builder, values[0], *a);
339            let (b0, b1) = unpack_pair_with_config(builder, values[1], *b);
340            smallvec![a0, a1, b0, b1]
341        }
342        ReturnAbi::Unpacked(_) => values.iter().copied().collect(),
343        ReturnAbi::Sret(_) => panic!("sret values must be loaded, not unpacked"),
344    }
345}
wasmer_compiler_cranelift/abi.rs

wasmer_compiler_cranelift/
abi.rs
