wasmer_compiler_singlepass/
emitter_riscv.rs

1//! RISC-V emitter scaffolding.
2
3use crate::{
4    codegen_error,
5    common_decl::Size,
6    location::{Location as AbstractLocation, Reg},
7    machine::MaybeImmediate,
8    machine_riscv::{ImmType, RISCV_RETURN_VALUE_REGISTERS},
9    riscv_decl::{ArgumentRegisterAllocator, RiscvRegister},
10};
11pub use crate::{
12    machine::{Label, Offset},
13    riscv_decl::{FPR, GPR},
14};
15use dynasmrt::{DynasmApi, DynasmLabelApi, VecAssembler, riscv::RiscvRelocation};
16use wasmer_compiler::types::{
17    function::FunctionBody,
18    section::{CustomSection, CustomSectionProtection, SectionBody},
19};
20use wasmer_types::{
21    CompileError, FunctionIndex, FunctionType, Type, VMOffsets,
22    target::{CallingConvention, CpuFeature},
23};
24
25type Assembler = VecAssembler<RiscvRelocation>;
26
27/// Force `dynasm!` to use the correct arch (riscv64) when cross-compiling.
28macro_rules! dynasm {
29    ($a:expr ; $($tt:tt)*) => {
30        dynasm::dynasm!(
31            $a
32            ; .arch riscv64
33            ; .feature g
34            ; $($tt)*
35        )
36    };
37}
38
39/// Location abstraction specialized to RISC-V.
40pub type Location = AbstractLocation<GPR, FPR>;
41
42/// Branch conditions for RISC-V.
43#[derive(Copy, Clone, Debug, Eq, PartialEq)]
44pub enum Condition {
45    /// Signed less than
46    Lt,
47    /// Signed less than or equal
48    Le,
49    /// Signed greater than
50    Gt,
51    /// Signed greater than or equal
52    Ge,
53    /// Signed equal
54    Eq,
55    /// Signed not equal
56    Ne,
57
58    /// Unsigned less than
59    Ltu,
60    /// Unsigned less than or equal
61    Leu,
62    /// Unsigned greater than
63    Gtu,
64    /// Unsigned greater than or equal
65    Geu,
66}
67
68/// Floating-point number rounding mode
69#[derive(Copy, Clone, Debug, Eq, PartialEq)]
70pub enum RoundingMode {
71    /// Round to Nearest, ties to Even (default)
72    Rne,
73    /// Round towards Zero
74    Rtz,
75    /// Round Down (towards -∞)
76    Rdn,
77    /// Round Up (towards +∞)
78    Rup,
79}
80
81/// Atomic binary operation type
82#[derive(Copy, Clone, Debug, Eq, PartialEq)]
83pub enum AtomicBinaryOp {
84    Add,
85    Sub,
86    Or,
87    And,
88    Xor,
89    Exchange,
90}
91
92/// Scratch register used in function call trampolines.
93const SCRATCH_REG: GPR = GPR::X28;
94
95/// Emitter trait for RISC-V.
96#[allow(unused)]
97pub trait EmitterRiscv {
98    fn emit_jmp_on_condition(
99        &mut self,
100        cond: Condition,
101        loc_a: Location,
102        loc_b: Location,
103        label: Label,
104        tmp: GPR,
105    ) -> Result<(), CompileError>;
106    fn emit_reserved_sd(
107        &mut self,
108        size: Size,
109        dest: GPR,
110        addr: GPR,
111        source: GPR,
112    ) -> Result<(), CompileError>;
113    /// Returns the SIMD (FPU) feature if available.
114    fn get_simd_arch(&self) -> Option<&CpuFeature>;
115    /// Generates a new internal label.
116    fn get_label(&mut self) -> Label;
117    /// Gets the current code offset.
118    fn get_offset(&self) -> Offset;
119    /// Returns the size of a jump instruction in bytes.
120    fn get_jmp_instr_size(&self) -> u8;
121
122    /// Finalize the function, e.g., resolve labels.
123    fn finalize_function(&mut self) -> Result<(), CompileError>;
124
125    fn emit_label(&mut self, label: Label) -> Result<(), CompileError>;
126    fn emit_brk(&mut self) -> Result<(), CompileError>;
127    fn emit_ld(
128        &mut self,
129        sz: Size,
130        signed: bool,
131        reg: Location,
132        addr: Location,
133    ) -> Result<(), CompileError>;
134
135    fn emit_sd(&mut self, sz: Size, reg: Location, addr: Location) -> Result<(), CompileError>;
136    fn emit_push(&mut self, size: Size, src: Location) -> Result<(), CompileError>;
137    fn emit_pop(&mut self, size: Size, dst: Location) -> Result<(), CompileError>;
138
139    fn emit_add(
140        &mut self,
141        sz: Size,
142        src1: Location,
143        src2: Location,
144        dst: Location,
145    ) -> Result<(), CompileError>;
146    fn emit_sub(
147        &mut self,
148        sz: Size,
149        src1: Location,
150        src2: Location,
151        dst: Location,
152    ) -> Result<(), CompileError>;
153    fn emit_mul(
154        &mut self,
155        sz: Size,
156        src1: Location,
157        src2: Location,
158        dst: Location,
159    ) -> Result<(), CompileError>;
160    fn emit_sdiv(
161        &mut self,
162        sz: Size,
163        src1: Location,
164        src2: Location,
165        dst: Location,
166    ) -> Result<(), CompileError>;
167    fn emit_udiv(
168        &mut self,
169        sz: Size,
170        src1: Location,
171        src2: Location,
172        dst: Location,
173    ) -> Result<(), CompileError>;
174    fn emit_srem(
175        &mut self,
176        sz: Size,
177        src1: Location,
178        src2: Location,
179        dst: Location,
180    ) -> Result<(), CompileError>;
181    fn emit_urem(
182        &mut self,
183        sz: Size,
184        src1: Location,
185        src2: Location,
186        dst: Location,
187    ) -> Result<(), CompileError>;
188    fn emit_and(
189        &mut self,
190        sz: Size,
191        src1: Location,
192        src2: Location,
193        dst: Location,
194    ) -> Result<(), CompileError>;
195    fn emit_or(
196        &mut self,
197        sz: Size,
198        src1: Location,
199        src2: Location,
200        dst: Location,
201    ) -> Result<(), CompileError>;
202    fn emit_xor(
203        &mut self,
204        sz: Size,
205        src1: Location,
206        src2: Location,
207        dst: Location,
208    ) -> Result<(), CompileError>;
209    fn emit_sll(
210        &mut self,
211        sz: Size,
212        src1: Location,
213        src2: Location,
214        dst: Location,
215    ) -> Result<(), CompileError>;
216    fn emit_srl(
217        &mut self,
218        sz: Size,
219        src1: Location,
220        src2: Location,
221        dst: Location,
222    ) -> Result<(), CompileError>;
223    fn emit_sra(
224        &mut self,
225        sz: Size,
226        src1: Location,
227        src2: Location,
228        dst: Location,
229    ) -> Result<(), CompileError>;
230    fn emit_extend(
231        &mut self,
232        sz: Size,
233        signed: bool,
234        src: Location,
235        dst: Location,
236    ) -> Result<(), CompileError>;
237    fn emit_not(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
238    fn emit_neg(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
239
240    fn emit_mov(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
241
242    fn emit_ret(&mut self) -> Result<(), CompileError>;
243
244    fn emit_udf(&mut self, payload: u8) -> Result<(), CompileError>;
245
246    fn emit_mov_imm(&mut self, dst: Location, val: i64) -> Result<(), CompileError>;
247
248    // Floating-point type instructions
249    fn emit_fneg(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
250    fn emit_fmin(
251        &mut self,
252        sz: Size,
253        src1: Location,
254        src2: Location,
255        dst: Location,
256    ) -> Result<(), CompileError>;
257    fn emit_fmax(
258        &mut self,
259        sz: Size,
260        src1: Location,
261        src2: Location,
262        dst: Location,
263    ) -> Result<(), CompileError>;
264    fn emit_fdiv(
265        &mut self,
266        sz: Size,
267        src1: Location,
268        src2: Location,
269        dst: Location,
270    ) -> Result<(), CompileError>;
271    fn emit_fsqrt(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
272    fn emit_fcvt(
273        &mut self,
274        signed: bool,
275        sz_in: Size,
276        src: Location,
277        sz_out: Size,
278        dst: Location,
279    ) -> Result<(), CompileError>;
280    fn emit_fcvt_with_rounding(
281        &mut self,
282        rounding: RoundingMode,
283        size: Size,
284        src: Location,
285        dst: Location,
286        tmp: GPR,
287    ) -> Result<(), CompileError>;
288    fn emit_swap_fscr(&mut self, reg: GPR) -> Result<(), CompileError>;
289
290    // Control-flow related instructions
291    fn emit_cmp(
292        &mut self,
293        c: Condition,
294        loc_a: Location,
295        loc_b: Location,
296        ret: Location,
297    ) -> Result<(), CompileError>;
298    fn emit_fcmp(
299        &mut self,
300        c: Condition,
301        size: Size,
302        loc_a: Location,
303        loc_b: Location,
304        ret: Location,
305    ) -> Result<(), CompileError>;
306
307    fn emit_on_false_label(
308        &mut self,
309        cond: Location,
310        label: Label,
311        tmp: GPR,
312    ) -> Result<(), CompileError>;
313    fn emit_on_false_label_far(
314        &mut self,
315        cond: Location,
316        label: Label,
317        tmp: GPR,
318    ) -> Result<(), CompileError>;
319    fn emit_on_true_label_far(
320        &mut self,
321        cond: Location,
322        label: Label,
323        tmp: GPR,
324    ) -> Result<(), CompileError>;
325    fn emit_on_true_label(
326        &mut self,
327        cond: Location,
328        label: Label,
329        tmp: GPR,
330    ) -> Result<(), CompileError>;
331
332    fn emit_j_label(&mut self, label: Label, reg: Option<GPR>) -> Result<(), CompileError>;
333    fn emit_j_register(&mut self, reg: GPR) -> Result<(), CompileError>;
334    fn emit_load_label(&mut self, reg: GPR, label: Label) -> Result<(), CompileError>;
335    fn emit_call_label(&mut self, label: Label) -> Result<(), CompileError>;
336    fn emit_call_register(&mut self, reg: GPR) -> Result<(), CompileError>;
337
338    fn emit_rwfence(&mut self) -> Result<(), CompileError>;
339    fn emit_atomic_binop(
340        &mut self,
341        op: AtomicBinaryOp,
342        size: Size,
343        dest: GPR,
344        addr: GPR,
345        source: GPR,
346    ) -> Result<(), CompileError>;
347    fn emit_reserved_ld(&mut self, size: Size, reg: GPR, addr: GPR) -> Result<(), CompileError>;
348}
349
350impl EmitterRiscv for Assembler {
351    fn get_simd_arch(&self) -> Option<&CpuFeature> {
352        todo!()
353    }
354
355    fn get_label(&mut self) -> Label {
356        self.new_dynamic_label()
357    }
358
359    fn get_offset(&self) -> Offset {
360        self.offset()
361    }
362
363    fn get_jmp_instr_size(&self) -> u8 {
364        todo!()
365    }
366
367    fn finalize_function(&mut self) -> Result<(), CompileError> {
368        Ok(())
369    }
370
371    fn emit_label(&mut self, label: Label) -> Result<(), CompileError> {
372        dynasm!(self ; => label);
373        Ok(())
374    }
375
376    fn emit_brk(&mut self) -> Result<(), CompileError> {
377        dynasm!(self ; ebreak);
378        Ok(())
379    }
380
381    fn emit_udf(&mut self, payload: u8) -> Result<(), CompileError> {
382        dynasm!(self
383            ; li.12 a0, payload as _
384            ; unimp);
385        Ok(())
386    }
387
388    fn emit_ld(
389        &mut self,
390        sz: Size,
391        signed: bool,
392        reg: Location,
393        addr: Location,
394    ) -> Result<(), CompileError> {
395        let Location::Memory(addr, disp) = addr else {
396            codegen_error!("singlepass can't emit LD {:?}, {:?}, {:?}", sz, reg, addr);
397        };
398        assert!(ImmType::Bits12.compatible_imm(disp as i64));
399
400        match (sz, signed, reg) {
401            (Size::S64, _, Location::GPR(reg)) => {
402                dynasm!(self ; ld X(reg), [X(addr), disp]);
403            }
404            (Size::S32, false, Location::GPR(reg)) => {
405                dynasm!(self ; lwu X(reg), [X(addr), disp]);
406            }
407            (Size::S32, true, Location::GPR(reg)) => {
408                dynasm!(self ; lw X(reg), [X(addr), disp]);
409            }
410            (Size::S16, false, Location::GPR(reg)) => {
411                dynasm!(self ; lhu X(reg), [X(addr), disp]);
412            }
413            (Size::S16, true, Location::GPR(reg)) => {
414                dynasm!(self ; lh X(reg), [X(addr), disp]);
415            }
416            (Size::S8, false, Location::GPR(reg)) => {
417                dynasm!(self ; lbu X(reg), [X(addr), disp]);
418            }
419            (Size::S8, true, Location::GPR(reg)) => {
420                dynasm!(self ; lb X(reg), [X(addr), disp]);
421            }
422            (Size::S64, _, Location::SIMD(reg)) => {
423                dynasm!(self ; fld F(reg), [X(addr), disp]);
424            }
425            (Size::S32, _, Location::SIMD(reg)) => {
426                dynasm!(self ; flw F(reg), [X(addr), disp]);
427            }
428            _ => codegen_error!("singlepass can't emit LD {:?}, {:?}, {:?}", sz, reg, addr),
429        }
430        Ok(())
431    }
432
433    fn emit_sd(&mut self, sz: Size, reg: Location, addr: Location) -> Result<(), CompileError> {
434        let Location::Memory(addr, disp) = addr else {
435            codegen_error!("singlepass can't emit SD {:?}, {:?}, {:?}", sz, reg, addr);
436        };
437        assert!(ImmType::Bits12.compatible_imm(disp as i64));
438
439        match (sz, reg) {
440            (Size::S64, Location::GPR(reg)) => {
441                dynasm!(self ; sd X(reg), [X(addr), disp]);
442            }
443            (Size::S32, Location::GPR(reg)) => {
444                dynasm!(self ; sw X(reg), [X(addr), disp]);
445            }
446            (Size::S16, Location::GPR(reg)) => {
447                dynasm!(self ; sh X(reg), [X(addr), disp]);
448            }
449            (Size::S8, Location::GPR(reg)) => {
450                dynasm!(self ; sb X(reg), [X(addr), disp]);
451            }
452            (Size::S64, Location::SIMD(reg)) => {
453                dynasm!(self ; fsd F(reg), [X(addr), disp]);
454            }
455            (Size::S32, Location::SIMD(reg)) => {
456                dynasm!(self ; fsw F(reg), [X(addr), disp]);
457            }
458            _ => codegen_error!("singlepass can't emit SD {:?}, {:?}, {:?}", sz, reg, addr),
459        }
460        Ok(())
461    }
462
463    fn emit_push(&mut self, size: Size, src: Location) -> Result<(), CompileError> {
464        match (size, src) {
465            (Size::S64, Location::GPR(_)) => {
466                self.emit_sub(
467                    Size::S64,
468                    Location::GPR(GPR::Sp),
469                    Location::Imm64(8),
470                    Location::GPR(GPR::Sp),
471                )?;
472                self.emit_sd(Size::S64, src, Location::Memory(GPR::Sp, 0))?;
473            }
474            _ => codegen_error!("singlepass can't emit PUSH {:?} {:?}", size, src),
475        }
476        Ok(())
477    }
478
479    fn emit_pop(&mut self, size: Size, dst: Location) -> Result<(), CompileError> {
480        match (size, dst) {
481            (Size::S64, Location::GPR(_)) => {
482                self.emit_ld(Size::S64, false, dst, Location::Memory(GPR::Sp, 0))?;
483                self.emit_add(
484                    Size::S64,
485                    Location::GPR(GPR::Sp),
486                    Location::Imm64(8),
487                    Location::GPR(GPR::Sp),
488                )?;
489            }
490            _ => codegen_error!("singlepass can't emit POP {:?} {:?}", size, dst),
491        }
492        Ok(())
493    }
494
495    fn emit_add(
496        &mut self,
497        sz: Size,
498        src1: Location,
499        src2: Location,
500        dst: Location,
501    ) -> Result<(), CompileError> {
502        match (sz, src1, src2, dst) {
503            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
504                dynasm!(self ; add X(dst), X(src1), X(src2));
505            }
506            (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst))
507            | (Size::S64, Location::Imm64(imm), Location::GPR(src1), Location::GPR(dst)) => {
508                assert!(ImmType::Bits12.compatible_imm(imm as i64));
509                dynasm!(self ; addi X(dst), X(src1), imm as _);
510            }
511            (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
512            | (Size::S64, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
513                assert!(ImmType::Bits12.compatible_imm(imm as i64));
514                dynasm!(self ; addi X(dst), X(src1), imm as _);
515            }
516            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
517                dynasm!(self ; addw X(dst), X(src1), X(src2));
518            }
519            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
520            | (Size::S32, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
521                assert!(ImmType::Bits12.compatible_imm(imm as i64));
522                dynasm!(self ; addiw X(dst), X(src1), imm as _);
523            }
524
525            (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
526                dynasm!(self ; fadd.s F(dst), F(src1), F(src2));
527            }
528            (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
529                dynasm!(self ; fadd.d F(dst), F(src1), F(src2));
530            }
531            _ => codegen_error!(
532                "singlepass can't emit ADD {:?} {:?} {:?} {:?}",
533                sz,
534                src1,
535                src2,
536                dst
537            ),
538        }
539        Ok(())
540    }
541
542    fn emit_sub(
543        &mut self,
544        sz: Size,
545        src1: Location,
546        src2: Location,
547        dst: Location,
548    ) -> Result<(), CompileError> {
549        match (sz, src1, src2, dst) {
550            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
551                dynasm!(self ; sub X(dst), X(src1), X(src2));
552            }
553            (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
554                assert!(ImmType::Bits12Subtraction.compatible_imm(imm as i64));
555                dynasm!(self ; addi X(dst), X(src1), -(imm as i32) as _);
556            }
557            (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
558                assert!(ImmType::Bits12Subtraction.compatible_imm(imm as i64));
559                dynasm!(self ; addi X(dst), X(src1), -(imm as i32) as _);
560            }
561            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
562                dynasm!(self ; subw X(dst), X(src1), X(src2));
563            }
564            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
565                assert!(ImmType::Bits12Subtraction.compatible_imm(imm as i64));
566                dynasm!(self ; addiw X(dst), X(src1), -(imm as i32) as _);
567            }
568            (Size::S32, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
569                assert!(ImmType::Bits12Subtraction.compatible_imm(imm as i64));
570                dynasm!(self ; addiw X(dst), X(src1), -(imm as i32) as _);
571            }
572            (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
573                dynasm!(self ; fsub.s F(dst), F(src1), F(src2));
574            }
575            (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
576                dynasm!(self ; fsub.d F(dst), F(src1), F(src2));
577            }
578            _ => codegen_error!(
579                "singlepass can't emit SUB {:?} {:?} {:?} {:?}",
580                sz,
581                src1,
582                src2,
583                dst
584            ),
585        }
586        Ok(())
587    }
588
589    fn emit_mul(
590        &mut self,
591        sz: Size,
592        src1: Location,
593        src2: Location,
594        dst: Location,
595    ) -> Result<(), CompileError> {
596        match (sz, src1, src2, dst) {
597            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
598                dynasm!(self ; mul X(dst), X(src1), X(src2));
599            }
600            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
601                dynasm!(self ; mulw X(dst), X(src1), X(src2));
602            }
603
604            (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
605                dynasm!(self ; fmul.s F(dst), F(src1), F(src2));
606            }
607            (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
608                dynasm!(self ; fmul.d F(dst), F(src1), F(src2));
609            }
610
611            _ => codegen_error!(
612                "singlepass can't emit MUL {:?} {:?} {:?} {:?}",
613                sz,
614                src1,
615                src2,
616                dst
617            ),
618        }
619        Ok(())
620    }
621
622    fn emit_udiv(
623        &mut self,
624        sz: Size,
625        src1: Location,
626        src2: Location,
627        dst: Location,
628    ) -> Result<(), CompileError> {
629        match (sz, src1, src2, dst) {
630            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
631                dynasm!(self ; divuw X(dst), X(src1), X(src2));
632            }
633            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
634                dynasm!(self ; divu X(dst), X(src1), X(src2));
635            }
636            _ => codegen_error!(
637                "singlepass can't emit UDIV {:?} {:?} {:?} {:?}",
638                sz,
639                src1,
640                src2,
641                dst
642            ),
643        }
644        Ok(())
645    }
646    fn emit_sdiv(
647        &mut self,
648        sz: Size,
649        src1: Location,
650        src2: Location,
651        dst: Location,
652    ) -> Result<(), CompileError> {
653        match (sz, src1, src2, dst) {
654            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
655                dynasm!(self ; divw X(dst), X(src1), X(src2));
656            }
657            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
658                dynasm!(self ; div X(dst), X(src1), X(src2));
659            }
660            _ => codegen_error!(
661                "singlepass can't emit SDIV {:?} {:?} {:?} {:?}",
662                sz,
663                src1,
664                src2,
665                dst
666            ),
667        }
668        Ok(())
669    }
670
671    fn emit_urem(
672        &mut self,
673        sz: Size,
674        src1: Location,
675        src2: Location,
676        dst: Location,
677    ) -> Result<(), CompileError> {
678        match (sz, src1, src2, dst) {
679            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
680                dynasm!(self ; remuw X(dst), X(src1), X(src2));
681            }
682            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
683                dynasm!(self ; remu X(dst), X(src1), X(src2));
684            }
685            _ => codegen_error!(
686                "singlepass can't emit UREM {:?} {:?} {:?} {:?}",
687                sz,
688                src1,
689                src2,
690                dst
691            ),
692        }
693        Ok(())
694    }
695    fn emit_srem(
696        &mut self,
697        sz: Size,
698        src1: Location,
699        src2: Location,
700        dst: Location,
701    ) -> Result<(), CompileError> {
702        match (sz, src1, src2, dst) {
703            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
704                dynasm!(self ; remw X(dst), X(src1), X(src2));
705            }
706            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
707                dynasm!(self ; rem X(dst), X(src1), X(src2));
708            }
709            _ => codegen_error!(
710                "singlepass can't emit REM {:?} {:?} {:?} {:?}",
711                sz,
712                src1,
713                src2,
714                dst
715            ),
716        }
717        Ok(())
718    }
719
720    fn emit_and(
721        &mut self,
722        sz: Size,
723        src1: Location,
724        src2: Location,
725        dst: Location,
726    ) -> Result<(), CompileError> {
727        match (sz, src1, src2, dst) {
728            (
729                Size::S32 | Size::S64,
730                Location::GPR(src1),
731                Location::GPR(src2),
732                Location::GPR(dst),
733            ) => {
734                dynasm!(self ; and X(dst), X(src1), X(src2));
735            }
736            (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
737                assert!(ImmType::Bits12Subtraction.compatible_imm(imm as i64));
738                dynasm!(self ; andi X(dst), X(src1), imm as _);
739            }
740            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
741                assert!(ImmType::Bits12Subtraction.compatible_imm(imm as i64));
742                dynasm!(self ; andi X(dst), X(src1), imm as _);
743            }
744            _ => codegen_error!(
745                "singlepass can't emit AND {:?} {:?} {:?} {:?}",
746                sz,
747                src1,
748                src2,
749                dst
750            ),
751        }
752        Ok(())
753    }
754
755    fn emit_or(
756        &mut self,
757        sz: Size,
758        src1: Location,
759        src2: Location,
760        dst: Location,
761    ) -> Result<(), CompileError> {
762        match (sz, src1, src2, dst) {
763            (
764                Size::S32 | Size::S64,
765                Location::GPR(src1),
766                Location::GPR(src2),
767                Location::GPR(dst),
768            ) => {
769                dynasm!(self ; or X(dst), X(src1), X(src2));
770            }
771            (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
772                assert!(ImmType::Bits12Subtraction.compatible_imm(imm as i64));
773                dynasm!(self ; ori X(dst), X(src1), imm as _);
774            }
775            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
776                assert!(ImmType::Bits12Subtraction.compatible_imm(imm as i64));
777                dynasm!(self ; ori X(dst), X(src1), imm as _);
778            }
779            _ => codegen_error!(
780                "singlepass can't emit OR {:?} {:?} {:?} {:?}",
781                sz,
782                src1,
783                src2,
784                dst
785            ),
786        }
787        Ok(())
788    }
789
790    fn emit_xor(
791        &mut self,
792        sz: Size,
793        src1: Location,
794        src2: Location,
795        dst: Location,
796    ) -> Result<(), CompileError> {
797        match (sz, src1, src2, dst) {
798            (
799                Size::S32 | Size::S64,
800                Location::GPR(src1),
801                Location::GPR(src2),
802                Location::GPR(dst),
803            ) => {
804                dynasm!(self ; xor X(dst), X(src1), X(src2));
805            }
806            (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
807                assert!(ImmType::Bits12Subtraction.compatible_imm(imm as i64));
808                dynasm!(self ; xori X(dst), X(src1), imm as _);
809            }
810            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
811                assert!(ImmType::Bits12Subtraction.compatible_imm(imm as i64));
812                dynasm!(self ; xori X(dst), X(src1), imm as _);
813            }
814            _ => codegen_error!(
815                "singlepass can't emit XOR {:?} {:?} {:?} {:?}",
816                sz,
817                src1,
818                src2,
819                dst
820            ),
821        }
822        Ok(())
823    }
824
825    fn emit_sll(
826        &mut self,
827        sz: Size,
828        src1: Location,
829        src2: Location,
830        dst: Location,
831    ) -> Result<(), CompileError> {
832        match (sz, src1, src2, dst) {
833            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
834                dynasm!(self ; sll X(dst), X(src1), X(src2));
835            }
836            (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
837                if imm >= u64::BITS as _ {
838                    codegen_error!("singlepass SLL with incompatible imm {}", imm);
839                }
840                dynasm!(self ; slli X(dst), X(src1), imm as _);
841            }
842            (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
843                if imm >= u64::BITS as _ {
844                    codegen_error!("singlepass SLL with incompatible imm {}", imm);
845                }
846                dynasm!(self ; slli X(dst), X(src1), imm as _);
847            }
848            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
849                dynasm!(self ; sllw X(dst), X(src1), X(src2));
850            }
851            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
852                if imm >= u32::BITS {
853                    codegen_error!("singlepass SLL with incompatible imm {}", imm);
854                }
855                dynasm!(self ; slliw X(dst), X(src1), imm as _);
856            }
857            _ => codegen_error!(
858                "singlepass can't emit SLL {:?} {:?} {:?} {:?}",
859                sz,
860                src1,
861                src2,
862                dst
863            ),
864        }
865        Ok(())
866    }
867
868    fn emit_srl(
869        &mut self,
870        sz: Size,
871        src1: Location,
872        src2: Location,
873        dst: Location,
874    ) -> Result<(), CompileError> {
875        match (sz, src1, src2, dst) {
876            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
877                dynasm!(self ; srl X(dst), X(src1), X(src2));
878            }
879            (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
880                if imm >= u64::BITS as _ {
881                    codegen_error!("singlepass SRL with incompatible imm {}", imm);
882                }
883                dynasm!(self ; srli X(dst), X(src1), imm as _);
884            }
885            (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
886                if imm >= u64::BITS as _ {
887                    codegen_error!("singlepass SRL with incompatible imm {}", imm);
888                }
889                dynasm!(self ; srli X(dst), X(src1), imm as _);
890            }
891            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
892                dynasm!(self ; srlw X(dst), X(src1), X(src2));
893            }
894            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
895                if imm >= u32::BITS {
896                    codegen_error!("singlepass SRL with incompatible imm {}", imm);
897                }
898                dynasm!(self ; srliw X(dst), X(src1), imm as _);
899            }
900            (Size::S32, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
901                if imm >= u32::BITS as _ {
902                    codegen_error!("singlepass SRL with incompatible imm {}", imm);
903                }
904                dynasm!(self ; srliw X(dst), X(src1), imm as _);
905            }
906            (Size::S32, Location::Imm32(imm), Location::GPR(src2), Location::GPR(dst)) => {
907                self.emit_mov_imm(Location::GPR(dst), imm as i64)?;
908                dynasm!(self ; srlw X(dst), X(dst), X(src2));
909            }
910            _ => codegen_error!(
911                "singlepass can't emit SRL {:?} {:?} {:?} {:?}",
912                sz,
913                src1,
914                src2,
915                dst
916            ),
917        }
918        Ok(())
919    }
920
921    fn emit_sra(
922        &mut self,
923        sz: Size,
924        src1: Location,
925        src2: Location,
926        dst: Location,
927    ) -> Result<(), CompileError> {
928        match (sz, src1, src2, dst) {
929            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
930                dynasm!(self ; sra X(dst), X(src1), X(src2));
931            }
932            (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
933                if imm >= u64::BITS as _ {
934                    codegen_error!("singlepass SRA with incompatible imm {}", imm);
935                }
936                dynasm!(self ; srai X(dst), X(src1), imm as _);
937            }
938            (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
939                if imm >= u64::BITS as _ {
940                    codegen_error!("singlepass SRA with incompatible imm {}", imm);
941                }
942                dynasm!(self ; srai X(dst), X(src1), imm as _);
943            }
944            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
945                dynasm!(self ; sraw X(dst), X(src1), X(src2));
946            }
947            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
948                if imm >= u32::BITS {
949                    codegen_error!("singlepass SRA with incompatible imm {}", imm);
950                }
951                dynasm!(self ; sraiw X(dst), X(src1), imm as _);
952            }
953            _ => codegen_error!(
954                "singlepass can't emit SRA {:?} {:?} {:?} {:?}",
955                sz,
956                src1,
957                src2,
958                dst
959            ),
960        }
961        Ok(())
962    }
963
964    fn emit_extend(
965        &mut self,
966        sz: Size,
967        signed: bool,
968        src: Location,
969        dst: Location,
970    ) -> Result<(), CompileError> {
971        let bit_shift = match sz {
972            Size::S8 => 56,
973            Size::S16 => 48,
974            Size::S32 => 32,
975            _ => codegen_error!("singlepass can't emit SEXT {:?} {:?} {:?}", sz, src, dst),
976        };
977
978        match (signed, src, dst) {
979            (true, Location::GPR(src), Location::GPR(dst)) => {
980                dynasm!(self
981                    ; slli X(dst), X(src), bit_shift
982                    ; srai X(dst), X(dst), bit_shift
983                );
984            }
985            (false, Location::GPR(src), Location::GPR(dst)) => {
986                dynasm!(self
987                    ; slli X(dst), X(src), bit_shift
988                    ; srli X(dst), X(dst), bit_shift
989                );
990            }
991            _ => codegen_error!("singlepass can't emit SEXT {:?} {:?} {:?}", sz, src, dst),
992        };
993        Ok(())
994    }
995
996    fn emit_not(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
997        match (sz, src, dst) {
998            (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
999                dynasm!(self ; not X(dst), X(src));
1000            }
1001            _ => codegen_error!("singlepass can't emit NOT {:?} {:?} {:?}", sz, src, dst),
1002        }
1003
1004        Ok(())
1005    }
1006
1007    fn emit_neg(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1008        match (sz, src, dst) {
1009            (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1010                dynasm!(self ; neg X(dst), X(src));
1011            }
1012            _ => codegen_error!("singlepass can't emit NEG {:?} {:?} {:?}", sz, src, dst),
1013        }
1014
1015        Ok(())
1016    }
1017
1018    fn emit_mov(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1019        match (sz, src, dst) {
1020            (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1021                dynasm!(self ; mv X(dst), X(src));
1022            }
1023            (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1024                dynasm!(self
1025                    ; slli X(dst), X(src), 32
1026                    ; srli X(dst), X(dst), 32
1027                );
1028            }
1029            (Size::S64, Location::GPR(src), Location::SIMD(dst)) => {
1030                dynasm!(self ; fmv.d.x F(dst), X(src));
1031            }
1032            (Size::S64, Location::SIMD(src), Location::GPR(dst)) => {
1033                dynasm!(self ; fmv.x.d X(dst), F(src));
1034            }
1035            (Size::S32, Location::GPR(src), Location::SIMD(dst)) => {
1036                dynasm!(self ; fmv.s.x F(dst), X(src));
1037            }
1038            (Size::S32, Location::SIMD(src), Location::GPR(dst)) => {
1039                dynasm!(self ; fmv.x.s X(dst), F(src));
1040            }
1041            (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
1042                dynasm!(self ; fmv.d F(dst), F(src));
1043            }
1044            (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
1045                dynasm!(self ; fmv.s F(dst), F(src));
1046            }
1047            _ => codegen_error!("singlepass can't emit MOV {:?} {:?} {:?}", sz, src, dst),
1048        }
1049
1050        Ok(())
1051    }
1052
1053    fn emit_ret(&mut self) -> Result<(), CompileError> {
1054        dynasm!(self ; ret);
1055        Ok(())
1056    }
1057
1058    fn emit_mov_imm(&mut self, dst: Location, val: i64) -> Result<(), CompileError> {
1059        // The number of used bits by the number including the sign bit.
1060        // i64::MIN.abs() will overflow, thus handle it specially.
1061        let used_bits = if val == i64::MIN {
1062            i64::BITS
1063        } else {
1064            i64::BITS - val.abs().leading_zeros() + 1
1065        };
1066
1067        match dst {
1068            Location::GPR(dst) => match used_bits {
1069                0..=12 => dynasm!(self ; li.12 X(dst), val as _),
1070                13..=32 => dynasm!(self ; li.32 X(dst), val as _),
1071                33..=43 => dynasm!(self ; li.43 X(dst), val),
1072                44..=54 => dynasm!(self ; li.54 X(dst), val),
1073                55..=64 => dynasm!(self ; li X(dst), val),
1074                _ => unreachable!(),
1075            },
1076            _ => codegen_error!("singlepass can't emit MOVIMM {:?}", dst),
1077        }
1078        Ok(())
1079    }
1080
1081    fn emit_fneg(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1082        match (sz, src, dst) {
1083            (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
1084                dynasm!(self ; fneg.d F(dst), F(src));
1085            }
1086            (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
1087                dynasm!(self ; fneg.s F(dst), F(src));
1088            }
1089            _ => codegen_error!("singlepass can't emit FNEG {:?} {:?} {:?}", sz, src, dst),
1090        }
1091
1092        Ok(())
1093    }
1094
1095    fn emit_fmin(
1096        &mut self,
1097        sz: Size,
1098        src1: Location,
1099        src2: Location,
1100        dst: Location,
1101    ) -> Result<(), CompileError> {
1102        match (sz, src1, src2, dst) {
1103            (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
1104                dynasm!(self ; fmin.s F(dst), F(src1), F(src2));
1105            }
1106            (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
1107                dynasm!(self ; fmin.d F(dst), F(src1), F(src2));
1108            }
1109            _ => codegen_error!(
1110                "singlepass can't emit FMIN {:?} {:?} {:?} {:?}",
1111                sz,
1112                src1,
1113                src2,
1114                dst
1115            ),
1116        }
1117        Ok(())
1118    }
1119
1120    fn emit_fmax(
1121        &mut self,
1122        sz: Size,
1123        src1: Location,
1124        src2: Location,
1125        dst: Location,
1126    ) -> Result<(), CompileError> {
1127        match (sz, src1, src2, dst) {
1128            (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
1129                dynasm!(self ; fmax.s F(dst), F(src1), F(src2));
1130            }
1131            (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
1132                dynasm!(self ; fmax.d F(dst), F(src1), F(src2));
1133            }
1134            _ => codegen_error!(
1135                "singlepass can't emit FMAX {:?} {:?} {:?} {:?}",
1136                sz,
1137                src1,
1138                src2,
1139                dst
1140            ),
1141        }
1142        Ok(())
1143    }
1144
1145    fn emit_fdiv(
1146        &mut self,
1147        sz: Size,
1148        src1: Location,
1149        src2: Location,
1150        dst: Location,
1151    ) -> Result<(), CompileError> {
1152        match (sz, src1, src2, dst) {
1153            (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
1154                dynasm!(self ; fdiv.s F(dst), F(src1), F(src2));
1155            }
1156            (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
1157                dynasm!(self ; fdiv.d F(dst), F(src1), F(src2));
1158            }
1159            _ => codegen_error!(
1160                "singlepass can't emit FDIV {:?} {:?} {:?} {:?}",
1161                sz,
1162                src1,
1163                src2,
1164                dst
1165            ),
1166        }
1167        Ok(())
1168    }
1169
1170    fn emit_fsqrt(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1171        match (sz, src, dst) {
1172            (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
1173                dynasm!(self ; fsqrt.s F(dst), F(src));
1174            }
1175            (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
1176                dynasm!(self ; fsqrt.d F(dst), F(src));
1177            }
1178            _ => codegen_error!("singlepass can't emit FSQRT {:?} {:?} {:?}", sz, src, dst),
1179        }
1180        Ok(())
1181    }
1182
1183    fn emit_fcvt(
1184        &mut self,
1185        signed: bool,
1186        sz_in: Size,
1187        src: Location,
1188        sz_out: Size,
1189        dst: Location,
1190    ) -> Result<(), CompileError> {
1191        match (signed, sz_in, src, sz_out, dst) {
1192            // floating-point -> floating-point types
1193            (_, Size::S32, Location::SIMD(src), Size::S64, Location::SIMD(dst)) => {
1194                dynasm!(self ; fcvt.d.s F(dst), F(src));
1195            }
1196            (_, Size::S64, Location::SIMD(src), Size::S32, Location::SIMD(dst)) => {
1197                dynasm!(self ; fcvt.s.d F(dst), F(src));
1198            }
1199            // floating-point -> int types
1200            (true, Size::S32, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
1201                dynasm!(self ; fcvt.w.s X(dst), F(src), rtz);
1202            }
1203            (true, Size::S64, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
1204                dynasm!(self ; fcvt.w.d X(dst), F(src), rtz);
1205            }
1206            (true, Size::S32, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
1207                dynasm!(self ; fcvt.l.s X(dst), F(src), rtz);
1208            }
1209            (true, Size::S64, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
1210                dynasm!(self ; fcvt.l.d X(dst), F(src), rtz);
1211            }
1212            (false, Size::S32, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
1213                dynasm!(self ; fcvt.wu.s X(dst), F(src), rtz);
1214            }
1215            (false, Size::S64, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
1216                dynasm!(self ; fcvt.wu.d X(dst), F(src), rtz);
1217            }
1218            (false, Size::S32, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
1219                dynasm!(self ; fcvt.lu.s X(dst), F(src), rtz);
1220            }
1221            (false, Size::S64, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
1222                dynasm!(self ; fcvt.lu.d X(dst), F(src), rtz);
1223            }
1224            // int -> floating-point types
1225            (true, Size::S32, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
1226                dynasm!(self ; fcvt.s.w F(dst), X(src));
1227            }
1228            (true, Size::S64, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
1229                dynasm!(self ; fcvt.s.l F(dst), X(src));
1230            }
1231            (true, Size::S32, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
1232                dynasm!(self ; fcvt.d.w F(dst), X(src));
1233            }
1234            (true, Size::S64, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
1235                dynasm!(self ; fcvt.d.l F(dst), X(src));
1236            }
1237
1238            (false, Size::S32, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
1239                dynasm!(self ; fcvt.s.wu F(dst), X(src));
1240            }
1241            (false, Size::S64, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
1242                dynasm!(self ; fcvt.s.lu F(dst), X(src));
1243            }
1244            (false, Size::S32, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
1245                dynasm!(self ; fcvt.d.wu F(dst), X(src));
1246            }
1247            (false, Size::S64, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
1248                dynasm!(self ; fcvt.d.lu F(dst), X(src));
1249            }
1250            _ => codegen_error!(
1251                "singlepass can't emit FCVT {:?} {:?} {:?} {:?}",
1252                sz_in,
1253                src,
1254                sz_out,
1255                dst
1256            ),
1257        }
1258        Ok(())
1259    }
1260
1261    fn emit_fcvt_with_rounding(
1262        &mut self,
1263        rounding: RoundingMode,
1264        size: Size,
1265        src: Location,
1266        dst: Location,
1267        tmp: GPR,
1268    ) -> Result<(), CompileError> {
1269        let (Location::SIMD(src), Location::SIMD(dst)) = (src, dst) else {
1270            codegen_error!(
1271                "singlepass can't emit FCVT with rounding for non-register operands: {:?} {:?} {:?} {:?}",
1272                src,
1273                dst,
1274                size,
1275                rounding
1276            )
1277        };
1278
1279        match (size, rounding) {
1280            (Size::S32, RoundingMode::Rne) => {
1281                dynasm!(self
1282                    ; fcvt.w.s X(tmp), F(src), rne
1283                    ; fcvt.s.w F(dst), X(tmp), rne
1284                    ; fsgnj.s F(dst), F(dst), F(src));
1285            }
1286            (Size::S32, RoundingMode::Rtz) => {
1287                dynasm!(self
1288                    ; fcvt.w.s X(tmp), F(src), rtz
1289                    ; fcvt.s.w F(dst), X(tmp), rtz
1290                    ; fsgnj.s F(dst), F(dst), F(src));
1291            }
1292            (Size::S32, RoundingMode::Rdn) => {
1293                dynasm!(self
1294                    ; fcvt.w.s X(tmp), F(src), rdn
1295                    ; fcvt.s.w F(dst), X(tmp), rdn
1296                    ; fsgnj.s F(dst), F(dst), F(src));
1297            }
1298            (Size::S32, RoundingMode::Rup) => {
1299                dynasm!(self
1300                    ; fcvt.w.s X(tmp), F(src), rup
1301                    ; fcvt.s.w F(dst), X(tmp), rup
1302                    ; fsgnj.s F(dst), F(dst), F(src));
1303            }
1304
1305            (Size::S64, RoundingMode::Rne) => {
1306                dynasm!(self
1307                    ; fcvt.l.d X(tmp), F(src), rne
1308                    ; fcvt.d.l F(dst), X(tmp), rne
1309                    ; fsgnj.d F(dst), F(dst), F(src));
1310            }
1311            (Size::S64, RoundingMode::Rtz) => {
1312                dynasm!(self
1313                    ; fcvt.l.d X(tmp), F(src), rtz
1314                    ; fcvt.d.l F(dst), X(tmp), rtz
1315                    ; fsgnj.d F(dst), F(dst), F(src));
1316            }
1317            (Size::S64, RoundingMode::Rdn) => {
1318                dynasm!(self
1319                    ; fcvt.l.d X(tmp), F(src), rdn
1320                    ; fcvt.d.l F(dst), X(tmp), rdn
1321                    ; fsgnj.d F(dst), F(dst), F(src));
1322            }
1323            (Size::S64, RoundingMode::Rup) => {
1324                dynasm!(self
1325                    ; fcvt.l.d X(tmp), F(src), rup
1326                    ; fcvt.d.l F(dst), X(tmp), rup
1327                    ; fsgnj.d F(dst), F(dst), F(src));
1328            }
1329            _ => codegen_error!(
1330                "singlepass can't emit FCVT with rounding {:?} {:?}",
1331                size,
1332                rounding
1333            ),
1334        }
1335
1336        Ok(())
1337    }
1338
1339    // Swap `source` register with FSCR.
1340    fn emit_swap_fscr(&mut self, reg: GPR) -> Result<(), CompileError> {
1341        dynasm!(self ; fscsr X(reg), X(reg));
1342        Ok(())
1343    }
1344
1345    fn emit_cmp(
1346        &mut self,
1347        c: Condition,
1348        loc_a: Location,
1349        loc_b: Location,
1350        ret: Location,
1351    ) -> Result<(), CompileError> {
1352        let (Location::GPR(loc_a), Location::GPR(loc_b), Location::GPR(ret)) = (loc_a, loc_b, ret)
1353        else {
1354            codegen_error!(
1355                "singlepass can't emit CMP {:?} {:?} {:?}",
1356                loc_a,
1357                loc_b,
1358                ret
1359            );
1360        };
1361
1362        match c {
1363            // signed comparison operations
1364            Condition::Lt => {
1365                dynasm!(self; slt X(ret), X(loc_a), X(loc_b));
1366            }
1367            Condition::Le => {
1368                dynasm!(self
1369                    ; slt X(ret), X(loc_b), X(loc_a)
1370                    ; xori X(ret), X(ret), 1);
1371            }
1372            Condition::Gt => {
1373                dynasm!(self; slt X(ret), X(loc_b), X(loc_a));
1374            }
1375            Condition::Ge => {
1376                dynasm!(self
1377                    ; slt X(ret), X(loc_a), X(loc_b)
1378                    ; xori X(ret), X(ret), 1);
1379            }
1380            Condition::Eq => {
1381                dynasm!(self
1382                    ; xor X(ret), X(loc_a), X(loc_b)
1383                    ; seqz X(ret), X(ret));
1384            }
1385            Condition::Ne => {
1386                dynasm!(self
1387                    ; xor X(ret), X(loc_a), X(loc_b)
1388                    ; snez X(ret), X(ret));
1389            }
1390            // unsigned comparison operations
1391            Condition::Ltu => {
1392                dynasm!(self; sltu X(ret), X(loc_a), X(loc_b));
1393            }
1394            Condition::Leu => {
1395                dynasm!(self
1396                    ; sltu X(ret), X(loc_b), X(loc_a)
1397                    ; xori X(ret), X(ret), 1);
1398            }
1399            Condition::Gtu => {
1400                dynasm!(self; sltu X(ret), X(loc_b), X(loc_a));
1401            }
1402            Condition::Geu => {
1403                dynasm!(self
1404                    ; sltu X(ret), X(loc_a), X(loc_b)
1405                    ; xori X(ret), X(ret), 1);
1406            }
1407        }
1408
1409        Ok(())
1410    }
1411
1412    fn emit_fcmp(
1413        &mut self,
1414        c: Condition,
1415        size: Size,
1416        loc_a: Location,
1417        loc_b: Location,
1418        ret: Location,
1419    ) -> Result<(), CompileError> {
1420        let (Location::SIMD(loc_a), Location::SIMD(loc_b), Location::GPR(ret)) =
1421            (loc_a, loc_b, ret)
1422        else {
1423            codegen_error!(
1424                "singlepass can't emit FCMP {:?} {:?} {:?}",
1425                loc_a,
1426                loc_b,
1427                ret
1428            );
1429        };
1430
1431        match (size, c) {
1432            (Size::S32, Condition::Lt) => {
1433                dynasm!(self ; flt.s X(ret), F(loc_a), F(loc_b));
1434            }
1435            (Size::S32, Condition::Le) => {
1436                dynasm!(self ; fle.s X(ret), F(loc_a), F(loc_b));
1437            }
1438            (Size::S32, Condition::Gt) => {
1439                dynasm!(self ; flt.s X(ret), F(loc_b), F(loc_a));
1440            }
1441            (Size::S32, Condition::Ge) => {
1442                dynasm!(self ; fle.s X(ret), F(loc_b), F(loc_a));
1443            }
1444            (Size::S32, Condition::Eq) => {
1445                dynasm!(self ; feq.s X(ret), F(loc_a), F(loc_b));
1446            }
1447            (Size::S32, Condition::Ne) => {
1448                dynasm!(self
1449                    ; feq.s X(ret), F(loc_a), F(loc_b)
1450                    ; xori X(ret), X(ret), 1);
1451            }
1452
1453            (Size::S64, Condition::Lt) => {
1454                dynasm!(self ; flt.d X(ret), F(loc_a), F(loc_b));
1455            }
1456            (Size::S64, Condition::Le) => {
1457                dynasm!(self ; fle.d X(ret), F(loc_a), F(loc_b));
1458            }
1459            (Size::S64, Condition::Gt) => {
1460                dynasm!(self ; flt.d X(ret), F(loc_b), F(loc_a));
1461            }
1462            (Size::S64, Condition::Ge) => {
1463                dynasm!(self; fle.d X(ret), F(loc_b), F(loc_a));
1464            }
1465            (Size::S64, Condition::Eq) => {
1466                dynasm!(self ; feq.d X(ret), F(loc_a), F(loc_b));
1467            }
1468            (Size::S64, Condition::Ne) => {
1469                dynasm!(self
1470                    ; feq.d X(ret), F(loc_a), F(loc_b)
1471                    ; xori X(ret), X(ret), 1);
1472            }
1473            _ => codegen_error!(
1474                "singlepass can't emit FCMP {:?} {:?} {:?}",
1475                loc_a,
1476                loc_b,
1477                ret
1478            ),
1479        }
1480
1481        Ok(())
1482    }
1483
1484    fn emit_jmp_on_condition(
1485        &mut self,
1486        cond: Condition,
1487        loc_a: Location,
1488        loc_b: Location,
1489        label: Label,
1490        tmp: GPR,
1491    ) -> Result<(), CompileError> {
1492        let (Location::GPR(loc_a), Location::GPR(loc_b)) = (loc_a, loc_b) else {
1493            codegen_error!("singlepass can't emit JMP_ON_COND {:?} {:?} ", loc_a, loc_b,);
1494        };
1495
1496        let jump: Label = self.get_label();
1497        let cont: Label = self.get_label();
1498
1499        match cond {
1500            Condition::Eq => {
1501                dynasm!(self; beq X(loc_a), X(loc_b), => jump);
1502            }
1503            Condition::Ne => {
1504                dynasm!(self; bne X(loc_a), X(loc_b), => jump);
1505            }
1506            Condition::Ltu => {
1507                dynasm!(self; bltu X(loc_a), X(loc_b), => jump);
1508            }
1509            Condition::Leu => {
1510                dynasm!(self; bleu X(loc_a), X(loc_b), => jump);
1511            }
1512            Condition::Gtu => {
1513                dynasm!(self; bgtu X(loc_a), X(loc_b), => jump);
1514            }
1515            Condition::Geu => {
1516                dynasm!(self; bgeu X(loc_a), X(loc_b), => jump);
1517            }
1518            _ => codegen_error!(
1519                "singlepass can't emit jump on conditional branch {:?}",
1520                cond
1521            ),
1522        }
1523
1524        self.emit_j_label(cont, Some(tmp))?;
1525        self.emit_label(jump)?;
1526        self.emit_j_label(label, Some(tmp))?;
1527
1528        self.emit_label(cont)?;
1529
1530        Ok(())
1531    }
1532
1533    fn emit_on_false_label(
1534        &mut self,
1535        cond: Location,
1536        label: Label,
1537        tmp: GPR,
1538    ) -> Result<(), CompileError> {
1539        match cond {
1540            Location::GPR(cond) => {
1541                dynasm!(self; beqz X(cond), => label);
1542            }
1543            _ if cond.is_imm() => {
1544                let imm = cond.imm_value_scalar().unwrap();
1545                if imm == 0 {
1546                    return self.emit_j_label(label, Some(tmp));
1547                }
1548            }
1549            _ => codegen_error!("singlepass can't emit jump to false branch {:?}", cond),
1550        }
1551        Ok(())
1552    }
1553    fn emit_on_false_label_far(
1554        &mut self,
1555        cond: Location,
1556        label: Label,
1557        tmp: GPR,
1558    ) -> Result<(), CompileError> {
1559        let cont: Label = self.get_label();
1560        match cond {
1561            Location::GPR(cond) => {
1562                // Use the negative condition to jump after the "j" instruction that will
1563                // go to the requsted `label`.
1564                dynasm!(self; bnez X(cond), => cont);
1565            }
1566            _ if cond.is_imm() => {
1567                let imm = cond.imm_value_scalar().unwrap();
1568                if imm == 0 {
1569                    return self.emit_j_label(label, Some(tmp));
1570                } else {
1571                    self.emit_j_label(cont, Some(tmp))?;
1572                }
1573            }
1574            _ => codegen_error!("singlepass can't emit jump to false branch {:?}", cond),
1575        }
1576
1577        self.emit_j_label(label, Some(tmp))?;
1578        self.emit_label(cont)?;
1579        Ok(())
1580    }
1581    fn emit_on_true_label(
1582        &mut self,
1583        cond: Location,
1584        label: Label,
1585        tmp: GPR,
1586    ) -> Result<(), CompileError> {
1587        match cond {
1588            Location::GPR(cond) => {
1589                dynasm!(self; bnez X(cond), => label);
1590            }
1591            _ if cond.is_imm() => {
1592                let imm = cond.imm_value_scalar().unwrap();
1593                if imm != 0 {
1594                    return self.emit_j_label(label, Some(tmp));
1595                }
1596            }
1597            _ => codegen_error!("singlepass can't emit jump to true branch {:?}", cond),
1598        }
1599        Ok(())
1600    }
1601    fn emit_on_true_label_far(
1602        &mut self,
1603        cond: Location,
1604        label: Label,
1605        tmp: GPR,
1606    ) -> Result<(), CompileError> {
1607        let cont: Label = self.get_label();
1608        let jump_label: Label = self.get_label();
1609        match cond {
1610            Location::GPR(cond) => {
1611                dynasm!(self; bnez X(cond), => jump_label);
1612            }
1613            _ if cond.is_imm() => {
1614                let imm = cond.imm_value_scalar().unwrap();
1615                if imm == 1 {
1616                    return self.emit_j_label(jump_label, Some(tmp));
1617                }
1618            }
1619            _ => codegen_error!("singlepass can't emit jump to false branch {:?}", cond),
1620        }
1621
1622        // skip over the jump to target
1623        self.emit_j_label(cont, Some(tmp))?;
1624
1625        self.emit_label(jump_label)?;
1626        self.emit_j_label(label, Some(tmp))?;
1627
1628        self.emit_label(cont)?;
1629
1630        Ok(())
1631    }
1632
1633    fn emit_j_label(&mut self, label: Label, reg: Option<GPR>) -> Result<(), CompileError> {
1634        if let Some(reg) = reg {
1635            dynasm!(self ; jump => label, X(reg));
1636        } else {
1637            dynasm!(self ; j => label);
1638        }
1639        Ok(())
1640    }
1641
1642    fn emit_j_register(&mut self, reg: GPR) -> Result<(), CompileError> {
1643        dynasm!(self ; jalr zero, X(reg), 0);
1644        Ok(())
1645    }
1646
1647    fn emit_call_label(&mut self, label: Label) -> Result<(), CompileError> {
1648        dynasm!(self ; call =>label);
1649        Ok(())
1650    }
1651
1652    fn emit_call_register(&mut self, reg: GPR) -> Result<(), CompileError> {
1653        dynasm!(self ; jalr ra, X(reg), 0);
1654        Ok(())
1655    }
1656
1657    fn emit_load_label(&mut self, reg: GPR, label: Label) -> Result<(), CompileError> {
1658        dynasm!(self ; la X(reg), => label);
1659        Ok(())
1660    }
1661
1662    fn emit_rwfence(&mut self) -> Result<(), CompileError> {
1663        dynasm!(self ; fence rw, rw);
1664        Ok(())
1665    }
1666
1667    fn emit_atomic_binop(
1668        &mut self,
1669        op: AtomicBinaryOp,
1670        size: Size,
1671        dest: GPR,
1672        addr: GPR,
1673        source: GPR,
1674    ) -> Result<(), CompileError> {
1675        match (size, op) {
1676            (Size::S64, AtomicBinaryOp::Add) => {
1677                dynasm!(self ; amoadd.d.aqrl X(dest), X(source), [X(addr)])
1678            }
1679            (Size::S64, AtomicBinaryOp::Or) => {
1680                dynasm!(self ; amoor.d.aqrl X(dest), X(source), [X(addr)])
1681            }
1682            (Size::S64, AtomicBinaryOp::Xor) => {
1683                dynasm!(self ; amoxor.d.aqrl X(dest), X(source), [X(addr)])
1684            }
1685            (Size::S64, AtomicBinaryOp::And) => {
1686                dynasm!(self ; amoand.d.aqrl X(dest), X(source), [X(addr)])
1687            }
1688            (Size::S64, AtomicBinaryOp::Exchange) => {
1689                dynasm!(self ; amoswap.d.aqrl X(dest), X(source), [X(addr)])
1690            }
1691
1692            (Size::S32, AtomicBinaryOp::Add) => {
1693                dynasm!(self ; amoadd.w.aqrl X(dest), X(source), [X(addr)])
1694            }
1695            (Size::S32, AtomicBinaryOp::Or) => {
1696                dynasm!(self ; amoor.w.aqrl X(dest), X(source), [X(addr)])
1697            }
1698            (Size::S32, AtomicBinaryOp::Xor) => {
1699                dynasm!(self ; amoxor.w.aqrl X(dest), X(source), [X(addr)])
1700            }
1701            (Size::S32, AtomicBinaryOp::And) => {
1702                dynasm!(self ; amoand.w.aqrl X(dest), X(source), [X(addr)])
1703            }
1704            (Size::S32, AtomicBinaryOp::Exchange) => {
1705                dynasm!(self ; amoswap.w.aqrl X(dest), X(source), [X(addr)])
1706            }
1707            _ => codegen_error!("singlepass can't emit atomic binop {:?} {:?}", size, op),
1708        }
1709
1710        Ok(())
1711    }
1712
1713    fn emit_reserved_ld(&mut self, size: Size, reg: GPR, addr: GPR) -> Result<(), CompileError> {
1714        match size {
1715            Size::S32 => {
1716                dynasm!(self ; lr.w.aqrl X(reg), [X(addr)])
1717            }
1718            Size::S64 => {
1719                dynasm!(self ; lr.d.aqrl X(reg), [X(addr)])
1720            }
1721            _ => codegen_error!("singlepass can't emit atomic reserved ld {:?}", size),
1722        }
1723
1724        Ok(())
1725    }
1726
1727    fn emit_reserved_sd(
1728        &mut self,
1729        size: Size,
1730        dest: GPR,
1731        addr: GPR,
1732        source: GPR,
1733    ) -> Result<(), CompileError> {
1734        match size {
1735            Size::S32 => {
1736                dynasm!(self ; sc.w.rl X(dest), X(source), [X(addr)])
1737            }
1738            Size::S64 => {
1739                dynasm!(self ; sc.d.rl X(dest), X(source), [X(addr)])
1740            }
1741            _ => codegen_error!("singlepass can't emit atomic reserved sd {:?}", size),
1742        }
1743
1744        Ok(())
1745    }
1746}
1747
1748pub fn gen_std_trampoline_riscv(
1749    sig: &FunctionType,
1750    _calling_convention: CallingConvention,
1751) -> Result<FunctionBody, CompileError> {
1752    let mut a = Assembler::new(0);
1753
1754    // Callee-save registers must be used.
1755    let fptr = GPR::X26;
1756    let args = GPR::X27;
1757
1758    dynasm!(a
1759        ; addi sp, sp, -32
1760        ; sd s0, [sp,24]
1761        ; sd ra, [sp,16]
1762        ; sd X(fptr), [sp, 8]
1763        ; sd X(args), [sp, 0]
1764        ; mv s0, sp // use frame-pointer register for later restore
1765        ; mv X(fptr), a1
1766        ; mv X(args), a2
1767    );
1768
1769    let stack_args = sig.params().len().saturating_sub(7); //1st arg is ctx, not an actual arg
1770    let stack_return_slots = sig
1771        .results()
1772        .len()
1773        .saturating_sub(RISCV_RETURN_VALUE_REGISTERS.len());
1774    let mut stack_offset = (stack_args + stack_return_slots) as u32 * 8;
1775    if stack_offset > 0 {
1776        stack_offset = stack_offset.next_multiple_of(16);
1777        if ImmType::Bits12Subtraction.compatible_imm(stack_offset as _) {
1778            dynasm!(a ; addi sp, sp, -(stack_offset as i32));
1779        } else {
1780            a.emit_mov_imm(Location::GPR(SCRATCH_REG), stack_offset as _)?;
1781            dynasm!(a ; sub sp, sp, X(SCRATCH_REG));
1782        }
1783    }
1784
1785    // Move arguments to their locations.
1786    // `callee_vmctx` is already in the first argument register, so no need to move.
1787    let mut caller_stack_offset: i32 = stack_return_slots as i32 * 8;
1788    for (i, param) in sig.params().iter().enumerate() {
1789        let sz = match *param {
1790            Type::I32 | Type::F32 => Size::S32,
1791            Type::I64 | Type::F64 => Size::S64,
1792            Type::ExternRef => Size::S64,
1793            Type::FuncRef => Size::S64,
1794            _ => codegen_error!(
1795                "singlepass unsupported param type for trampoline {:?}",
1796                *param
1797            ),
1798        };
1799        match i {
1800            0..=6 => {
1801                a.emit_ld(
1802                    sz,
1803                    false,
1804                    Location::GPR(GPR::from_index(i + 10 + 1).unwrap()),
1805                    Location::Memory(args, (i * 16) as i32),
1806                )?;
1807            }
1808            _ => {
1809                let args_offset = (i * 16) as i32;
1810                let arg_location = if ImmType::Bits12.compatible_imm(args_offset as i64) {
1811                    Location::Memory(args, args_offset)
1812                } else {
1813                    a.emit_mov_imm(Location::GPR(SCRATCH_REG), args_offset as i64)?;
1814                    a.emit_add(
1815                        Size::S64,
1816                        Location::GPR(args),
1817                        Location::GPR(SCRATCH_REG),
1818                        Location::GPR(SCRATCH_REG),
1819                    )?;
1820                    Location::Memory(SCRATCH_REG, 0)
1821                };
1822                a.emit_ld(sz, false, Location::GPR(SCRATCH_REG), arg_location)?;
1823
1824                let arg_dest_location =
1825                    if ImmType::Bits12.compatible_imm(caller_stack_offset as i64) {
1826                        Location::Memory(GPR::Sp, caller_stack_offset)
1827                    } else {
1828                        a.emit_mov_imm(Location::GPR(GPR::X29), caller_stack_offset as i64)?;
1829                        a.emit_add(
1830                            Size::S64,
1831                            Location::GPR(GPR::X29),
1832                            Location::GPR(GPR::Sp),
1833                            Location::GPR(GPR::X29),
1834                        )?;
1835                        Location::Memory(GPR::X29, 0)
1836                    };
1837                a.emit_sd(sz, Location::GPR(SCRATCH_REG), arg_dest_location)?;
1838                caller_stack_offset += 8;
1839            }
1840        }
1841    }
1842
1843    dynasm!(a
1844        ; jalr ra, X(fptr), 0);
1845
1846    // Write return values.
1847    let mut n_stack_return_slots: usize = 0;
1848    for i in 0..sig.results().len() {
1849        let src = if let Some(&reg) = RISCV_RETURN_VALUE_REGISTERS.get(i) {
1850            reg
1851        } else {
1852            a.emit_ld(
1853                Size::S64,
1854                false,
1855                Location::GPR(SCRATCH_REG),
1856                Location::Memory(GPR::Sp, (n_stack_return_slots as u32 * 8) as _),
1857            )?;
1858            n_stack_return_slots += 1;
1859            SCRATCH_REG
1860        };
1861        a.emit_sd(
1862            Size::S64,
1863            Location::GPR(src),
1864            Location::Memory(args, (i * 16) as _),
1865        )?;
1866    }
1867
1868    // Restore stack.
1869    dynasm!(a
1870        ; ld X(args), [s0,0]
1871        ; ld X(fptr), [s0,8]
1872        ; ld ra, [s0,16]
1873        ; ld s0, [s0,24]
1874    );
1875    let restored_stack_offset = 32 + stack_offset;
1876    if ImmType::Bits12.compatible_imm(restored_stack_offset as _) {
1877        dynasm!(a; addi sp, sp, restored_stack_offset as _);
1878    } else {
1879        a.emit_mov_imm(Location::GPR(SCRATCH_REG), restored_stack_offset as _)?;
1880        dynasm!(a; add sp, sp, X(SCRATCH_REG));
1881    }
1882    dynasm!(a; ret);
1883
1884    let mut body = a.finalize().unwrap();
1885
1886    body.shrink_to_fit();
1887    Ok(FunctionBody {
1888        body,
1889        unwind_info: None,
1890    })
1891}
1892
1893/// Generates dynamic import function call trampoline for a function type.
1894pub fn gen_std_dynamic_import_trampoline_riscv(
1895    vmoffsets: &VMOffsets,
1896    sig: &FunctionType,
1897) -> Result<FunctionBody, CompileError> {
1898    let mut a = Assembler::new(0);
1899    // Allocate argument array.
1900    let stack_offset: usize = 16 * std::cmp::max(sig.params().len(), sig.results().len());
1901
1902    // Save RA and the scratch register
1903    a.emit_push(Size::S64, Location::GPR(GPR::X1))?;
1904    a.emit_push(Size::S64, Location::GPR(SCRATCH_REG))?;
1905
1906    if stack_offset != 0 {
1907        if ImmType::Bits12Subtraction.compatible_imm(stack_offset as _) {
1908            a.emit_sub(
1909                Size::S64,
1910                Location::GPR(GPR::Sp),
1911                Location::Imm64(stack_offset as _),
1912                Location::GPR(GPR::Sp),
1913            )?;
1914        } else {
1915            a.emit_mov_imm(Location::GPR(SCRATCH_REG), stack_offset as _)?;
1916            a.emit_sub(
1917                Size::S64,
1918                Location::GPR(GPR::Sp),
1919                Location::GPR(SCRATCH_REG),
1920                Location::GPR(GPR::Sp),
1921            )?;
1922        }
1923    }
1924
1925    // Copy arguments.
1926    if !sig.params().is_empty() {
1927        let mut argalloc = ArgumentRegisterAllocator::default();
1928        argalloc.next(Type::I64).unwrap(); // skip VMContext
1929
1930        let mut stack_param_count: usize = 0;
1931
1932        for (i, ty) in sig.params().iter().enumerate() {
1933            let source_loc = match argalloc.next(*ty)? {
1934                Some(RiscvRegister::GPR(gpr)) => Location::GPR(gpr),
1935                Some(RiscvRegister::FPR(fpr)) => Location::SIMD(fpr),
1936                None => {
1937                    a.emit_ld(
1938                        Size::S64,
1939                        false,
1940                        Location::GPR(SCRATCH_REG),
1941                        Location::Memory(GPR::Sp, (stack_offset + 16 + stack_param_count) as _),
1942                    )?;
1943                    stack_param_count += 8;
1944                    Location::GPR(SCRATCH_REG)
1945                }
1946            };
1947            a.emit_sd(
1948                Size::S64,
1949                source_loc,
1950                Location::Memory(GPR::Sp, (i * 16) as _),
1951            )?;
1952            // Zero upper 64 bits.
1953            a.emit_sd(
1954                Size::S64,
1955                Location::GPR(GPR::XZero),
1956                Location::Memory(GPR::Sp, (i * 16 + 8) as _),
1957            )?;
1958        }
1959    }
1960
1961    // Load target address.
1962    let offset = vmoffsets.vmdynamicfunction_import_context_address();
1963    a.emit_ld(
1964        Size::S64,
1965        false,
1966        Location::GPR(SCRATCH_REG),
1967        Location::Memory(GPR::X10, offset as i32),
1968    )?;
1969    // Load values array.
1970    a.emit_add(
1971        Size::S64,
1972        Location::GPR(GPR::Sp),
1973        Location::Imm64(0),
1974        Location::GPR(GPR::X11),
1975    )?;
1976
1977    // Call target.
1978    a.emit_call_register(SCRATCH_REG)?;
1979
1980    // Fetch return value.
1981    if !sig.results().is_empty() {
1982        assert_eq!(sig.results().len(), 1);
1983        a.emit_ld(
1984            Size::S64,
1985            false,
1986            Location::GPR(GPR::X10),
1987            Location::Memory(GPR::Sp, 0),
1988        )?;
1989    }
1990
1991    // Release values array.
1992    if stack_offset != 0 {
1993        if ImmType::Bits12.compatible_imm(stack_offset as _) {
1994            a.emit_add(
1995                Size::S64,
1996                Location::GPR(GPR::Sp),
1997                Location::Imm64(stack_offset as _),
1998                Location::GPR(GPR::Sp),
1999            )?;
2000        } else {
2001            a.emit_mov_imm(Location::GPR(SCRATCH_REG), stack_offset as _)?;
2002            a.emit_add(
2003                Size::S64,
2004                Location::GPR(GPR::Sp),
2005                Location::GPR(SCRATCH_REG),
2006                Location::GPR(GPR::Sp),
2007            )?;
2008        }
2009    }
2010    a.emit_pop(Size::S64, Location::GPR(SCRATCH_REG))?;
2011    a.emit_pop(Size::S64, Location::GPR(GPR::X1))?;
2012
2013    // Return.
2014    a.emit_ret()?;
2015
2016    let mut body = a.finalize().unwrap();
2017    body.shrink_to_fit();
2018    Ok(FunctionBody {
2019        body,
2020        unwind_info: None,
2021    })
2022}
2023
2024// Singlepass calls import functions through a trampoline.
2025pub fn gen_import_call_trampoline_riscv(
2026    vmoffsets: &VMOffsets,
2027    index: FunctionIndex,
2028    sig: &FunctionType,
2029    _calling_convention: CallingConvention,
2030) -> Result<CustomSection, CompileError> {
2031    let mut a = Assembler::new(0);
2032
2033    // Singlepass internally treats all arguments as integers
2034    // For the standard System V calling convention requires
2035    //  floating point arguments to be passed in FPR registers.
2036    //  Translation is expensive, so only do it if needed.
2037    if sig
2038        .params()
2039        .iter()
2040        .any(|&x| x == Type::F32 || x == Type::F64)
2041    {
2042        const PARAM_REGS: &[GPR] = &[
2043            // X10 is skipped intentionally
2044            GPR::X11,
2045            GPR::X12,
2046            GPR::X13,
2047            GPR::X14,
2048            GPR::X15,
2049            GPR::X16,
2050            GPR::X17,
2051        ];
2052        const PARAM_REGS_COUNT: usize = PARAM_REGS.len();
2053
2054        // Allocate stack space for arguments.
2055        let stack_offset: i32 = if sig.params().len() > PARAM_REGS_COUNT {
2056            (PARAM_REGS_COUNT * 8) as i32
2057        } else {
2058            (sig.params().len() as i32) * 8
2059        };
2060        if stack_offset > 0 {
2061            if ImmType::Bits12Subtraction.compatible_imm(stack_offset as _) {
2062                a.emit_sub(
2063                    Size::S64,
2064                    Location::GPR(GPR::Sp),
2065                    Location::Imm64(stack_offset as _),
2066                    Location::GPR(GPR::Sp),
2067                )?;
2068            } else {
2069                a.emit_mov_imm(Location::GPR(SCRATCH_REG), stack_offset as _)?;
2070                a.emit_sub(
2071                    Size::S64,
2072                    Location::GPR(GPR::Sp),
2073                    Location::GPR(SCRATCH_REG),
2074                    Location::GPR(GPR::Sp),
2075                )?;
2076            }
2077        }
2078
2079        // Store all arguments to the stack to prevent overwrite.
2080
2081        let mut param_locations = vec![];
2082        /* Clippy is wrong about using `i` to index `PARAM_REGS` here. */
2083        #[allow(clippy::needless_range_loop)]
2084        for i in 0..sig.params().len() {
2085            let loc = match i {
2086                0..PARAM_REGS_COUNT => {
2087                    let loc = Location::Memory(GPR::Sp, (i * 8) as i32);
2088                    a.emit_sd(Size::S64, Location::GPR(PARAM_REGS[i]), loc)?;
2089                    loc
2090                }
2091                _ => Location::Memory(GPR::Sp, stack_offset + ((i - PARAM_REGS_COUNT) * 8) as i32),
2092            };
2093            param_locations.push(loc);
2094        }
2095
2096        // Copy arguments.
2097        let mut caller_stack_offset: i32 = 0;
2098        let mut argalloc = ArgumentRegisterAllocator::default();
2099        argalloc.next(Type::I64).unwrap(); // skip VMContext
2100        for (i, ty) in sig.params().iter().enumerate() {
2101            let prev_loc = param_locations[i];
2102            let targ = match argalloc.next(*ty)? {
2103                Some(RiscvRegister::GPR(gpr)) => Location::GPR(gpr),
2104                Some(RiscvRegister::FPR(neon)) => Location::SIMD(neon),
2105                None => {
2106                    // No register can be allocated. Put this argument on the stack.
2107                    a.emit_ld(Size::S64, false, Location::GPR(SCRATCH_REG), prev_loc)?;
2108                    a.emit_sd(
2109                        Size::S64,
2110                        Location::GPR(SCRATCH_REG),
2111                        Location::Memory(GPR::Sp, stack_offset + caller_stack_offset),
2112                    )?;
2113                    caller_stack_offset += 8;
2114                    continue;
2115                }
2116            };
2117            a.emit_ld(Size::S64, false, targ, prev_loc)?;
2118        }
2119
2120        // Restore stack pointer.
2121        if stack_offset > 0 {
2122            if ImmType::Bits12.compatible_imm(stack_offset as _) {
2123                a.emit_add(
2124                    Size::S64,
2125                    Location::GPR(GPR::Sp),
2126                    Location::Imm64(stack_offset as _),
2127                    Location::GPR(GPR::Sp),
2128                )?;
2129            } else {
2130                a.emit_mov_imm(Location::GPR(SCRATCH_REG), stack_offset as _)?;
2131                a.emit_add(
2132                    Size::S64,
2133                    Location::GPR(GPR::Sp),
2134                    Location::GPR(SCRATCH_REG),
2135                    Location::GPR(GPR::Sp),
2136                )?;
2137            }
2138        }
2139    }
2140
2141    // Emits a tail call trampoline that loads the address of the target import function
2142    // from Ctx and jumps to it.
2143
2144    let offset = vmoffsets.vmctx_vmfunction_import(index);
2145
2146    a.emit_ld(
2147        Size::S64,
2148        false,
2149        Location::GPR(SCRATCH_REG),
2150        Location::Memory(GPR::X10, offset as i32), // function pointer
2151    )?;
2152    a.emit_ld(
2153        Size::S64,
2154        false,
2155        Location::GPR(GPR::X10),
2156        Location::Memory(GPR::X10, offset as i32 + 8), // target vmctx
2157    )?;
2158
2159    a.emit_j_register(SCRATCH_REG)?;
2160
2161    let mut contents = a.finalize().unwrap();
2162    contents.shrink_to_fit();
2163    let section_body = SectionBody::new_with_vec(contents);
2164
2165    Ok(CustomSection {
2166        protection: CustomSectionProtection::ReadExecute,
2167        alignment: None,
2168        bytes: section_body,
2169        relocations: vec![],
2170    })
2171}
wasmer_compiler_singlepass/emitter_riscv.rs

wasmer_compiler_singlepass/
emitter_riscv.rs
