wasmer_compiler_singlepass/
emitter_riscv.rs

1//! RISC-V emitter scaffolding.
2
3use crate::{
4    codegen_error,
5    common_decl::Size,
6    location::{Location as AbstractLocation, Reg},
7    machine::MaybeImmediate,
8    machine_riscv::{ImmType, RISCV_RETURN_VALUE_REGISTERS},
9    riscv_decl::{ArgumentRegisterAllocator, RiscvRegister},
10};
11pub use crate::{
12    machine::{Label, Offset},
13    riscv_decl::{FPR, GPR},
14};
15use dynasmrt::{DynasmApi, DynasmLabelApi, VecAssembler, riscv::RiscvRelocation};
16use wasmer_compiler::types::{
17    function::FunctionBody,
18    section::{CustomSection, CustomSectionProtection, SectionBody},
19};
20use wasmer_types::{
21    CompileError, FunctionIndex, FunctionType, Type, VMOffsets,
22    target::{CallingConvention, CpuFeature},
23};
24
25type Assembler = VecAssembler<RiscvRelocation>;
26
27/// Force `dynasm!` to use the correct arch (riscv64) when cross-compiling.
28macro_rules! dynasm {
29    ($a:expr ; $($tt:tt)*) => {
30        dynasm::dynasm!(
31            $a
32            ; .arch riscv64
33            ; .feature g
34            ; $($tt)*
35        )
36    };
37}
38
39/// Location abstraction specialized to RISC-V.
40pub type Location = AbstractLocation<GPR, FPR>;
41
42/// Branch conditions for RISC-V.
43#[derive(Copy, Clone, Debug, Eq, PartialEq)]
44pub enum Condition {
45    /// Signed less than
46    Lt,
47    /// Signed less than or equal
48    Le,
49    /// Signed greater than
50    Gt,
51    /// Signed greater than or equal
52    Ge,
53    /// Signed equal
54    Eq,
55    /// Signed not equal
56    Ne,
57
58    /// Unsigned less than
59    Ltu,
60    /// Unsigned less than or equal
61    Leu,
62    /// Unsigned greater than
63    Gtu,
64    /// Unsigned greater than or equal
65    Geu,
66}
67
68/// Floating-point number rounding mode
69#[derive(Copy, Clone, Debug, Eq, PartialEq)]
70pub enum RoundingMode {
71    /// Round to Nearest, ties to Even (default)
72    Rne,
73    /// Round towards Zero
74    Rtz,
75    /// Round Down (towards -∞)
76    Rdn,
77    /// Round Up (towards +∞)
78    Rup,
79}
80
81/// Atomic binary operation type
82#[derive(Copy, Clone, Debug, Eq, PartialEq)]
83pub enum AtomicBinaryOp {
84    Add,
85    Sub,
86    Or,
87    And,
88    Xor,
89    Exchange,
90}
91
92/// Scratch register used in function call trampolines.
93const SCRATCH_REG: GPR = GPR::X28;
94
95/// Emitter trait for RISC-V.
96#[allow(unused)]
97pub trait EmitterRiscv {
98    fn emit_jmp_on_condition(
99        &mut self,
100        cond: Condition,
101        loc_a: Location,
102        loc_b: Location,
103        label: Label,
104        tmp: GPR,
105    ) -> Result<(), CompileError>;
106    fn emit_reserved_sd(
107        &mut self,
108        size: Size,
109        dest: GPR,
110        addr: GPR,
111        source: GPR,
112    ) -> Result<(), CompileError>;
113    /// Returns the SIMD (FPU) feature if available.
114    fn get_simd_arch(&self) -> Option<&CpuFeature>;
115    /// Generates a new internal label.
116    fn get_label(&mut self) -> Label;
117    /// Gets the current code offset.
118    fn get_offset(&self) -> Offset;
119    /// Returns the size of a jump instruction in bytes.
120    fn get_jmp_instr_size(&self) -> u8;
121
122    /// Finalize the function, e.g., resolve labels.
123    fn finalize_function(&mut self) -> Result<(), CompileError>;
124
125    fn emit_label(&mut self, label: Label) -> Result<(), CompileError>;
126    fn emit_brk(&mut self) -> Result<(), CompileError>;
127    fn emit_ld(
128        &mut self,
129        sz: Size,
130        signed: bool,
131        reg: Location,
132        addr: Location,
133    ) -> Result<(), CompileError>;
134
135    fn emit_sd(&mut self, sz: Size, reg: Location, addr: Location) -> Result<(), CompileError>;
136    fn emit_push(&mut self, size: Size, src: Location) -> Result<(), CompileError>;
137    fn emit_pop(&mut self, size: Size, dst: Location) -> Result<(), CompileError>;
138
139    fn emit_add(
140        &mut self,
141        sz: Size,
142        src1: Location,
143        src2: Location,
144        dst: Location,
145    ) -> Result<(), CompileError>;
146    fn emit_sub(
147        &mut self,
148        sz: Size,
149        src1: Location,
150        src2: Location,
151        dst: Location,
152    ) -> Result<(), CompileError>;
153    fn emit_mul(
154        &mut self,
155        sz: Size,
156        src1: Location,
157        src2: Location,
158        dst: Location,
159    ) -> Result<(), CompileError>;
160    fn emit_sdiv(
161        &mut self,
162        sz: Size,
163        src1: Location,
164        src2: Location,
165        dst: Location,
166    ) -> Result<(), CompileError>;
167    fn emit_udiv(
168        &mut self,
169        sz: Size,
170        src1: Location,
171        src2: Location,
172        dst: Location,
173    ) -> Result<(), CompileError>;
174    fn emit_srem(
175        &mut self,
176        sz: Size,
177        src1: Location,
178        src2: Location,
179        dst: Location,
180    ) -> Result<(), CompileError>;
181    fn emit_urem(
182        &mut self,
183        sz: Size,
184        src1: Location,
185        src2: Location,
186        dst: Location,
187    ) -> Result<(), CompileError>;
188    fn emit_and(
189        &mut self,
190        sz: Size,
191        src1: Location,
192        src2: Location,
193        dst: Location,
194    ) -> Result<(), CompileError>;
195    fn emit_or(
196        &mut self,
197        sz: Size,
198        src1: Location,
199        src2: Location,
200        dst: Location,
201    ) -> Result<(), CompileError>;
202    fn emit_xor(
203        &mut self,
204        sz: Size,
205        src1: Location,
206        src2: Location,
207        dst: Location,
208    ) -> Result<(), CompileError>;
209    fn emit_sll(
210        &mut self,
211        sz: Size,
212        src1: Location,
213        src2: Location,
214        dst: Location,
215    ) -> Result<(), CompileError>;
216    fn emit_srl(
217        &mut self,
218        sz: Size,
219        src1: Location,
220        src2: Location,
221        dst: Location,
222    ) -> Result<(), CompileError>;
223    fn emit_sra(
224        &mut self,
225        sz: Size,
226        src1: Location,
227        src2: Location,
228        dst: Location,
229    ) -> Result<(), CompileError>;
230    fn emit_extend(
231        &mut self,
232        sz: Size,
233        signed: bool,
234        src: Location,
235        dst: Location,
236    ) -> Result<(), CompileError>;
237    fn emit_not(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
238    fn emit_neg(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
239
240    fn emit_mov(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
241
242    fn emit_ret(&mut self) -> Result<(), CompileError>;
243
244    fn emit_udf(&mut self, payload: u8) -> Result<(), CompileError>;
245
246    fn emit_mov_imm(&mut self, dst: Location, val: i64) -> Result<(), CompileError>;
247
248    // Floating-point type instructions
249    fn emit_fneg(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
250    fn emit_fmin(
251        &mut self,
252        sz: Size,
253        src1: Location,
254        src2: Location,
255        dst: Location,
256    ) -> Result<(), CompileError>;
257    fn emit_fmax(
258        &mut self,
259        sz: Size,
260        src1: Location,
261        src2: Location,
262        dst: Location,
263    ) -> Result<(), CompileError>;
264    fn emit_fdiv(
265        &mut self,
266        sz: Size,
267        src1: Location,
268        src2: Location,
269        dst: Location,
270    ) -> Result<(), CompileError>;
271    fn emit_fsqrt(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
272    fn emit_fcvt(
273        &mut self,
274        signed: bool,
275        sz_in: Size,
276        src: Location,
277        sz_out: Size,
278        dst: Location,
279    ) -> Result<(), CompileError>;
280    fn emit_fcvt_with_rounding(
281        &mut self,
282        rounding: RoundingMode,
283        size: Size,
284        src: Location,
285        dst: Location,
286        tmp: GPR,
287    ) -> Result<(), CompileError>;
288    fn emit_swap_fscr(&mut self, reg: GPR) -> Result<(), CompileError>;
289
290    // Control-flow related instructions
291    fn emit_cmp(
292        &mut self,
293        c: Condition,
294        loc_a: Location,
295        loc_b: Location,
296        ret: Location,
297    ) -> Result<(), CompileError>;
298    fn emit_fcmp(
299        &mut self,
300        c: Condition,
301        size: Size,
302        loc_a: Location,
303        loc_b: Location,
304        ret: Location,
305    ) -> Result<(), CompileError>;
306
307    fn emit_on_false_label(
308        &mut self,
309        cond: Location,
310        label: Label,
311        tmp: GPR,
312    ) -> Result<(), CompileError>;
313    fn emit_on_false_label_far(
314        &mut self,
315        cond: Location,
316        label: Label,
317        tmp: GPR,
318    ) -> Result<(), CompileError>;
319    fn emit_on_true_label_far(
320        &mut self,
321        cond: Location,
322        label: Label,
323        tmp: GPR,
324    ) -> Result<(), CompileError>;
325    fn emit_on_true_label(
326        &mut self,
327        cond: Location,
328        label: Label,
329        tmp: GPR,
330    ) -> Result<(), CompileError>;
331
332    fn emit_j_label(&mut self, label: Label, reg: Option<GPR>) -> Result<(), CompileError>;
333    fn emit_j_register(&mut self, reg: GPR) -> Result<(), CompileError>;
334    fn emit_load_label(&mut self, reg: GPR, label: Label) -> Result<(), CompileError>;
335    fn emit_call_label(&mut self, label: Label) -> Result<(), CompileError>;
336    fn emit_call_register(&mut self, reg: GPR) -> Result<(), CompileError>;
337
338    fn emit_rwfence(&mut self) -> Result<(), CompileError>;
339    fn emit_atomic_binop(
340        &mut self,
341        op: AtomicBinaryOp,
342        size: Size,
343        dest: GPR,
344        addr: GPR,
345        source: GPR,
346    ) -> Result<(), CompileError>;
347    fn emit_reserved_ld(&mut self, size: Size, reg: GPR, addr: GPR) -> Result<(), CompileError>;
348}
349
350impl EmitterRiscv for Assembler {
351    fn get_simd_arch(&self) -> Option<&CpuFeature> {
352        todo!()
353    }
354
355    fn get_label(&mut self) -> Label {
356        self.new_dynamic_label()
357    }
358
359    fn get_offset(&self) -> Offset {
360        self.offset()
361    }
362
363    fn get_jmp_instr_size(&self) -> u8 {
364        todo!()
365    }
366
367    fn finalize_function(&mut self) -> Result<(), CompileError> {
368        Ok(())
369    }
370
371    fn emit_label(&mut self, label: Label) -> Result<(), CompileError> {
372        dynasm!(self ; => label);
373        Ok(())
374    }
375
376    fn emit_brk(&mut self) -> Result<(), CompileError> {
377        dynasm!(self ; ebreak);
378        Ok(())
379    }
380
381    fn emit_udf(&mut self, payload: u8) -> Result<(), CompileError> {
382        dynasm!(self
383            ; li.12 a0, payload as _
384            ; unimp);
385        Ok(())
386    }
387
388    fn emit_ld(
389        &mut self,
390        sz: Size,
391        signed: bool,
392        reg: Location,
393        addr: Location,
394    ) -> Result<(), CompileError> {
395        let Location::Memory(addr, disp) = addr else {
396            codegen_error!("singlepass can't emit LD {:?}, {:?}, {:?}", sz, reg, addr);
397        };
398        assert!(ImmType::Bits12.compatible_imm(disp as i64));
399
400        match (sz, signed, reg) {
401            (Size::S64, _, Location::GPR(reg)) => {
402                dynasm!(self ; ld X(reg), [X(addr), disp]);
403            }
404            (Size::S32, false, Location::GPR(reg)) => {
405                dynasm!(self ; lwu X(reg), [X(addr), disp]);
406            }
407            (Size::S32, true, Location::GPR(reg)) => {
408                dynasm!(self ; lw X(reg), [X(addr), disp]);
409            }
410            (Size::S16, false, Location::GPR(reg)) => {
411                dynasm!(self ; lhu X(reg), [X(addr), disp]);
412            }
413            (Size::S16, true, Location::GPR(reg)) => {
414                dynasm!(self ; lh X(reg), [X(addr), disp]);
415            }
416            (Size::S8, false, Location::GPR(reg)) => {
417                dynasm!(self ; lbu X(reg), [X(addr), disp]);
418            }
419            (Size::S8, true, Location::GPR(reg)) => {
420                dynasm!(self ; lb X(reg), [X(addr), disp]);
421            }
422            (Size::S64, _, Location::SIMD(reg)) => {
423                dynasm!(self ; fld F(reg), [X(addr), disp]);
424            }
425            (Size::S32, _, Location::SIMD(reg)) => {
426                dynasm!(self ; flw F(reg), [X(addr), disp]);
427            }
428            _ => codegen_error!("singlepass can't emit LD {:?}, {:?}, {:?}", sz, reg, addr),
429        }
430        Ok(())
431    }
432
433    fn emit_sd(&mut self, sz: Size, reg: Location, addr: Location) -> Result<(), CompileError> {
434        let Location::Memory(addr, disp) = addr else {
435            codegen_error!("singlepass can't emit SD {:?}, {:?}, {:?}", sz, reg, addr);
436        };
437        assert!(ImmType::Bits12.compatible_imm(disp as i64));
438
439        match (sz, reg) {
440            (Size::S64, Location::GPR(reg)) => {
441                dynasm!(self ; sd X(reg), [X(addr), disp]);
442            }
443            (Size::S32, Location::GPR(reg)) => {
444                dynasm!(self ; sw X(reg), [X(addr), disp]);
445            }
446            (Size::S16, Location::GPR(reg)) => {
447                dynasm!(self ; sh X(reg), [X(addr), disp]);
448            }
449            (Size::S8, Location::GPR(reg)) => {
450                dynasm!(self ; sb X(reg), [X(addr), disp]);
451            }
452            (Size::S64, Location::SIMD(reg)) => {
453                dynasm!(self ; fsd F(reg), [X(addr), disp]);
454            }
455            (Size::S32, Location::SIMD(reg)) => {
456                dynasm!(self ; fsw F(reg), [X(addr), disp]);
457            }
458            _ => codegen_error!("singlepass can't emit SD {:?}, {:?}, {:?}", sz, reg, addr),
459        }
460        Ok(())
461    }
462
463    fn emit_push(&mut self, size: Size, src: Location) -> Result<(), CompileError> {
464        match (size, src) {
465            (Size::S64, Location::GPR(_)) => {
466                self.emit_sub(
467                    Size::S64,
468                    Location::GPR(GPR::Sp),
469                    Location::Imm64(8),
470                    Location::GPR(GPR::Sp),
471                )?;
472                self.emit_sd(Size::S64, src, Location::Memory(GPR::Sp, 0))?;
473            }
474            _ => codegen_error!("singlepass can't emit PUSH {:?} {:?}", size, src),
475        }
476        Ok(())
477    }
478
479    fn emit_pop(&mut self, size: Size, dst: Location) -> Result<(), CompileError> {
480        match (size, dst) {
481            (Size::S64, Location::GPR(_)) => {
482                self.emit_ld(Size::S64, false, dst, Location::Memory(GPR::Sp, 0))?;
483                self.emit_add(
484                    Size::S64,
485                    Location::GPR(GPR::Sp),
486                    Location::Imm64(8),
487                    Location::GPR(GPR::Sp),
488                )?;
489            }
490            _ => codegen_error!("singlepass can't emit POP {:?} {:?}", size, dst),
491        }
492        Ok(())
493    }
494
495    fn emit_add(
496        &mut self,
497        sz: Size,
498        src1: Location,
499        src2: Location,
500        dst: Location,
501    ) -> Result<(), CompileError> {
502        match (sz, src1, src2, dst) {
503            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
504                dynasm!(self ; add X(dst), X(src1), X(src2));
505            }
506            (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst))
507            | (Size::S64, Location::Imm64(imm), Location::GPR(src1), Location::GPR(dst)) => {
508                assert!(ImmType::Bits12.compatible_imm(imm as i64));
509                dynasm!(self ; addi X(dst), X(src1), imm as _);
510            }
511            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
512                dynasm!(self ; addw X(dst), X(src1), X(src2));
513            }
514            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
515            | (Size::S32, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
516                assert!(ImmType::Bits12.compatible_imm(imm as i64));
517                dynasm!(self ; addiw X(dst), X(src1), imm as _);
518            }
519
520            (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
521                dynasm!(self ; fadd.s F(dst), F(src1), F(src2));
522            }
523            (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
524                dynasm!(self ; fadd.d F(dst), F(src1), F(src2));
525            }
526            _ => codegen_error!(
527                "singlepass can't emit ADD {:?} {:?} {:?} {:?}",
528                sz,
529                src1,
530                src2,
531                dst
532            ),
533        }
534        Ok(())
535    }
536
537    fn emit_sub(
538        &mut self,
539        sz: Size,
540        src1: Location,
541        src2: Location,
542        dst: Location,
543    ) -> Result<(), CompileError> {
544        match (sz, src1, src2, dst) {
545            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
546                dynasm!(self ; sub X(dst), X(src1), X(src2));
547            }
548            (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
549                assert!(ImmType::Bits12Subtraction.compatible_imm(imm as i64));
550                dynasm!(self ; addi X(dst), X(src1), -(imm as i32) as _);
551            }
552            (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
553                assert!(ImmType::Bits12Subtraction.compatible_imm(imm as i64));
554                dynasm!(self ; addi X(dst), X(src1), -(imm as i32) as _);
555            }
556            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
557                dynasm!(self ; subw X(dst), X(src1), X(src2));
558            }
559            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
560                assert!(ImmType::Bits12Subtraction.compatible_imm(imm as i64));
561                dynasm!(self ; addiw X(dst), X(src1), -(imm as i32) as _);
562            }
563            (Size::S32, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
564                assert!(ImmType::Bits12Subtraction.compatible_imm(imm as i64));
565                dynasm!(self ; addiw X(dst), X(src1), -(imm as i32) as _);
566            }
567            (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
568                dynasm!(self ; fsub.s F(dst), F(src1), F(src2));
569            }
570            (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
571                dynasm!(self ; fsub.d F(dst), F(src1), F(src2));
572            }
573            _ => codegen_error!(
574                "singlepass can't emit SUB {:?} {:?} {:?} {:?}",
575                sz,
576                src1,
577                src2,
578                dst
579            ),
580        }
581        Ok(())
582    }
583
584    fn emit_mul(
585        &mut self,
586        sz: Size,
587        src1: Location,
588        src2: Location,
589        dst: Location,
590    ) -> Result<(), CompileError> {
591        match (sz, src1, src2, dst) {
592            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
593                dynasm!(self ; mul X(dst), X(src1), X(src2));
594            }
595            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
596                dynasm!(self ; mulw X(dst), X(src1), X(src2));
597            }
598
599            (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
600                dynasm!(self ; fmul.s F(dst), F(src1), F(src2));
601            }
602            (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
603                dynasm!(self ; fmul.d F(dst), F(src1), F(src2));
604            }
605
606            _ => codegen_error!(
607                "singlepass can't emit MUL {:?} {:?} {:?} {:?}",
608                sz,
609                src1,
610                src2,
611                dst
612            ),
613        }
614        Ok(())
615    }
616
617    fn emit_udiv(
618        &mut self,
619        sz: Size,
620        src1: Location,
621        src2: Location,
622        dst: Location,
623    ) -> Result<(), CompileError> {
624        match (sz, src1, src2, dst) {
625            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
626                dynasm!(self ; divuw X(dst), X(src1), X(src2));
627            }
628            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
629                dynasm!(self ; divu X(dst), X(src1), X(src2));
630            }
631            _ => codegen_error!(
632                "singlepass can't emit UDIV {:?} {:?} {:?} {:?}",
633                sz,
634                src1,
635                src2,
636                dst
637            ),
638        }
639        Ok(())
640    }
641    fn emit_sdiv(
642        &mut self,
643        sz: Size,
644        src1: Location,
645        src2: Location,
646        dst: Location,
647    ) -> Result<(), CompileError> {
648        match (sz, src1, src2, dst) {
649            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
650                dynasm!(self ; divw X(dst), X(src1), X(src2));
651            }
652            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
653                dynasm!(self ; div X(dst), X(src1), X(src2));
654            }
655            _ => codegen_error!(
656                "singlepass can't emit SDIV {:?} {:?} {:?} {:?}",
657                sz,
658                src1,
659                src2,
660                dst
661            ),
662        }
663        Ok(())
664    }
665
666    fn emit_urem(
667        &mut self,
668        sz: Size,
669        src1: Location,
670        src2: Location,
671        dst: Location,
672    ) -> Result<(), CompileError> {
673        match (sz, src1, src2, dst) {
674            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
675                dynasm!(self ; remuw X(dst), X(src1), X(src2));
676            }
677            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
678                dynasm!(self ; remu X(dst), X(src1), X(src2));
679            }
680            _ => codegen_error!(
681                "singlepass can't emit UREM {:?} {:?} {:?} {:?}",
682                sz,
683                src1,
684                src2,
685                dst
686            ),
687        }
688        Ok(())
689    }
690    fn emit_srem(
691        &mut self,
692        sz: Size,
693        src1: Location,
694        src2: Location,
695        dst: Location,
696    ) -> Result<(), CompileError> {
697        match (sz, src1, src2, dst) {
698            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
699                dynasm!(self ; remw X(dst), X(src1), X(src2));
700            }
701            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
702                dynasm!(self ; rem X(dst), X(src1), X(src2));
703            }
704            _ => codegen_error!(
705                "singlepass can't emit REM {:?} {:?} {:?} {:?}",
706                sz,
707                src1,
708                src2,
709                dst
710            ),
711        }
712        Ok(())
713    }
714
715    fn emit_and(
716        &mut self,
717        sz: Size,
718        src1: Location,
719        src2: Location,
720        dst: Location,
721    ) -> Result<(), CompileError> {
722        match (sz, src1, src2, dst) {
723            (
724                Size::S32 | Size::S64,
725                Location::GPR(src1),
726                Location::GPR(src2),
727                Location::GPR(dst),
728            ) => {
729                dynasm!(self ; and X(dst), X(src1), X(src2));
730            }
731            (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
732                assert!(ImmType::Bits12Subtraction.compatible_imm(imm as i64));
733                dynasm!(self ; andi X(dst), X(src1), imm as _);
734            }
735            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
736                assert!(ImmType::Bits12Subtraction.compatible_imm(imm as i64));
737                dynasm!(self ; andi X(dst), X(src1), imm as _);
738            }
739            _ => codegen_error!(
740                "singlepass can't emit AND {:?} {:?} {:?} {:?}",
741                sz,
742                src1,
743                src2,
744                dst
745            ),
746        }
747        Ok(())
748    }
749
750    fn emit_or(
751        &mut self,
752        sz: Size,
753        src1: Location,
754        src2: Location,
755        dst: Location,
756    ) -> Result<(), CompileError> {
757        match (sz, src1, src2, dst) {
758            (
759                Size::S32 | Size::S64,
760                Location::GPR(src1),
761                Location::GPR(src2),
762                Location::GPR(dst),
763            ) => {
764                dynasm!(self ; or X(dst), X(src1), X(src2));
765            }
766            (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
767                assert!(ImmType::Bits12Subtraction.compatible_imm(imm as i64));
768                dynasm!(self ; ori X(dst), X(src1), imm as _);
769            }
770            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
771                assert!(ImmType::Bits12Subtraction.compatible_imm(imm as i64));
772                dynasm!(self ; ori X(dst), X(src1), imm as _);
773            }
774            _ => codegen_error!(
775                "singlepass can't emit OR {:?} {:?} {:?} {:?}",
776                sz,
777                src1,
778                src2,
779                dst
780            ),
781        }
782        Ok(())
783    }
784
785    fn emit_xor(
786        &mut self,
787        sz: Size,
788        src1: Location,
789        src2: Location,
790        dst: Location,
791    ) -> Result<(), CompileError> {
792        match (sz, src1, src2, dst) {
793            (
794                Size::S32 | Size::S64,
795                Location::GPR(src1),
796                Location::GPR(src2),
797                Location::GPR(dst),
798            ) => {
799                dynasm!(self ; xor X(dst), X(src1), X(src2));
800            }
801            (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
802                assert!(ImmType::Bits12Subtraction.compatible_imm(imm as i64));
803                dynasm!(self ; xori X(dst), X(src1), imm as _);
804            }
805            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
806                assert!(ImmType::Bits12Subtraction.compatible_imm(imm as i64));
807                dynasm!(self ; xori X(dst), X(src1), imm as _);
808            }
809            _ => codegen_error!(
810                "singlepass can't emit XOR {:?} {:?} {:?} {:?}",
811                sz,
812                src1,
813                src2,
814                dst
815            ),
816        }
817        Ok(())
818    }
819
820    fn emit_sll(
821        &mut self,
822        sz: Size,
823        src1: Location,
824        src2: Location,
825        dst: Location,
826    ) -> Result<(), CompileError> {
827        match (sz, src1, src2, dst) {
828            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
829                dynasm!(self ; sll X(dst), X(src1), X(src2));
830            }
831            (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
832                if imm >= u64::BITS as _ {
833                    codegen_error!("singlepass SLL with incompatible imm {}", imm);
834                }
835                dynasm!(self ; slli X(dst), X(src1), imm as _);
836            }
837            (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
838                if imm >= u64::BITS as _ {
839                    codegen_error!("singlepass SLL with incompatible imm {}", imm);
840                }
841                dynasm!(self ; slli X(dst), X(src1), imm as _);
842            }
843            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
844                dynasm!(self ; sllw X(dst), X(src1), X(src2));
845            }
846            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
847                if imm >= u32::BITS {
848                    codegen_error!("singlepass SLL with incompatible imm {}", imm);
849                }
850                dynasm!(self ; slliw X(dst), X(src1), imm as _);
851            }
852            _ => codegen_error!(
853                "singlepass can't emit SLL {:?} {:?} {:?} {:?}",
854                sz,
855                src1,
856                src2,
857                dst
858            ),
859        }
860        Ok(())
861    }
862
863    fn emit_srl(
864        &mut self,
865        sz: Size,
866        src1: Location,
867        src2: Location,
868        dst: Location,
869    ) -> Result<(), CompileError> {
870        match (sz, src1, src2, dst) {
871            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
872                dynasm!(self ; srl X(dst), X(src1), X(src2));
873            }
874            (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
875                if imm >= u64::BITS as _ {
876                    codegen_error!("singlepass SRL with incompatible imm {}", imm);
877                }
878                dynasm!(self ; srli X(dst), X(src1), imm as _);
879            }
880            (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
881                if imm >= u64::BITS as _ {
882                    codegen_error!("singlepass SRL with incompatible imm {}", imm);
883                }
884                dynasm!(self ; srli X(dst), X(src1), imm as _);
885            }
886            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
887                dynasm!(self ; srlw X(dst), X(src1), X(src2));
888            }
889            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
890                if imm >= u32::BITS {
891                    codegen_error!("singlepass SRL with incompatible imm {}", imm);
892                }
893                dynasm!(self ; srliw X(dst), X(src1), imm as _);
894            }
895            (Size::S32, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
896                if imm >= u32::BITS as _ {
897                    codegen_error!("singlepass SRL with incompatible imm {}", imm);
898                }
899                dynasm!(self ; srliw X(dst), X(src1), imm as _);
900            }
901            (Size::S32, Location::Imm32(imm), Location::GPR(src2), Location::GPR(dst)) => {
902                self.emit_mov_imm(Location::GPR(dst), imm as i64)?;
903                dynasm!(self ; srlw X(dst), X(dst), X(src2));
904            }
905            _ => codegen_error!(
906                "singlepass can't emit SRL {:?} {:?} {:?} {:?}",
907                sz,
908                src1,
909                src2,
910                dst
911            ),
912        }
913        Ok(())
914    }
915
916    fn emit_sra(
917        &mut self,
918        sz: Size,
919        src1: Location,
920        src2: Location,
921        dst: Location,
922    ) -> Result<(), CompileError> {
923        match (sz, src1, src2, dst) {
924            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
925                dynasm!(self ; sra X(dst), X(src1), X(src2));
926            }
927            (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
928                if imm >= u64::BITS as _ {
929                    codegen_error!("singlepass SRA with incompatible imm {}", imm);
930                }
931                dynasm!(self ; srai X(dst), X(src1), imm as _);
932            }
933            (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
934                if imm >= u64::BITS as _ {
935                    codegen_error!("singlepass SRA with incompatible imm {}", imm);
936                }
937                dynasm!(self ; srai X(dst), X(src1), imm as _);
938            }
939            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
940                dynasm!(self ; sraw X(dst), X(src1), X(src2));
941            }
942            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
943                if imm >= u32::BITS {
944                    codegen_error!("singlepass SRA with incompatible imm {}", imm);
945                }
946                dynasm!(self ; sraiw X(dst), X(src1), imm as _);
947            }
948            _ => codegen_error!(
949                "singlepass can't emit SRA {:?} {:?} {:?} {:?}",
950                sz,
951                src1,
952                src2,
953                dst
954            ),
955        }
956        Ok(())
957    }
958
959    fn emit_extend(
960        &mut self,
961        sz: Size,
962        signed: bool,
963        src: Location,
964        dst: Location,
965    ) -> Result<(), CompileError> {
966        let bit_shift = match sz {
967            Size::S8 => 56,
968            Size::S16 => 48,
969            Size::S32 => 32,
970            _ => codegen_error!("singlepass can't emit SEXT {:?} {:?} {:?}", sz, src, dst),
971        };
972
973        match (signed, src, dst) {
974            (true, Location::GPR(src), Location::GPR(dst)) => {
975                dynasm!(self
976                    ; slli X(dst), X(src), bit_shift
977                    ; srai X(dst), X(dst), bit_shift
978                );
979            }
980            (false, Location::GPR(src), Location::GPR(dst)) => {
981                dynasm!(self
982                    ; slli X(dst), X(src), bit_shift
983                    ; srli X(dst), X(dst), bit_shift
984                );
985            }
986            _ => codegen_error!("singlepass can't emit SEXT {:?} {:?} {:?}", sz, src, dst),
987        };
988        Ok(())
989    }
990
991    fn emit_not(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
992        match (sz, src, dst) {
993            (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
994                dynasm!(self ; not X(dst), X(src));
995            }
996            _ => codegen_error!("singlepass can't emit NOT {:?} {:?} {:?}", sz, src, dst),
997        }
998
999        Ok(())
1000    }
1001
1002    fn emit_neg(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1003        match (sz, src, dst) {
1004            (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1005                dynasm!(self ; neg X(dst), X(src));
1006            }
1007            _ => codegen_error!("singlepass can't emit NEG {:?} {:?} {:?}", sz, src, dst),
1008        }
1009
1010        Ok(())
1011    }
1012
1013    fn emit_mov(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1014        match (sz, src, dst) {
1015            (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1016                dynasm!(self ; mv X(dst), X(src));
1017            }
1018            (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1019                dynasm!(self
1020                    ; slli X(dst), X(src), 32
1021                    ; srli X(dst), X(dst), 32
1022                );
1023            }
1024            (Size::S64, Location::GPR(src), Location::SIMD(dst)) => {
1025                dynasm!(self ; fmv.d.x F(dst), X(src));
1026            }
1027            (Size::S64, Location::SIMD(src), Location::GPR(dst)) => {
1028                dynasm!(self ; fmv.x.d X(dst), F(src));
1029            }
1030            (Size::S32, Location::GPR(src), Location::SIMD(dst)) => {
1031                dynasm!(self ; fmv.s.x F(dst), X(src));
1032            }
1033            (Size::S32, Location::SIMD(src), Location::GPR(dst)) => {
1034                dynasm!(self ; fmv.x.s X(dst), F(src));
1035            }
1036            (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
1037                dynasm!(self ; fmv.d F(dst), F(src));
1038            }
1039            (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
1040                dynasm!(self ; fmv.s F(dst), F(src));
1041            }
1042            _ => codegen_error!("singlepass can't emit MOV {:?} {:?} {:?}", sz, src, dst),
1043        }
1044
1045        Ok(())
1046    }
1047
1048    fn emit_ret(&mut self) -> Result<(), CompileError> {
1049        dynasm!(self ; ret);
1050        Ok(())
1051    }
1052
1053    fn emit_mov_imm(&mut self, dst: Location, val: i64) -> Result<(), CompileError> {
1054        // The number of used bits by the number including the sign bit.
1055        // i64::MIN.abs() will overflow, thus handle it specially.
1056        let used_bits = if val == i64::MIN {
1057            i64::BITS
1058        } else {
1059            i64::BITS - val.abs().leading_zeros() + 1
1060        };
1061
1062        match dst {
1063            Location::GPR(dst) => match used_bits {
1064                0..=12 => dynasm!(self ; li.12 X(dst), val as _),
1065                13..=32 => dynasm!(self ; li.32 X(dst), val as _),
1066                33..=43 => dynasm!(self ; li.43 X(dst), val),
1067                44..=54 => dynasm!(self ; li.54 X(dst), val),
1068                55..=64 => dynasm!(self ; li X(dst), val),
1069                _ => unreachable!(),
1070            },
1071            _ => codegen_error!("singlepass can't emit MOVIMM {:?}", dst),
1072        }
1073        Ok(())
1074    }
1075
1076    fn emit_fneg(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1077        match (sz, src, dst) {
1078            (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
1079                dynasm!(self ; fneg.d F(dst), F(src));
1080            }
1081            (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
1082                dynasm!(self ; fneg.s F(dst), F(src));
1083            }
1084            _ => codegen_error!("singlepass can't emit FNEG {:?} {:?} {:?}", sz, src, dst),
1085        }
1086
1087        Ok(())
1088    }
1089
1090    fn emit_fmin(
1091        &mut self,
1092        sz: Size,
1093        src1: Location,
1094        src2: Location,
1095        dst: Location,
1096    ) -> Result<(), CompileError> {
1097        match (sz, src1, src2, dst) {
1098            (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
1099                dynasm!(self ; fmin.s F(dst), F(src1), F(src2));
1100            }
1101            (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
1102                dynasm!(self ; fmin.d F(dst), F(src1), F(src2));
1103            }
1104            _ => codegen_error!(
1105                "singlepass can't emit FMIN {:?} {:?} {:?} {:?}",
1106                sz,
1107                src1,
1108                src2,
1109                dst
1110            ),
1111        }
1112        Ok(())
1113    }
1114
1115    fn emit_fmax(
1116        &mut self,
1117        sz: Size,
1118        src1: Location,
1119        src2: Location,
1120        dst: Location,
1121    ) -> Result<(), CompileError> {
1122        match (sz, src1, src2, dst) {
1123            (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
1124                dynasm!(self ; fmax.s F(dst), F(src1), F(src2));
1125            }
1126            (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
1127                dynasm!(self ; fmax.d F(dst), F(src1), F(src2));
1128            }
1129            _ => codegen_error!(
1130                "singlepass can't emit FMAX {:?} {:?} {:?} {:?}",
1131                sz,
1132                src1,
1133                src2,
1134                dst
1135            ),
1136        }
1137        Ok(())
1138    }
1139
1140    fn emit_fdiv(
1141        &mut self,
1142        sz: Size,
1143        src1: Location,
1144        src2: Location,
1145        dst: Location,
1146    ) -> Result<(), CompileError> {
1147        match (sz, src1, src2, dst) {
1148            (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
1149                dynasm!(self ; fdiv.s F(dst), F(src1), F(src2));
1150            }
1151            (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
1152                dynasm!(self ; fdiv.d F(dst), F(src1), F(src2));
1153            }
1154            _ => codegen_error!(
1155                "singlepass can't emit FDIV {:?} {:?} {:?} {:?}",
1156                sz,
1157                src1,
1158                src2,
1159                dst
1160            ),
1161        }
1162        Ok(())
1163    }
1164
1165    fn emit_fsqrt(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1166        match (sz, src, dst) {
1167            (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
1168                dynasm!(self ; fsqrt.s F(dst), F(src));
1169            }
1170            (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
1171                dynasm!(self ; fsqrt.d F(dst), F(src));
1172            }
1173            _ => codegen_error!("singlepass can't emit FSQRT {:?} {:?} {:?}", sz, src, dst),
1174        }
1175        Ok(())
1176    }
1177
1178    fn emit_fcvt(
1179        &mut self,
1180        signed: bool,
1181        sz_in: Size,
1182        src: Location,
1183        sz_out: Size,
1184        dst: Location,
1185    ) -> Result<(), CompileError> {
1186        match (signed, sz_in, src, sz_out, dst) {
1187            // floating-point -> floating-point types
1188            (_, Size::S32, Location::SIMD(src), Size::S64, Location::SIMD(dst)) => {
1189                dynasm!(self ; fcvt.d.s F(dst), F(src));
1190            }
1191            (_, Size::S64, Location::SIMD(src), Size::S32, Location::SIMD(dst)) => {
1192                dynasm!(self ; fcvt.s.d F(dst), F(src));
1193            }
1194            // floating-point -> int types
1195            (true, Size::S32, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
1196                dynasm!(self ; fcvt.w.s X(dst), F(src), rtz);
1197            }
1198            (true, Size::S64, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
1199                dynasm!(self ; fcvt.w.d X(dst), F(src), rtz);
1200            }
1201            (true, Size::S32, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
1202                dynasm!(self ; fcvt.l.s X(dst), F(src), rtz);
1203            }
1204            (true, Size::S64, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
1205                dynasm!(self ; fcvt.l.d X(dst), F(src), rtz);
1206            }
1207            (false, Size::S32, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
1208                dynasm!(self ; fcvt.wu.s X(dst), F(src), rtz);
1209            }
1210            (false, Size::S64, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
1211                dynasm!(self ; fcvt.wu.d X(dst), F(src), rtz);
1212            }
1213            (false, Size::S32, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
1214                dynasm!(self ; fcvt.lu.s X(dst), F(src), rtz);
1215            }
1216            (false, Size::S64, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
1217                dynasm!(self ; fcvt.lu.d X(dst), F(src), rtz);
1218            }
1219            // int -> floating-point types
1220            (true, Size::S32, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
1221                dynasm!(self ; fcvt.s.w F(dst), X(src));
1222            }
1223            (true, Size::S64, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
1224                dynasm!(self ; fcvt.s.l F(dst), X(src));
1225            }
1226            (true, Size::S32, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
1227                dynasm!(self ; fcvt.d.w F(dst), X(src));
1228            }
1229            (true, Size::S64, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
1230                dynasm!(self ; fcvt.d.l F(dst), X(src));
1231            }
1232
1233            (false, Size::S32, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
1234                dynasm!(self ; fcvt.s.wu F(dst), X(src));
1235            }
1236            (false, Size::S64, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
1237                dynasm!(self ; fcvt.s.lu F(dst), X(src));
1238            }
1239            (false, Size::S32, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
1240                dynasm!(self ; fcvt.d.wu F(dst), X(src));
1241            }
1242            (false, Size::S64, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
1243                dynasm!(self ; fcvt.d.lu F(dst), X(src));
1244            }
1245            _ => codegen_error!(
1246                "singlepass can't emit FCVT {:?} {:?} {:?} {:?}",
1247                sz_in,
1248                src,
1249                sz_out,
1250                dst
1251            ),
1252        }
1253        Ok(())
1254    }
1255
1256    fn emit_fcvt_with_rounding(
1257        &mut self,
1258        rounding: RoundingMode,
1259        size: Size,
1260        src: Location,
1261        dst: Location,
1262        tmp: GPR,
1263    ) -> Result<(), CompileError> {
1264        let (Location::SIMD(src), Location::SIMD(dst)) = (src, dst) else {
1265            codegen_error!(
1266                "singlepass can't emit FCVT with rounding for non-register operands: {:?} {:?} {:?} {:?}",
1267                src,
1268                dst,
1269                size,
1270                rounding
1271            )
1272        };
1273
1274        match (size, rounding) {
1275            (Size::S32, RoundingMode::Rne) => {
1276                dynasm!(self
1277                    ; fcvt.w.s X(tmp), F(src), rne
1278                    ; fcvt.s.w F(dst), X(tmp), rne
1279                    ; fsgnj.s F(dst), F(dst), F(src));
1280            }
1281            (Size::S32, RoundingMode::Rtz) => {
1282                dynasm!(self
1283                    ; fcvt.w.s X(tmp), F(src), rtz
1284                    ; fcvt.s.w F(dst), X(tmp), rtz
1285                    ; fsgnj.s F(dst), F(dst), F(src));
1286            }
1287            (Size::S32, RoundingMode::Rdn) => {
1288                dynasm!(self
1289                    ; fcvt.w.s X(tmp), F(src), rdn
1290                    ; fcvt.s.w F(dst), X(tmp), rdn
1291                    ; fsgnj.s F(dst), F(dst), F(src));
1292            }
1293            (Size::S32, RoundingMode::Rup) => {
1294                dynasm!(self
1295                    ; fcvt.w.s X(tmp), F(src), rup
1296                    ; fcvt.s.w F(dst), X(tmp), rup
1297                    ; fsgnj.s F(dst), F(dst), F(src));
1298            }
1299
1300            (Size::S64, RoundingMode::Rne) => {
1301                dynasm!(self
1302                    ; fcvt.l.d X(tmp), F(src), rne
1303                    ; fcvt.d.l F(dst), X(tmp), rne
1304                    ; fsgnj.d F(dst), F(dst), F(src));
1305            }
1306            (Size::S64, RoundingMode::Rtz) => {
1307                dynasm!(self
1308                    ; fcvt.l.d X(tmp), F(src), rtz
1309                    ; fcvt.d.l F(dst), X(tmp), rtz
1310                    ; fsgnj.d F(dst), F(dst), F(src));
1311            }
1312            (Size::S64, RoundingMode::Rdn) => {
1313                dynasm!(self
1314                    ; fcvt.l.d X(tmp), F(src), rdn
1315                    ; fcvt.d.l F(dst), X(tmp), rdn
1316                    ; fsgnj.d F(dst), F(dst), F(src));
1317            }
1318            (Size::S64, RoundingMode::Rup) => {
1319                dynasm!(self
1320                    ; fcvt.l.d X(tmp), F(src), rup
1321                    ; fcvt.d.l F(dst), X(tmp), rup
1322                    ; fsgnj.d F(dst), F(dst), F(src));
1323            }
1324            _ => codegen_error!(
1325                "singlepass can't emit FCVT with rounding {:?} {:?}",
1326                size,
1327                rounding
1328            ),
1329        }
1330
1331        Ok(())
1332    }
1333
1334    // Swap `source` register with FSCR.
1335    fn emit_swap_fscr(&mut self, reg: GPR) -> Result<(), CompileError> {
1336        dynasm!(self ; fscsr X(reg), X(reg));
1337        Ok(())
1338    }
1339
1340    fn emit_cmp(
1341        &mut self,
1342        c: Condition,
1343        loc_a: Location,
1344        loc_b: Location,
1345        ret: Location,
1346    ) -> Result<(), CompileError> {
1347        let (Location::GPR(loc_a), Location::GPR(loc_b), Location::GPR(ret)) = (loc_a, loc_b, ret)
1348        else {
1349            codegen_error!(
1350                "singlepass can't emit CMP {:?} {:?} {:?}",
1351                loc_a,
1352                loc_b,
1353                ret
1354            );
1355        };
1356
1357        match c {
1358            // signed comparison operations
1359            Condition::Lt => {
1360                dynasm!(self; slt X(ret), X(loc_a), X(loc_b));
1361            }
1362            Condition::Le => {
1363                dynasm!(self
1364                    ; slt X(ret), X(loc_b), X(loc_a)
1365                    ; xori X(ret), X(ret), 1);
1366            }
1367            Condition::Gt => {
1368                dynasm!(self; slt X(ret), X(loc_b), X(loc_a));
1369            }
1370            Condition::Ge => {
1371                dynasm!(self
1372                    ; slt X(ret), X(loc_a), X(loc_b)
1373                    ; xori X(ret), X(ret), 1);
1374            }
1375            Condition::Eq => {
1376                dynasm!(self
1377                    ; xor X(ret), X(loc_a), X(loc_b)
1378                    ; seqz X(ret), X(ret));
1379            }
1380            Condition::Ne => {
1381                dynasm!(self
1382                    ; xor X(ret), X(loc_a), X(loc_b)
1383                    ; snez X(ret), X(ret));
1384            }
1385            // unsigned comparison operations
1386            Condition::Ltu => {
1387                dynasm!(self; sltu X(ret), X(loc_a), X(loc_b));
1388            }
1389            Condition::Leu => {
1390                dynasm!(self
1391                    ; sltu X(ret), X(loc_b), X(loc_a)
1392                    ; xori X(ret), X(ret), 1);
1393            }
1394            Condition::Gtu => {
1395                dynasm!(self; sltu X(ret), X(loc_b), X(loc_a));
1396            }
1397            Condition::Geu => {
1398                dynasm!(self
1399                    ; sltu X(ret), X(loc_a), X(loc_b)
1400                    ; xori X(ret), X(ret), 1);
1401            }
1402        }
1403
1404        Ok(())
1405    }
1406
1407    fn emit_fcmp(
1408        &mut self,
1409        c: Condition,
1410        size: Size,
1411        loc_a: Location,
1412        loc_b: Location,
1413        ret: Location,
1414    ) -> Result<(), CompileError> {
1415        let (Location::SIMD(loc_a), Location::SIMD(loc_b), Location::GPR(ret)) =
1416            (loc_a, loc_b, ret)
1417        else {
1418            codegen_error!(
1419                "singlepass can't emit FCMP {:?} {:?} {:?}",
1420                loc_a,
1421                loc_b,
1422                ret
1423            );
1424        };
1425
1426        match (size, c) {
1427            (Size::S32, Condition::Lt) => {
1428                dynasm!(self ; flt.s X(ret), F(loc_a), F(loc_b));
1429            }
1430            (Size::S32, Condition::Le) => {
1431                dynasm!(self ; fle.s X(ret), F(loc_a), F(loc_b));
1432            }
1433            (Size::S32, Condition::Gt) => {
1434                dynasm!(self ; flt.s X(ret), F(loc_b), F(loc_a));
1435            }
1436            (Size::S32, Condition::Ge) => {
1437                dynasm!(self ; fle.s X(ret), F(loc_b), F(loc_a));
1438            }
1439            (Size::S32, Condition::Eq) => {
1440                dynasm!(self ; feq.s X(ret), F(loc_a), F(loc_b));
1441            }
1442            (Size::S32, Condition::Ne) => {
1443                dynasm!(self
1444                    ; feq.s X(ret), F(loc_a), F(loc_b)
1445                    ; xori X(ret), X(ret), 1);
1446            }
1447
1448            (Size::S64, Condition::Lt) => {
1449                dynasm!(self ; flt.d X(ret), F(loc_a), F(loc_b));
1450            }
1451            (Size::S64, Condition::Le) => {
1452                dynasm!(self ; fle.d X(ret), F(loc_a), F(loc_b));
1453            }
1454            (Size::S64, Condition::Gt) => {
1455                dynasm!(self ; flt.d X(ret), F(loc_b), F(loc_a));
1456            }
1457            (Size::S64, Condition::Ge) => {
1458                dynasm!(self; fle.d X(ret), F(loc_b), F(loc_a));
1459            }
1460            (Size::S64, Condition::Eq) => {
1461                dynasm!(self ; feq.d X(ret), F(loc_a), F(loc_b));
1462            }
1463            (Size::S64, Condition::Ne) => {
1464                dynasm!(self
1465                    ; feq.d X(ret), F(loc_a), F(loc_b)
1466                    ; xori X(ret), X(ret), 1);
1467            }
1468            _ => codegen_error!(
1469                "singlepass can't emit FCMP {:?} {:?} {:?}",
1470                loc_a,
1471                loc_b,
1472                ret
1473            ),
1474        }
1475
1476        Ok(())
1477    }
1478
1479    fn emit_jmp_on_condition(
1480        &mut self,
1481        cond: Condition,
1482        loc_a: Location,
1483        loc_b: Location,
1484        label: Label,
1485        tmp: GPR,
1486    ) -> Result<(), CompileError> {
1487        let (Location::GPR(loc_a), Location::GPR(loc_b)) = (loc_a, loc_b) else {
1488            codegen_error!("singlepass can't emit JMP_ON_COND {:?} {:?} ", loc_a, loc_b,);
1489        };
1490
1491        let jump: Label = self.get_label();
1492        let cont: Label = self.get_label();
1493
1494        match cond {
1495            Condition::Eq => {
1496                dynasm!(self; beq X(loc_a), X(loc_b), => jump);
1497            }
1498            Condition::Ne => {
1499                dynasm!(self; bne X(loc_a), X(loc_b), => jump);
1500            }
1501            Condition::Ltu => {
1502                dynasm!(self; bltu X(loc_a), X(loc_b), => jump);
1503            }
1504            Condition::Leu => {
1505                dynasm!(self; bleu X(loc_a), X(loc_b), => jump);
1506            }
1507            Condition::Gtu => {
1508                dynasm!(self; bgtu X(loc_a), X(loc_b), => jump);
1509            }
1510            Condition::Geu => {
1511                dynasm!(self; bgeu X(loc_a), X(loc_b), => jump);
1512            }
1513            _ => codegen_error!(
1514                "singlepass can't emit jump on conditional branch {:?}",
1515                cond
1516            ),
1517        }
1518
1519        self.emit_j_label(cont, Some(tmp))?;
1520        self.emit_label(jump)?;
1521        self.emit_j_label(label, Some(tmp))?;
1522
1523        self.emit_label(cont)?;
1524
1525        Ok(())
1526    }
1527
1528    fn emit_on_false_label(
1529        &mut self,
1530        cond: Location,
1531        label: Label,
1532        tmp: GPR,
1533    ) -> Result<(), CompileError> {
1534        match cond {
1535            Location::GPR(cond) => {
1536                dynasm!(self; beqz X(cond), => label);
1537            }
1538            _ if cond.is_imm() => {
1539                let imm = cond.imm_value_scalar().unwrap();
1540                if imm == 0 {
1541                    return self.emit_j_label(label, Some(tmp));
1542                }
1543            }
1544            _ => codegen_error!("singlepass can't emit jump to false branch {:?}", cond),
1545        }
1546        Ok(())
1547    }
1548    fn emit_on_false_label_far(
1549        &mut self,
1550        cond: Location,
1551        label: Label,
1552        tmp: GPR,
1553    ) -> Result<(), CompileError> {
1554        let cont: Label = self.get_label();
1555        match cond {
1556            Location::GPR(cond) => {
1557                // Use the negative condition to jump after the "j" instruction that will
1558                // go to the requsted `label`.
1559                dynasm!(self; bnez X(cond), => cont);
1560            }
1561            _ if cond.is_imm() => {
1562                let imm = cond.imm_value_scalar().unwrap();
1563                if imm == 0 {
1564                    return self.emit_j_label(label, Some(tmp));
1565                } else {
1566                    self.emit_j_label(cont, Some(tmp))?;
1567                }
1568            }
1569            _ => codegen_error!("singlepass can't emit jump to false branch {:?}", cond),
1570        }
1571
1572        self.emit_j_label(label, Some(tmp))?;
1573        self.emit_label(cont)?;
1574        Ok(())
1575    }
1576    fn emit_on_true_label(
1577        &mut self,
1578        cond: Location,
1579        label: Label,
1580        tmp: GPR,
1581    ) -> Result<(), CompileError> {
1582        match cond {
1583            Location::GPR(cond) => {
1584                dynasm!(self; bnez X(cond), => label);
1585            }
1586            _ if cond.is_imm() => {
1587                let imm = cond.imm_value_scalar().unwrap();
1588                if imm != 0 {
1589                    return self.emit_j_label(label, Some(tmp));
1590                }
1591            }
1592            _ => codegen_error!("singlepass can't emit jump to true branch {:?}", cond),
1593        }
1594        Ok(())
1595    }
1596    fn emit_on_true_label_far(
1597        &mut self,
1598        cond: Location,
1599        label: Label,
1600        tmp: GPR,
1601    ) -> Result<(), CompileError> {
1602        let cont: Label = self.get_label();
1603        let jump_label: Label = self.get_label();
1604        match cond {
1605            Location::GPR(cond) => {
1606                dynasm!(self; bnez X(cond), => jump_label);
1607            }
1608            _ if cond.is_imm() => {
1609                let imm = cond.imm_value_scalar().unwrap();
1610                if imm == 1 {
1611                    return self.emit_j_label(jump_label, Some(tmp));
1612                }
1613            }
1614            _ => codegen_error!("singlepass can't emit jump to false branch {:?}", cond),
1615        }
1616
1617        // skip over the jump to target
1618        self.emit_j_label(cont, Some(tmp))?;
1619
1620        self.emit_label(jump_label)?;
1621        self.emit_j_label(label, Some(tmp))?;
1622
1623        self.emit_label(cont)?;
1624
1625        Ok(())
1626    }
1627
1628    fn emit_j_label(&mut self, label: Label, reg: Option<GPR>) -> Result<(), CompileError> {
1629        if let Some(reg) = reg {
1630            dynasm!(self ; jump => label, X(reg));
1631        } else {
1632            dynasm!(self ; j => label);
1633        }
1634        Ok(())
1635    }
1636
1637    fn emit_j_register(&mut self, reg: GPR) -> Result<(), CompileError> {
1638        dynasm!(self ; jalr zero, X(reg), 0);
1639        Ok(())
1640    }
1641
1642    fn emit_call_label(&mut self, label: Label) -> Result<(), CompileError> {
1643        dynasm!(self ; call =>label);
1644        Ok(())
1645    }
1646
1647    fn emit_call_register(&mut self, reg: GPR) -> Result<(), CompileError> {
1648        dynasm!(self ; jalr ra, X(reg), 0);
1649        Ok(())
1650    }
1651
1652    fn emit_load_label(&mut self, reg: GPR, label: Label) -> Result<(), CompileError> {
1653        dynasm!(self ; la X(reg), => label);
1654        Ok(())
1655    }
1656
1657    fn emit_rwfence(&mut self) -> Result<(), CompileError> {
1658        dynasm!(self ; fence rw, rw);
1659        Ok(())
1660    }
1661
1662    fn emit_atomic_binop(
1663        &mut self,
1664        op: AtomicBinaryOp,
1665        size: Size,
1666        dest: GPR,
1667        addr: GPR,
1668        source: GPR,
1669    ) -> Result<(), CompileError> {
1670        match (size, op) {
1671            (Size::S64, AtomicBinaryOp::Add) => {
1672                dynasm!(self ; amoadd.d.aqrl X(dest), X(source), [X(addr)])
1673            }
1674            (Size::S64, AtomicBinaryOp::Or) => {
1675                dynasm!(self ; amoor.d.aqrl X(dest), X(source), [X(addr)])
1676            }
1677            (Size::S64, AtomicBinaryOp::Xor) => {
1678                dynasm!(self ; amoxor.d.aqrl X(dest), X(source), [X(addr)])
1679            }
1680            (Size::S64, AtomicBinaryOp::And) => {
1681                dynasm!(self ; amoand.d.aqrl X(dest), X(source), [X(addr)])
1682            }
1683            (Size::S64, AtomicBinaryOp::Exchange) => {
1684                dynasm!(self ; amoswap.d.aqrl X(dest), X(source), [X(addr)])
1685            }
1686
1687            (Size::S32, AtomicBinaryOp::Add) => {
1688                dynasm!(self ; amoadd.w.aqrl X(dest), X(source), [X(addr)])
1689            }
1690            (Size::S32, AtomicBinaryOp::Or) => {
1691                dynasm!(self ; amoor.w.aqrl X(dest), X(source), [X(addr)])
1692            }
1693            (Size::S32, AtomicBinaryOp::Xor) => {
1694                dynasm!(self ; amoxor.w.aqrl X(dest), X(source), [X(addr)])
1695            }
1696            (Size::S32, AtomicBinaryOp::And) => {
1697                dynasm!(self ; amoand.w.aqrl X(dest), X(source), [X(addr)])
1698            }
1699            (Size::S32, AtomicBinaryOp::Exchange) => {
1700                dynasm!(self ; amoswap.w.aqrl X(dest), X(source), [X(addr)])
1701            }
1702            _ => codegen_error!("singlepass can't emit atomic binop {:?} {:?}", size, op),
1703        }
1704
1705        Ok(())
1706    }
1707
1708    fn emit_reserved_ld(&mut self, size: Size, reg: GPR, addr: GPR) -> Result<(), CompileError> {
1709        match size {
1710            Size::S32 => {
1711                dynasm!(self ; lr.w.aqrl X(reg), [X(addr)])
1712            }
1713            Size::S64 => {
1714                dynasm!(self ; lr.d.aqrl X(reg), [X(addr)])
1715            }
1716            _ => codegen_error!("singlepass can't emit atomic reserved ld {:?}", size),
1717        }
1718
1719        Ok(())
1720    }
1721
1722    fn emit_reserved_sd(
1723        &mut self,
1724        size: Size,
1725        dest: GPR,
1726        addr: GPR,
1727        source: GPR,
1728    ) -> Result<(), CompileError> {
1729        match size {
1730            Size::S32 => {
1731                dynasm!(self ; sc.w.rl X(dest), X(source), [X(addr)])
1732            }
1733            Size::S64 => {
1734                dynasm!(self ; sc.d.rl X(dest), X(source), [X(addr)])
1735            }
1736            _ => codegen_error!("singlepass can't emit atomic reserved sd {:?}", size),
1737        }
1738
1739        Ok(())
1740    }
1741}
1742
1743pub fn gen_std_trampoline_riscv(
1744    sig: &FunctionType,
1745    _calling_convention: CallingConvention,
1746) -> Result<FunctionBody, CompileError> {
1747    let mut a = Assembler::new(0);
1748
1749    // Callee-save registers must be used.
1750    let fptr = GPR::X26;
1751    let args = GPR::X27;
1752
1753    dynasm!(a
1754        ; addi sp, sp, -32
1755        ; sd s0, [sp,24]
1756        ; sd ra, [sp,16]
1757        ; sd X(fptr), [sp, 8]
1758        ; sd X(args), [sp, 0]
1759        ; mv s0, sp // use frame-pointer register for later restore
1760        ; mv X(fptr), a1
1761        ; mv X(args), a2
1762    );
1763
1764    let stack_args = sig.params().len().saturating_sub(7); //1st arg is ctx, not an actual arg
1765    let stack_return_slots = sig
1766        .results()
1767        .len()
1768        .saturating_sub(RISCV_RETURN_VALUE_REGISTERS.len());
1769    let mut stack_offset = (stack_args + stack_return_slots) as u32 * 8;
1770    if stack_offset > 0 {
1771        stack_offset = stack_offset.next_multiple_of(16);
1772        if ImmType::Bits12Subtraction.compatible_imm(stack_offset as _) {
1773            dynasm!(a ; addi sp, sp, -(stack_offset as i32));
1774        } else {
1775            a.emit_mov_imm(Location::GPR(SCRATCH_REG), stack_offset as _)?;
1776            dynasm!(a ; sub sp, sp, X(SCRATCH_REG));
1777        }
1778    }
1779
1780    // Move arguments to their locations.
1781    // `callee_vmctx` is already in the first argument register, so no need to move.
1782    let mut caller_stack_offset: i32 = stack_return_slots as i32 * 8;
1783    for (i, param) in sig.params().iter().enumerate() {
1784        let sz = match *param {
1785            Type::I32 | Type::F32 => Size::S32,
1786            Type::I64 | Type::F64 => Size::S64,
1787            Type::ExternRef => Size::S64,
1788            Type::FuncRef => Size::S64,
1789            _ => codegen_error!(
1790                "singlepass unsupported param type for trampoline {:?}",
1791                *param
1792            ),
1793        };
1794        match i {
1795            0..=6 => {
1796                a.emit_ld(
1797                    sz,
1798                    false,
1799                    Location::GPR(GPR::from_index(i + 10 + 1).unwrap()),
1800                    Location::Memory(args, (i * 16) as i32),
1801                )?;
1802            }
1803            _ => {
1804                let args_offset = (i * 16) as i32;
1805                let arg_location = if ImmType::Bits12.compatible_imm(args_offset as i64) {
1806                    Location::Memory(args, args_offset)
1807                } else {
1808                    a.emit_mov_imm(Location::GPR(SCRATCH_REG), args_offset as i64)?;
1809                    a.emit_add(
1810                        Size::S64,
1811                        Location::GPR(args),
1812                        Location::GPR(SCRATCH_REG),
1813                        Location::GPR(SCRATCH_REG),
1814                    )?;
1815                    Location::Memory(SCRATCH_REG, 0)
1816                };
1817                a.emit_ld(sz, false, Location::GPR(SCRATCH_REG), arg_location)?;
1818
1819                let arg_dest_location =
1820                    if ImmType::Bits12.compatible_imm(caller_stack_offset as i64) {
1821                        Location::Memory(GPR::Sp, caller_stack_offset)
1822                    } else {
1823                        a.emit_mov_imm(Location::GPR(GPR::X29), caller_stack_offset as i64)?;
1824                        a.emit_add(
1825                            Size::S64,
1826                            Location::GPR(GPR::X29),
1827                            Location::GPR(GPR::Sp),
1828                            Location::GPR(GPR::X29),
1829                        )?;
1830                        Location::Memory(GPR::X29, 0)
1831                    };
1832                a.emit_sd(sz, Location::GPR(SCRATCH_REG), arg_dest_location)?;
1833                caller_stack_offset += 8;
1834            }
1835        }
1836    }
1837
1838    dynasm!(a
1839        ; jalr ra, X(fptr), 0);
1840
1841    // Write return values.
1842    let mut n_stack_return_slots: usize = 0;
1843    for i in 0..sig.results().len() {
1844        let src = if let Some(&reg) = RISCV_RETURN_VALUE_REGISTERS.get(i) {
1845            reg
1846        } else {
1847            a.emit_ld(
1848                Size::S64,
1849                false,
1850                Location::GPR(SCRATCH_REG),
1851                Location::Memory(GPR::Sp, (n_stack_return_slots as u32 * 8) as _),
1852            )?;
1853            n_stack_return_slots += 1;
1854            SCRATCH_REG
1855        };
1856        a.emit_sd(
1857            Size::S64,
1858            Location::GPR(src),
1859            Location::Memory(args, (i * 16) as _),
1860        )?;
1861    }
1862
1863    // Restore stack.
1864    dynasm!(a
1865        ; ld X(args), [s0,0]
1866        ; ld X(fptr), [s0,8]
1867        ; ld ra, [s0,16]
1868        ; ld s0, [s0,24]
1869    );
1870    let restored_stack_offset = 32 + stack_offset;
1871    if ImmType::Bits12.compatible_imm(restored_stack_offset as _) {
1872        dynasm!(a; addi sp, sp, restored_stack_offset as _);
1873    } else {
1874        a.emit_mov_imm(Location::GPR(SCRATCH_REG), restored_stack_offset as _)?;
1875        dynasm!(a; add sp, sp, X(SCRATCH_REG));
1876    }
1877    dynasm!(a; ret);
1878
1879    let mut body = a.finalize().unwrap();
1880
1881    body.shrink_to_fit();
1882    Ok(FunctionBody {
1883        body,
1884        unwind_info: None,
1885    })
1886}
1887
1888/// Generates dynamic import function call trampoline for a function type.
1889pub fn gen_std_dynamic_import_trampoline_riscv(
1890    vmoffsets: &VMOffsets,
1891    sig: &FunctionType,
1892) -> Result<FunctionBody, CompileError> {
1893    let mut a = Assembler::new(0);
1894    // Allocate argument array.
1895    let stack_offset: usize = 16 * std::cmp::max(sig.params().len(), sig.results().len());
1896
1897    // Save RA and the scratch register
1898    a.emit_push(Size::S64, Location::GPR(GPR::X1))?;
1899    a.emit_push(Size::S64, Location::GPR(SCRATCH_REG))?;
1900
1901    if stack_offset != 0 {
1902        if ImmType::Bits12Subtraction.compatible_imm(stack_offset as _) {
1903            a.emit_sub(
1904                Size::S64,
1905                Location::GPR(GPR::Sp),
1906                Location::Imm64(stack_offset as _),
1907                Location::GPR(GPR::Sp),
1908            )?;
1909        } else {
1910            a.emit_mov_imm(Location::GPR(SCRATCH_REG), stack_offset as _)?;
1911            a.emit_sub(
1912                Size::S64,
1913                Location::GPR(GPR::Sp),
1914                Location::GPR(SCRATCH_REG),
1915                Location::GPR(GPR::Sp),
1916            )?;
1917        }
1918    }
1919
1920    // Copy arguments.
1921    if !sig.params().is_empty() {
1922        let mut argalloc = ArgumentRegisterAllocator::default();
1923        argalloc.next(Type::I64).unwrap(); // skip VMContext
1924
1925        let mut stack_param_count: usize = 0;
1926
1927        for (i, ty) in sig.params().iter().enumerate() {
1928            let source_loc = match argalloc.next(*ty)? {
1929                Some(RiscvRegister::GPR(gpr)) => Location::GPR(gpr),
1930                Some(RiscvRegister::FPR(fpr)) => Location::SIMD(fpr),
1931                None => {
1932                    a.emit_ld(
1933                        Size::S64,
1934                        false,
1935                        Location::GPR(SCRATCH_REG),
1936                        Location::Memory(GPR::Sp, (stack_offset + 16 + stack_param_count) as _),
1937                    )?;
1938                    stack_param_count += 8;
1939                    Location::GPR(SCRATCH_REG)
1940                }
1941            };
1942            a.emit_sd(
1943                Size::S64,
1944                source_loc,
1945                Location::Memory(GPR::Sp, (i * 16) as _),
1946            )?;
1947            // Zero upper 64 bits.
1948            a.emit_sd(
1949                Size::S64,
1950                Location::GPR(GPR::XZero),
1951                Location::Memory(GPR::Sp, (i * 16 + 8) as _),
1952            )?;
1953        }
1954    }
1955
1956    // Load target address.
1957    let offset = vmoffsets.vmdynamicfunction_import_context_address();
1958    a.emit_ld(
1959        Size::S64,
1960        false,
1961        Location::GPR(SCRATCH_REG),
1962        Location::Memory(GPR::X10, offset as i32),
1963    )?;
1964    // Load values array.
1965    a.emit_add(
1966        Size::S64,
1967        Location::GPR(GPR::Sp),
1968        Location::Imm64(0),
1969        Location::GPR(GPR::X11),
1970    )?;
1971
1972    // Call target.
1973    a.emit_call_register(SCRATCH_REG)?;
1974
1975    // Fetch return value.
1976    if !sig.results().is_empty() {
1977        assert_eq!(sig.results().len(), 1);
1978        a.emit_ld(
1979            Size::S64,
1980            false,
1981            Location::GPR(GPR::X10),
1982            Location::Memory(GPR::Sp, 0),
1983        )?;
1984    }
1985
1986    // Release values array.
1987    if stack_offset != 0 {
1988        if ImmType::Bits12.compatible_imm(stack_offset as _) {
1989            a.emit_add(
1990                Size::S64,
1991                Location::GPR(GPR::Sp),
1992                Location::Imm64(stack_offset as _),
1993                Location::GPR(GPR::Sp),
1994            )?;
1995        } else {
1996            a.emit_mov_imm(Location::GPR(SCRATCH_REG), stack_offset as _)?;
1997            a.emit_add(
1998                Size::S64,
1999                Location::GPR(GPR::Sp),
2000                Location::GPR(SCRATCH_REG),
2001                Location::GPR(GPR::Sp),
2002            )?;
2003        }
2004    }
2005    a.emit_pop(Size::S64, Location::GPR(SCRATCH_REG))?;
2006    a.emit_pop(Size::S64, Location::GPR(GPR::X1))?;
2007
2008    // Return.
2009    a.emit_ret()?;
2010
2011    let mut body = a.finalize().unwrap();
2012    body.shrink_to_fit();
2013    Ok(FunctionBody {
2014        body,
2015        unwind_info: None,
2016    })
2017}
2018
2019// Singlepass calls import functions through a trampoline.
2020pub fn gen_import_call_trampoline_riscv(
2021    vmoffsets: &VMOffsets,
2022    index: FunctionIndex,
2023    sig: &FunctionType,
2024    _calling_convention: CallingConvention,
2025) -> Result<CustomSection, CompileError> {
2026    let mut a = Assembler::new(0);
2027
2028    // Singlepass internally treats all arguments as integers
2029    // For the standard System V calling convention requires
2030    //  floating point arguments to be passed in FPR registers.
2031    //  Translation is expensive, so only do it if needed.
2032    if sig
2033        .params()
2034        .iter()
2035        .any(|&x| x == Type::F32 || x == Type::F64)
2036    {
2037        const PARAM_REGS: &[GPR] = &[
2038            // X10 is skipped intentionally
2039            GPR::X11,
2040            GPR::X12,
2041            GPR::X13,
2042            GPR::X14,
2043            GPR::X15,
2044            GPR::X16,
2045            GPR::X17,
2046        ];
2047        const PARAM_REGS_COUNT: usize = PARAM_REGS.len();
2048
2049        // Allocate stack space for arguments.
2050        let stack_offset: i32 = if sig.params().len() > PARAM_REGS_COUNT {
2051            (PARAM_REGS_COUNT * 8) as i32
2052        } else {
2053            (sig.params().len() as i32) * 8
2054        };
2055        if stack_offset > 0 {
2056            if ImmType::Bits12Subtraction.compatible_imm(stack_offset as _) {
2057                a.emit_sub(
2058                    Size::S64,
2059                    Location::GPR(GPR::Sp),
2060                    Location::Imm64(stack_offset as _),
2061                    Location::GPR(GPR::Sp),
2062                )?;
2063            } else {
2064                a.emit_mov_imm(Location::GPR(SCRATCH_REG), stack_offset as _)?;
2065                a.emit_sub(
2066                    Size::S64,
2067                    Location::GPR(GPR::Sp),
2068                    Location::GPR(SCRATCH_REG),
2069                    Location::GPR(GPR::Sp),
2070                )?;
2071            }
2072        }
2073
2074        // Store all arguments to the stack to prevent overwrite.
2075
2076        let mut param_locations = vec![];
2077        /* Clippy is wrong about using `i` to index `PARAM_REGS` here. */
2078        #[allow(clippy::needless_range_loop)]
2079        for i in 0..sig.params().len() {
2080            let loc = match i {
2081                0..PARAM_REGS_COUNT => {
2082                    let loc = Location::Memory(GPR::Sp, (i * 8) as i32);
2083                    a.emit_sd(Size::S64, Location::GPR(PARAM_REGS[i]), loc)?;
2084                    loc
2085                }
2086                _ => Location::Memory(GPR::Sp, stack_offset + ((i - PARAM_REGS_COUNT) * 8) as i32),
2087            };
2088            param_locations.push(loc);
2089        }
2090
2091        // Copy arguments.
2092        let mut caller_stack_offset: i32 = 0;
2093        let mut argalloc = ArgumentRegisterAllocator::default();
2094        argalloc.next(Type::I64).unwrap(); // skip VMContext
2095        for (i, ty) in sig.params().iter().enumerate() {
2096            let prev_loc = param_locations[i];
2097            let targ = match argalloc.next(*ty)? {
2098                Some(RiscvRegister::GPR(gpr)) => Location::GPR(gpr),
2099                Some(RiscvRegister::FPR(neon)) => Location::SIMD(neon),
2100                None => {
2101                    // No register can be allocated. Put this argument on the stack.
2102                    a.emit_ld(Size::S64, false, Location::GPR(SCRATCH_REG), prev_loc)?;
2103                    a.emit_sd(
2104                        Size::S64,
2105                        Location::GPR(SCRATCH_REG),
2106                        Location::Memory(GPR::Sp, stack_offset + caller_stack_offset),
2107                    )?;
2108                    caller_stack_offset += 8;
2109                    continue;
2110                }
2111            };
2112            a.emit_ld(Size::S64, false, targ, prev_loc)?;
2113        }
2114
2115        // Restore stack pointer.
2116        if stack_offset > 0 {
2117            if ImmType::Bits12.compatible_imm(stack_offset as _) {
2118                a.emit_add(
2119                    Size::S64,
2120                    Location::GPR(GPR::Sp),
2121                    Location::Imm64(stack_offset as _),
2122                    Location::GPR(GPR::Sp),
2123                )?;
2124            } else {
2125                a.emit_mov_imm(Location::GPR(SCRATCH_REG), stack_offset as _)?;
2126                a.emit_add(
2127                    Size::S64,
2128                    Location::GPR(GPR::Sp),
2129                    Location::GPR(SCRATCH_REG),
2130                    Location::GPR(GPR::Sp),
2131                )?;
2132            }
2133        }
2134    }
2135
2136    // Emits a tail call trampoline that loads the address of the target import function
2137    // from Ctx and jumps to it.
2138
2139    let offset = vmoffsets.vmctx_vmfunction_import(index);
2140
2141    a.emit_ld(
2142        Size::S64,
2143        false,
2144        Location::GPR(SCRATCH_REG),
2145        Location::Memory(GPR::X10, offset as i32), // function pointer
2146    )?;
2147    a.emit_ld(
2148        Size::S64,
2149        false,
2150        Location::GPR(GPR::X10),
2151        Location::Memory(GPR::X10, offset as i32 + 8), // target vmctx
2152    )?;
2153
2154    a.emit_j_register(SCRATCH_REG)?;
2155
2156    let mut contents = a.finalize().unwrap();
2157    contents.shrink_to_fit();
2158    let section_body = SectionBody::new_with_vec(contents);
2159
2160    Ok(CustomSection {
2161        protection: CustomSectionProtection::ReadExecute,
2162        alignment: None,
2163        bytes: section_body,
2164        relocations: vec![],
2165    })
2166}
wasmer_compiler_singlepass/emitter_riscv.rs

wasmer_compiler_singlepass/
emitter_riscv.rs
