Skip to main content

wasmer_compiler_singlepass/
emitter_riscv.rs

1//! RISC-V emitter scaffolding.
2
3use crate::{
4    codegen_error,
5    common_decl::Size,
6    location::{Location as AbstractLocation, Reg},
7    machine::MaybeImmediate,
8    machine_riscv::{ImmType, RISCV_RETURN_VALUE_REGISTERS},
9    output_reporter::ChunkedOutputReporter,
10    riscv_decl::{ArgumentRegisterAllocator, RiscvRegister},
11};
12pub use crate::{
13    machine::{Label, Offset},
14    riscv_decl::{FPR, GPR},
15};
16use dynasmrt::{DynasmApi, DynasmLabelApi, VecAssembler, riscv::RiscvRelocation};
17use wasmer_compiler::types::{
18    function::FunctionBody,
19    section::{CustomSection, CustomSectionProtection, SectionBody},
20};
21use wasmer_types::{
22    CompilationProgressCallback, CompileError, FunctionIndex, FunctionType, Type, VMOffsets,
23    target::{CallingConvention, CpuFeature},
24    vmctx_offset,
25};
26
27type Assembler = VecAssembler<RiscvRelocation>;
28
29/// Force `dynasm!` to use the correct arch (riscv64) when cross-compiling.
30macro_rules! dynasm {
31    ($a:expr ; $($tt:tt)*) => {
32        dynasm::dynasm!(
33            $a
34            ; .arch riscv64
35            ; .feature g
36            ; $($tt)*
37        )
38    };
39}
40
41/// Location abstraction specialized to RISC-V.
42pub type Location = AbstractLocation<GPR, FPR>;
43
44/// Branch conditions for RISC-V.
45#[derive(Copy, Clone, Debug, Eq, PartialEq)]
46pub enum Condition {
47    /// Signed less than
48    Lt,
49    /// Signed less than or equal
50    Le,
51    /// Signed greater than
52    Gt,
53    /// Signed greater than or equal
54    Ge,
55    /// Signed equal
56    Eq,
57    /// Signed not equal
58    Ne,
59
60    /// Unsigned less than
61    Ltu,
62    /// Unsigned less than or equal
63    Leu,
64    /// Unsigned greater than
65    Gtu,
66    /// Unsigned greater than or equal
67    Geu,
68}
69
70/// Floating-point number rounding mode
71#[derive(Copy, Clone, Debug, Eq, PartialEq)]
72pub enum RoundingMode {
73    /// Round to Nearest, ties to Even (default)
74    Rne,
75    /// Round towards Zero
76    Rtz,
77    /// Round Down (towards -∞)
78    Rdn,
79    /// Round Up (towards +∞)
80    Rup,
81}
82
83/// Atomic binary operation type
84#[derive(Copy, Clone, Debug, Eq, PartialEq)]
85pub enum AtomicBinaryOp {
86    Add,
87    Sub,
88    Or,
89    And,
90    Xor,
91    Exchange,
92}
93
94/// Scratch register used in function call trampolines.
95const SCRATCH_REG: GPR = GPR::X28;
96
97/// Emitter trait for RISC-V.
98#[allow(unused)]
99pub trait EmitterRiscv {
100    fn emit_jmp_on_condition(
101        &mut self,
102        cond: Condition,
103        loc_a: Location,
104        loc_b: Location,
105        label: Label,
106        tmp: GPR,
107    ) -> Result<(), CompileError>;
108    fn emit_reserved_sd(
109        &mut self,
110        size: Size,
111        dest: GPR,
112        addr: GPR,
113        source: GPR,
114    ) -> Result<(), CompileError>;
115    /// Returns the SIMD (FPU) feature if available.
116    fn get_simd_arch(&self) -> Option<&CpuFeature>;
117    /// Generates a new internal label.
118    fn get_label(&mut self) -> Label;
119    /// Gets the current code offset.
120    fn get_offset(&self) -> Offset;
121    /// Returns the size of a jump instruction in bytes.
122    fn get_jmp_instr_size(&self) -> u8;
123
124    /// Finalize the function, e.g., resolve labels.
125    fn finalize_function(&mut self) -> Result<(), CompileError>;
126
127    fn emit_label(&mut self, label: Label) -> Result<(), CompileError>;
128    fn emit_brk(&mut self) -> Result<(), CompileError>;
129    fn emit_ld(
130        &mut self,
131        sz: Size,
132        signed: bool,
133        reg: Location,
134        addr: Location,
135    ) -> Result<(), CompileError>;
136
137    fn emit_sd(&mut self, sz: Size, reg: Location, addr: Location) -> Result<(), CompileError>;
138    fn emit_push(&mut self, size: Size, src: Location) -> Result<(), CompileError>;
139    fn emit_pop(&mut self, size: Size, dst: Location) -> Result<(), CompileError>;
140
141    fn emit_add(
142        &mut self,
143        sz: Size,
144        src1: Location,
145        src2: Location,
146        dst: Location,
147    ) -> Result<(), CompileError>;
148    fn emit_sub(
149        &mut self,
150        sz: Size,
151        src1: Location,
152        src2: Location,
153        dst: Location,
154    ) -> Result<(), CompileError>;
155    fn emit_mul(
156        &mut self,
157        sz: Size,
158        src1: Location,
159        src2: Location,
160        dst: Location,
161    ) -> Result<(), CompileError>;
162    fn emit_sdiv(
163        &mut self,
164        sz: Size,
165        src1: Location,
166        src2: Location,
167        dst: Location,
168    ) -> Result<(), CompileError>;
169    fn emit_udiv(
170        &mut self,
171        sz: Size,
172        src1: Location,
173        src2: Location,
174        dst: Location,
175    ) -> Result<(), CompileError>;
176    fn emit_srem(
177        &mut self,
178        sz: Size,
179        src1: Location,
180        src2: Location,
181        dst: Location,
182    ) -> Result<(), CompileError>;
183    fn emit_urem(
184        &mut self,
185        sz: Size,
186        src1: Location,
187        src2: Location,
188        dst: Location,
189    ) -> Result<(), CompileError>;
190    fn emit_and(
191        &mut self,
192        sz: Size,
193        src1: Location,
194        src2: Location,
195        dst: Location,
196    ) -> Result<(), CompileError>;
197    fn emit_or(
198        &mut self,
199        sz: Size,
200        src1: Location,
201        src2: Location,
202        dst: Location,
203    ) -> Result<(), CompileError>;
204    fn emit_xor(
205        &mut self,
206        sz: Size,
207        src1: Location,
208        src2: Location,
209        dst: Location,
210    ) -> Result<(), CompileError>;
211    fn emit_sll(
212        &mut self,
213        sz: Size,
214        src1: Location,
215        src2: Location,
216        dst: Location,
217    ) -> Result<(), CompileError>;
218    fn emit_srl(
219        &mut self,
220        sz: Size,
221        src1: Location,
222        src2: Location,
223        dst: Location,
224    ) -> Result<(), CompileError>;
225    fn emit_sra(
226        &mut self,
227        sz: Size,
228        src1: Location,
229        src2: Location,
230        dst: Location,
231    ) -> Result<(), CompileError>;
232    fn emit_extend(
233        &mut self,
234        sz: Size,
235        signed: bool,
236        src: Location,
237        dst: Location,
238    ) -> Result<(), CompileError>;
239    fn emit_not(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
240    fn emit_neg(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
241
242    fn emit_mov(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
243
244    fn emit_ret(&mut self) -> Result<(), CompileError>;
245
246    fn emit_udf(&mut self, payload: u8) -> Result<(), CompileError>;
247
248    fn emit_mov_imm(&mut self, dst: Location, val: i64) -> Result<(), CompileError>;
249
250    // Floating-point type instructions
251    fn emit_fneg(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
252    fn emit_fmin(
253        &mut self,
254        sz: Size,
255        src1: Location,
256        src2: Location,
257        dst: Location,
258    ) -> Result<(), CompileError>;
259    fn emit_fmax(
260        &mut self,
261        sz: Size,
262        src1: Location,
263        src2: Location,
264        dst: Location,
265    ) -> Result<(), CompileError>;
266    fn emit_fdiv(
267        &mut self,
268        sz: Size,
269        src1: Location,
270        src2: Location,
271        dst: Location,
272    ) -> Result<(), CompileError>;
273    fn emit_fsqrt(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
274    fn emit_fcvt(
275        &mut self,
276        signed: bool,
277        sz_in: Size,
278        src: Location,
279        sz_out: Size,
280        dst: Location,
281    ) -> Result<(), CompileError>;
282    fn emit_fcvt_with_rounding(
283        &mut self,
284        rounding: RoundingMode,
285        size: Size,
286        src: Location,
287        dst: Location,
288        tmp: GPR,
289    ) -> Result<(), CompileError>;
290    fn emit_swap_fscr(&mut self, reg: GPR) -> Result<(), CompileError>;
291
292    // Control-flow related instructions
293    fn emit_cmp(
294        &mut self,
295        c: Condition,
296        loc_a: Location,
297        loc_b: Location,
298        ret: Location,
299    ) -> Result<(), CompileError>;
300    fn emit_fcmp(
301        &mut self,
302        c: Condition,
303        size: Size,
304        loc_a: Location,
305        loc_b: Location,
306        ret: Location,
307    ) -> Result<(), CompileError>;
308
309    fn emit_on_false_label(
310        &mut self,
311        cond: Location,
312        label: Label,
313        tmp: GPR,
314    ) -> Result<(), CompileError>;
315    fn emit_on_false_label_far(
316        &mut self,
317        cond: Location,
318        label: Label,
319        tmp: GPR,
320    ) -> Result<(), CompileError>;
321    fn emit_on_true_label_far(
322        &mut self,
323        cond: Location,
324        label: Label,
325        tmp: GPR,
326    ) -> Result<(), CompileError>;
327    fn emit_on_true_label(
328        &mut self,
329        cond: Location,
330        label: Label,
331        tmp: GPR,
332    ) -> Result<(), CompileError>;
333
334    fn emit_j_label(&mut self, label: Label, reg: Option<GPR>) -> Result<(), CompileError>;
335    fn emit_j_register(&mut self, reg: GPR) -> Result<(), CompileError>;
336    fn emit_load_label(&mut self, reg: GPR, label: Label) -> Result<(), CompileError>;
337    fn emit_call_label(&mut self, label: Label) -> Result<(), CompileError>;
338    fn emit_call_register(&mut self, reg: GPR) -> Result<(), CompileError>;
339
340    fn emit_rwfence(&mut self) -> Result<(), CompileError>;
341    fn emit_atomic_binop(
342        &mut self,
343        op: AtomicBinaryOp,
344        size: Size,
345        dest: GPR,
346        addr: GPR,
347        source: GPR,
348    ) -> Result<(), CompileError>;
349    fn emit_reserved_ld(&mut self, size: Size, reg: GPR, addr: GPR) -> Result<(), CompileError>;
350}
351
352impl EmitterRiscv for Assembler {
353    fn get_simd_arch(&self) -> Option<&CpuFeature> {
354        todo!()
355    }
356
357    fn get_label(&mut self) -> Label {
358        self.new_dynamic_label()
359    }
360
361    fn get_offset(&self) -> Offset {
362        self.offset()
363    }
364
365    fn get_jmp_instr_size(&self) -> u8 {
366        todo!()
367    }
368
369    fn finalize_function(&mut self) -> Result<(), CompileError> {
370        Ok(())
371    }
372
373    fn emit_label(&mut self, label: Label) -> Result<(), CompileError> {
374        dynasm!(self ; => label);
375        Ok(())
376    }
377
378    fn emit_brk(&mut self) -> Result<(), CompileError> {
379        dynasm!(self ; ebreak);
380        Ok(())
381    }
382
383    fn emit_udf(&mut self, payload: u8) -> Result<(), CompileError> {
384        dynasm!(self
385            ; li.12 a0, payload as _
386            ; unimp);
387        Ok(())
388    }
389
390    fn emit_ld(
391        &mut self,
392        sz: Size,
393        signed: bool,
394        reg: Location,
395        addr: Location,
396    ) -> Result<(), CompileError> {
397        let Location::Memory(addr, disp) = addr else {
398            codegen_error!("singlepass can't emit LD {:?}, {:?}, {:?}", sz, reg, addr);
399        };
400        assert!(ImmType::Bits12.compatible_imm(disp as i64));
401
402        match (sz, signed, reg) {
403            (Size::S64, _, Location::GPR(reg)) => {
404                dynasm!(self ; ld X(reg), [X(addr), disp]);
405            }
406            (Size::S32, false, Location::GPR(reg)) => {
407                dynasm!(self ; lwu X(reg), [X(addr), disp]);
408            }
409            (Size::S32, true, Location::GPR(reg)) => {
410                dynasm!(self ; lw X(reg), [X(addr), disp]);
411            }
412            (Size::S16, false, Location::GPR(reg)) => {
413                dynasm!(self ; lhu X(reg), [X(addr), disp]);
414            }
415            (Size::S16, true, Location::GPR(reg)) => {
416                dynasm!(self ; lh X(reg), [X(addr), disp]);
417            }
418            (Size::S8, false, Location::GPR(reg)) => {
419                dynasm!(self ; lbu X(reg), [X(addr), disp]);
420            }
421            (Size::S8, true, Location::GPR(reg)) => {
422                dynasm!(self ; lb X(reg), [X(addr), disp]);
423            }
424            (Size::S64, _, Location::SIMD(reg)) => {
425                dynasm!(self ; fld F(reg), [X(addr), disp]);
426            }
427            (Size::S32, _, Location::SIMD(reg)) => {
428                dynasm!(self ; flw F(reg), [X(addr), disp]);
429            }
430            _ => codegen_error!("singlepass can't emit LD {:?}, {:?}, {:?}", sz, reg, addr),
431        }
432        Ok(())
433    }
434
435    fn emit_sd(&mut self, sz: Size, reg: Location, addr: Location) -> Result<(), CompileError> {
436        let Location::Memory(addr, disp) = addr else {
437            codegen_error!("singlepass can't emit SD {:?}, {:?}, {:?}", sz, reg, addr);
438        };
439        assert!(ImmType::Bits12.compatible_imm(disp as i64));
440
441        match (sz, reg) {
442            (Size::S64, Location::GPR(reg)) => {
443                dynasm!(self ; sd X(reg), [X(addr), disp]);
444            }
445            (Size::S32, Location::GPR(reg)) => {
446                dynasm!(self ; sw X(reg), [X(addr), disp]);
447            }
448            (Size::S16, Location::GPR(reg)) => {
449                dynasm!(self ; sh X(reg), [X(addr), disp]);
450            }
451            (Size::S8, Location::GPR(reg)) => {
452                dynasm!(self ; sb X(reg), [X(addr), disp]);
453            }
454            (Size::S64, Location::SIMD(reg)) => {
455                dynasm!(self ; fsd F(reg), [X(addr), disp]);
456            }
457            (Size::S32, Location::SIMD(reg)) => {
458                dynasm!(self ; fsw F(reg), [X(addr), disp]);
459            }
460            _ => codegen_error!("singlepass can't emit SD {:?}, {:?}, {:?}", sz, reg, addr),
461        }
462        Ok(())
463    }
464
465    fn emit_push(&mut self, size: Size, src: Location) -> Result<(), CompileError> {
466        match (size, src) {
467            (Size::S64, Location::GPR(_)) => {
468                self.emit_sub(
469                    Size::S64,
470                    Location::GPR(GPR::Sp),
471                    Location::Imm64(8),
472                    Location::GPR(GPR::Sp),
473                )?;
474                self.emit_sd(Size::S64, src, Location::Memory(GPR::Sp, 0))?;
475            }
476            _ => codegen_error!("singlepass can't emit PUSH {:?} {:?}", size, src),
477        }
478        Ok(())
479    }
480
481    fn emit_pop(&mut self, size: Size, dst: Location) -> Result<(), CompileError> {
482        match (size, dst) {
483            (Size::S64, Location::GPR(_)) => {
484                self.emit_ld(Size::S64, false, dst, Location::Memory(GPR::Sp, 0))?;
485                self.emit_add(
486                    Size::S64,
487                    Location::GPR(GPR::Sp),
488                    Location::Imm64(8),
489                    Location::GPR(GPR::Sp),
490                )?;
491            }
492            _ => codegen_error!("singlepass can't emit POP {:?} {:?}", size, dst),
493        }
494        Ok(())
495    }
496
497    fn emit_add(
498        &mut self,
499        sz: Size,
500        src1: Location,
501        src2: Location,
502        dst: Location,
503    ) -> Result<(), CompileError> {
504        match (sz, src1, src2, dst) {
505            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
506                dynasm!(self ; add X(dst), X(src1), X(src2));
507            }
508            (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst))
509            | (Size::S64, Location::Imm64(imm), Location::GPR(src1), Location::GPR(dst)) => {
510                assert!(ImmType::Bits12.compatible_imm(imm as i64));
511                dynasm!(self ; addi X(dst), X(src1), imm as _);
512            }
513            (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
514            | (Size::S64, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
515                assert!(ImmType::Bits12.compatible_imm(imm as i64));
516                dynasm!(self ; addi X(dst), X(src1), imm as _);
517            }
518            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
519                dynasm!(self ; addw X(dst), X(src1), X(src2));
520            }
521            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
522            | (Size::S32, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
523                assert!(ImmType::Bits12.compatible_imm(imm as i64));
524                dynasm!(self ; addiw X(dst), X(src1), imm as _);
525            }
526
527            (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
528                dynasm!(self ; fadd.s F(dst), F(src1), F(src2));
529            }
530            (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
531                dynasm!(self ; fadd.d F(dst), F(src1), F(src2));
532            }
533            _ => codegen_error!(
534                "singlepass can't emit ADD {:?} {:?} {:?} {:?}",
535                sz,
536                src1,
537                src2,
538                dst
539            ),
540        }
541        Ok(())
542    }
543
544    fn emit_sub(
545        &mut self,
546        sz: Size,
547        src1: Location,
548        src2: Location,
549        dst: Location,
550    ) -> Result<(), CompileError> {
551        match (sz, src1, src2, dst) {
552            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
553                dynasm!(self ; sub X(dst), X(src1), X(src2));
554            }
555            (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
556                assert!(ImmType::Bits12Subtraction.compatible_imm(imm as i64));
557                dynasm!(self ; addi X(dst), X(src1), -(imm as i32) as _);
558            }
559            (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
560                assert!(ImmType::Bits12Subtraction.compatible_imm(imm as i64));
561                dynasm!(self ; addi X(dst), X(src1), -(imm as i32) as _);
562            }
563            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
564                dynasm!(self ; subw X(dst), X(src1), X(src2));
565            }
566            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
567                assert!(ImmType::Bits12Subtraction.compatible_imm(imm as i64));
568                dynasm!(self ; addiw X(dst), X(src1), -(imm as i32) as _);
569            }
570            (Size::S32, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
571                assert!(ImmType::Bits12Subtraction.compatible_imm(imm as i64));
572                dynasm!(self ; addiw X(dst), X(src1), -(imm as i32) as _);
573            }
574            (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
575                dynasm!(self ; fsub.s F(dst), F(src1), F(src2));
576            }
577            (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
578                dynasm!(self ; fsub.d F(dst), F(src1), F(src2));
579            }
580            _ => codegen_error!(
581                "singlepass can't emit SUB {:?} {:?} {:?} {:?}",
582                sz,
583                src1,
584                src2,
585                dst
586            ),
587        }
588        Ok(())
589    }
590
591    fn emit_mul(
592        &mut self,
593        sz: Size,
594        src1: Location,
595        src2: Location,
596        dst: Location,
597    ) -> Result<(), CompileError> {
598        match (sz, src1, src2, dst) {
599            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
600                dynasm!(self ; mul X(dst), X(src1), X(src2));
601            }
602            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
603                dynasm!(self ; mulw X(dst), X(src1), X(src2));
604            }
605
606            (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
607                dynasm!(self ; fmul.s F(dst), F(src1), F(src2));
608            }
609            (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
610                dynasm!(self ; fmul.d F(dst), F(src1), F(src2));
611            }
612
613            _ => codegen_error!(
614                "singlepass can't emit MUL {:?} {:?} {:?} {:?}",
615                sz,
616                src1,
617                src2,
618                dst
619            ),
620        }
621        Ok(())
622    }
623
624    fn emit_udiv(
625        &mut self,
626        sz: Size,
627        src1: Location,
628        src2: Location,
629        dst: Location,
630    ) -> Result<(), CompileError> {
631        match (sz, src1, src2, dst) {
632            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
633                dynasm!(self ; divuw X(dst), X(src1), X(src2));
634            }
635            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
636                dynasm!(self ; divu X(dst), X(src1), X(src2));
637            }
638            _ => codegen_error!(
639                "singlepass can't emit UDIV {:?} {:?} {:?} {:?}",
640                sz,
641                src1,
642                src2,
643                dst
644            ),
645        }
646        Ok(())
647    }
648    fn emit_sdiv(
649        &mut self,
650        sz: Size,
651        src1: Location,
652        src2: Location,
653        dst: Location,
654    ) -> Result<(), CompileError> {
655        match (sz, src1, src2, dst) {
656            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
657                dynasm!(self ; divw X(dst), X(src1), X(src2));
658            }
659            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
660                dynasm!(self ; div X(dst), X(src1), X(src2));
661            }
662            _ => codegen_error!(
663                "singlepass can't emit SDIV {:?} {:?} {:?} {:?}",
664                sz,
665                src1,
666                src2,
667                dst
668            ),
669        }
670        Ok(())
671    }
672
673    fn emit_urem(
674        &mut self,
675        sz: Size,
676        src1: Location,
677        src2: Location,
678        dst: Location,
679    ) -> Result<(), CompileError> {
680        match (sz, src1, src2, dst) {
681            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
682                dynasm!(self ; remuw X(dst), X(src1), X(src2));
683            }
684            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
685                dynasm!(self ; remu X(dst), X(src1), X(src2));
686            }
687            _ => codegen_error!(
688                "singlepass can't emit UREM {:?} {:?} {:?} {:?}",
689                sz,
690                src1,
691                src2,
692                dst
693            ),
694        }
695        Ok(())
696    }
697    fn emit_srem(
698        &mut self,
699        sz: Size,
700        src1: Location,
701        src2: Location,
702        dst: Location,
703    ) -> Result<(), CompileError> {
704        match (sz, src1, src2, dst) {
705            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
706                dynasm!(self ; remw X(dst), X(src1), X(src2));
707            }
708            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
709                dynasm!(self ; rem X(dst), X(src1), X(src2));
710            }
711            _ => codegen_error!(
712                "singlepass can't emit REM {:?} {:?} {:?} {:?}",
713                sz,
714                src1,
715                src2,
716                dst
717            ),
718        }
719        Ok(())
720    }
721
722    fn emit_and(
723        &mut self,
724        sz: Size,
725        src1: Location,
726        src2: Location,
727        dst: Location,
728    ) -> Result<(), CompileError> {
729        match (sz, src1, src2, dst) {
730            (
731                Size::S32 | Size::S64,
732                Location::GPR(src1),
733                Location::GPR(src2),
734                Location::GPR(dst),
735            ) => {
736                dynasm!(self ; and X(dst), X(src1), X(src2));
737            }
738            (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
739                assert!(ImmType::Bits12Subtraction.compatible_imm(imm as i64));
740                dynasm!(self ; andi X(dst), X(src1), imm as _);
741            }
742            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
743                assert!(ImmType::Bits12Subtraction.compatible_imm(imm as i64));
744                dynasm!(self ; andi X(dst), X(src1), imm as _);
745            }
746            _ => codegen_error!(
747                "singlepass can't emit AND {:?} {:?} {:?} {:?}",
748                sz,
749                src1,
750                src2,
751                dst
752            ),
753        }
754        Ok(())
755    }
756
757    fn emit_or(
758        &mut self,
759        sz: Size,
760        src1: Location,
761        src2: Location,
762        dst: Location,
763    ) -> Result<(), CompileError> {
764        match (sz, src1, src2, dst) {
765            (
766                Size::S32 | Size::S64,
767                Location::GPR(src1),
768                Location::GPR(src2),
769                Location::GPR(dst),
770            ) => {
771                dynasm!(self ; or X(dst), X(src1), X(src2));
772            }
773            (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
774                assert!(ImmType::Bits12Subtraction.compatible_imm(imm as i64));
775                dynasm!(self ; ori X(dst), X(src1), imm as _);
776            }
777            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
778                assert!(ImmType::Bits12Subtraction.compatible_imm(imm as i64));
779                dynasm!(self ; ori X(dst), X(src1), imm as _);
780            }
781            _ => codegen_error!(
782                "singlepass can't emit OR {:?} {:?} {:?} {:?}",
783                sz,
784                src1,
785                src2,
786                dst
787            ),
788        }
789        Ok(())
790    }
791
792    fn emit_xor(
793        &mut self,
794        sz: Size,
795        src1: Location,
796        src2: Location,
797        dst: Location,
798    ) -> Result<(), CompileError> {
799        match (sz, src1, src2, dst) {
800            (
801                Size::S32 | Size::S64,
802                Location::GPR(src1),
803                Location::GPR(src2),
804                Location::GPR(dst),
805            ) => {
806                dynasm!(self ; xor X(dst), X(src1), X(src2));
807            }
808            (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
809                assert!(ImmType::Bits12Subtraction.compatible_imm(imm as i64));
810                dynasm!(self ; xori X(dst), X(src1), imm as _);
811            }
812            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
813                assert!(ImmType::Bits12Subtraction.compatible_imm(imm as i64));
814                dynasm!(self ; xori X(dst), X(src1), imm as _);
815            }
816            _ => codegen_error!(
817                "singlepass can't emit XOR {:?} {:?} {:?} {:?}",
818                sz,
819                src1,
820                src2,
821                dst
822            ),
823        }
824        Ok(())
825    }
826
827    fn emit_sll(
828        &mut self,
829        sz: Size,
830        src1: Location,
831        src2: Location,
832        dst: Location,
833    ) -> Result<(), CompileError> {
834        match (sz, src1, src2, dst) {
835            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
836                dynasm!(self ; sll X(dst), X(src1), X(src2));
837            }
838            (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
839                if imm >= u64::BITS as _ {
840                    codegen_error!("singlepass SLL with incompatible imm {}", imm);
841                }
842                dynasm!(self ; slli X(dst), X(src1), imm as _);
843            }
844            (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
845                if imm >= u64::BITS as _ {
846                    codegen_error!("singlepass SLL with incompatible imm {}", imm);
847                }
848                dynasm!(self ; slli X(dst), X(src1), imm as _);
849            }
850            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
851                dynasm!(self ; sllw X(dst), X(src1), X(src2));
852            }
853            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
854                if imm >= u32::BITS {
855                    codegen_error!("singlepass SLL with incompatible imm {}", imm);
856                }
857                dynasm!(self ; slliw X(dst), X(src1), imm as _);
858            }
859            _ => codegen_error!(
860                "singlepass can't emit SLL {:?} {:?} {:?} {:?}",
861                sz,
862                src1,
863                src2,
864                dst
865            ),
866        }
867        Ok(())
868    }
869
870    fn emit_srl(
871        &mut self,
872        sz: Size,
873        src1: Location,
874        src2: Location,
875        dst: Location,
876    ) -> Result<(), CompileError> {
877        match (sz, src1, src2, dst) {
878            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
879                dynasm!(self ; srl X(dst), X(src1), X(src2));
880            }
881            (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
882                if imm >= u64::BITS as _ {
883                    codegen_error!("singlepass SRL with incompatible imm {}", imm);
884                }
885                dynasm!(self ; srli X(dst), X(src1), imm as _);
886            }
887            (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
888                if imm >= u64::BITS as _ {
889                    codegen_error!("singlepass SRL with incompatible imm {}", imm);
890                }
891                dynasm!(self ; srli X(dst), X(src1), imm as _);
892            }
893            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
894                dynasm!(self ; srlw X(dst), X(src1), X(src2));
895            }
896            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
897                if imm >= u32::BITS {
898                    codegen_error!("singlepass SRL with incompatible imm {}", imm);
899                }
900                dynasm!(self ; srliw X(dst), X(src1), imm as _);
901            }
902            (Size::S32, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
903                if imm >= u32::BITS as _ {
904                    codegen_error!("singlepass SRL with incompatible imm {}", imm);
905                }
906                dynasm!(self ; srliw X(dst), X(src1), imm as _);
907            }
908            (Size::S32, Location::Imm32(imm), Location::GPR(src2), Location::GPR(dst)) => {
909                self.emit_mov_imm(Location::GPR(dst), imm as i64)?;
910                dynasm!(self ; srlw X(dst), X(dst), X(src2));
911            }
912            _ => codegen_error!(
913                "singlepass can't emit SRL {:?} {:?} {:?} {:?}",
914                sz,
915                src1,
916                src2,
917                dst
918            ),
919        }
920        Ok(())
921    }
922
923    fn emit_sra(
924        &mut self,
925        sz: Size,
926        src1: Location,
927        src2: Location,
928        dst: Location,
929    ) -> Result<(), CompileError> {
930        match (sz, src1, src2, dst) {
931            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
932                dynasm!(self ; sra X(dst), X(src1), X(src2));
933            }
934            (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
935                if imm >= u64::BITS as _ {
936                    codegen_error!("singlepass SRA with incompatible imm {}", imm);
937                }
938                dynasm!(self ; srai X(dst), X(src1), imm as _);
939            }
940            (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
941                if imm >= u64::BITS as _ {
942                    codegen_error!("singlepass SRA with incompatible imm {}", imm);
943                }
944                dynasm!(self ; srai X(dst), X(src1), imm as _);
945            }
946            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
947                dynasm!(self ; sraw X(dst), X(src1), X(src2));
948            }
949            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
950                if imm >= u32::BITS {
951                    codegen_error!("singlepass SRA with incompatible imm {}", imm);
952                }
953                dynasm!(self ; sraiw X(dst), X(src1), imm as _);
954            }
955            _ => codegen_error!(
956                "singlepass can't emit SRA {:?} {:?} {:?} {:?}",
957                sz,
958                src1,
959                src2,
960                dst
961            ),
962        }
963        Ok(())
964    }
965
966    fn emit_extend(
967        &mut self,
968        sz: Size,
969        signed: bool,
970        src: Location,
971        dst: Location,
972    ) -> Result<(), CompileError> {
973        let bit_shift = match sz {
974            Size::S8 => 56,
975            Size::S16 => 48,
976            Size::S32 => 32,
977            _ => codegen_error!("singlepass can't emit SEXT {:?} {:?} {:?}", sz, src, dst),
978        };
979
980        match (signed, src, dst) {
981            (true, Location::GPR(src), Location::GPR(dst)) => {
982                dynasm!(self
983                    ; slli X(dst), X(src), bit_shift
984                    ; srai X(dst), X(dst), bit_shift
985                );
986            }
987            (false, Location::GPR(src), Location::GPR(dst)) => {
988                dynasm!(self
989                    ; slli X(dst), X(src), bit_shift
990                    ; srli X(dst), X(dst), bit_shift
991                );
992            }
993            _ => codegen_error!("singlepass can't emit SEXT {:?} {:?} {:?}", sz, src, dst),
994        };
995        Ok(())
996    }
997
998    fn emit_not(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
999        match (sz, src, dst) {
1000            (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1001                dynasm!(self ; not X(dst), X(src));
1002            }
1003            _ => codegen_error!("singlepass can't emit NOT {:?} {:?} {:?}", sz, src, dst),
1004        }
1005
1006        Ok(())
1007    }
1008
1009    fn emit_neg(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1010        match (sz, src, dst) {
1011            (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1012                dynasm!(self ; neg X(dst), X(src));
1013            }
1014            _ => codegen_error!("singlepass can't emit NEG {:?} {:?} {:?}", sz, src, dst),
1015        }
1016
1017        Ok(())
1018    }
1019
1020    fn emit_mov(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1021        match (sz, src, dst) {
1022            (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1023                dynasm!(self ; mv X(dst), X(src));
1024            }
1025            (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1026                dynasm!(self
1027                    ; slli X(dst), X(src), 32
1028                    ; srli X(dst), X(dst), 32
1029                );
1030            }
1031            (Size::S64, Location::GPR(src), Location::SIMD(dst)) => {
1032                dynasm!(self ; fmv.d.x F(dst), X(src));
1033            }
1034            (Size::S64, Location::SIMD(src), Location::GPR(dst)) => {
1035                dynasm!(self ; fmv.x.d X(dst), F(src));
1036            }
1037            (Size::S32, Location::GPR(src), Location::SIMD(dst)) => {
1038                dynasm!(self ; fmv.s.x F(dst), X(src));
1039            }
1040            (Size::S32, Location::SIMD(src), Location::GPR(dst)) => {
1041                dynasm!(self ; fmv.x.s X(dst), F(src));
1042            }
1043            (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
1044                dynasm!(self ; fmv.d F(dst), F(src));
1045            }
1046            (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
1047                dynasm!(self ; fmv.s F(dst), F(src));
1048            }
1049            _ => codegen_error!("singlepass can't emit MOV {:?} {:?} {:?}", sz, src, dst),
1050        }
1051
1052        Ok(())
1053    }
1054
1055    fn emit_ret(&mut self) -> Result<(), CompileError> {
1056        dynasm!(self ; ret);
1057        Ok(())
1058    }
1059
1060    fn emit_mov_imm(&mut self, dst: Location, val: i64) -> Result<(), CompileError> {
1061        // The number of used bits by the number including the sign bit.
1062        // i64::MIN.abs() will overflow, thus handle it specially.
1063        let used_bits = if val == i64::MIN {
1064            i64::BITS
1065        } else {
1066            i64::BITS - val.abs().leading_zeros() + 1
1067        };
1068
1069        match dst {
1070            Location::GPR(dst) => match used_bits {
1071                0..=12 => dynasm!(self ; li.12 X(dst), val as _),
1072                13..=32 => dynasm!(self ; li.32 X(dst), val as _),
1073                33..=43 => dynasm!(self ; li.43 X(dst), val),
1074                44..=54 => dynasm!(self ; li.54 X(dst), val),
1075                55..=64 => dynasm!(self ; li X(dst), val),
1076                _ => unreachable!(),
1077            },
1078            _ => codegen_error!("singlepass can't emit MOVIMM {:?}", dst),
1079        }
1080        Ok(())
1081    }
1082
1083    fn emit_fneg(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1084        match (sz, src, dst) {
1085            (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
1086                dynasm!(self ; fneg.d F(dst), F(src));
1087            }
1088            (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
1089                dynasm!(self ; fneg.s F(dst), F(src));
1090            }
1091            _ => codegen_error!("singlepass can't emit FNEG {:?} {:?} {:?}", sz, src, dst),
1092        }
1093
1094        Ok(())
1095    }
1096
1097    fn emit_fmin(
1098        &mut self,
1099        sz: Size,
1100        src1: Location,
1101        src2: Location,
1102        dst: Location,
1103    ) -> Result<(), CompileError> {
1104        match (sz, src1, src2, dst) {
1105            (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
1106                dynasm!(self ; fmin.s F(dst), F(src1), F(src2));
1107            }
1108            (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
1109                dynasm!(self ; fmin.d F(dst), F(src1), F(src2));
1110            }
1111            _ => codegen_error!(
1112                "singlepass can't emit FMIN {:?} {:?} {:?} {:?}",
1113                sz,
1114                src1,
1115                src2,
1116                dst
1117            ),
1118        }
1119        Ok(())
1120    }
1121
1122    fn emit_fmax(
1123        &mut self,
1124        sz: Size,
1125        src1: Location,
1126        src2: Location,
1127        dst: Location,
1128    ) -> Result<(), CompileError> {
1129        match (sz, src1, src2, dst) {
1130            (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
1131                dynasm!(self ; fmax.s F(dst), F(src1), F(src2));
1132            }
1133            (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
1134                dynasm!(self ; fmax.d F(dst), F(src1), F(src2));
1135            }
1136            _ => codegen_error!(
1137                "singlepass can't emit FMAX {:?} {:?} {:?} {:?}",
1138                sz,
1139                src1,
1140                src2,
1141                dst
1142            ),
1143        }
1144        Ok(())
1145    }
1146
1147    fn emit_fdiv(
1148        &mut self,
1149        sz: Size,
1150        src1: Location,
1151        src2: Location,
1152        dst: Location,
1153    ) -> Result<(), CompileError> {
1154        match (sz, src1, src2, dst) {
1155            (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
1156                dynasm!(self ; fdiv.s F(dst), F(src1), F(src2));
1157            }
1158            (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
1159                dynasm!(self ; fdiv.d F(dst), F(src1), F(src2));
1160            }
1161            _ => codegen_error!(
1162                "singlepass can't emit FDIV {:?} {:?} {:?} {:?}",
1163                sz,
1164                src1,
1165                src2,
1166                dst
1167            ),
1168        }
1169        Ok(())
1170    }
1171
1172    fn emit_fsqrt(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1173        match (sz, src, dst) {
1174            (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
1175                dynasm!(self ; fsqrt.s F(dst), F(src));
1176            }
1177            (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
1178                dynasm!(self ; fsqrt.d F(dst), F(src));
1179            }
1180            _ => codegen_error!("singlepass can't emit FSQRT {:?} {:?} {:?}", sz, src, dst),
1181        }
1182        Ok(())
1183    }
1184
1185    fn emit_fcvt(
1186        &mut self,
1187        signed: bool,
1188        sz_in: Size,
1189        src: Location,
1190        sz_out: Size,
1191        dst: Location,
1192    ) -> Result<(), CompileError> {
1193        match (signed, sz_in, src, sz_out, dst) {
1194            // floating-point -> floating-point types
1195            (_, Size::S32, Location::SIMD(src), Size::S64, Location::SIMD(dst)) => {
1196                dynasm!(self ; fcvt.d.s F(dst), F(src));
1197            }
1198            (_, Size::S64, Location::SIMD(src), Size::S32, Location::SIMD(dst)) => {
1199                dynasm!(self ; fcvt.s.d F(dst), F(src));
1200            }
1201            // floating-point -> int types
1202            (true, Size::S32, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
1203                dynasm!(self ; fcvt.w.s X(dst), F(src), rtz);
1204            }
1205            (true, Size::S64, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
1206                dynasm!(self ; fcvt.w.d X(dst), F(src), rtz);
1207            }
1208            (true, Size::S32, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
1209                dynasm!(self ; fcvt.l.s X(dst), F(src), rtz);
1210            }
1211            (true, Size::S64, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
1212                dynasm!(self ; fcvt.l.d X(dst), F(src), rtz);
1213            }
1214            (false, Size::S32, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
1215                dynasm!(self ; fcvt.wu.s X(dst), F(src), rtz);
1216            }
1217            (false, Size::S64, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
1218                dynasm!(self ; fcvt.wu.d X(dst), F(src), rtz);
1219            }
1220            (false, Size::S32, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
1221                dynasm!(self ; fcvt.lu.s X(dst), F(src), rtz);
1222            }
1223            (false, Size::S64, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
1224                dynasm!(self ; fcvt.lu.d X(dst), F(src), rtz);
1225            }
1226            // int -> floating-point types
1227            (true, Size::S32, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
1228                dynasm!(self ; fcvt.s.w F(dst), X(src));
1229            }
1230            (true, Size::S64, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
1231                dynasm!(self ; fcvt.s.l F(dst), X(src));
1232            }
1233            (true, Size::S32, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
1234                dynasm!(self ; fcvt.d.w F(dst), X(src));
1235            }
1236            (true, Size::S64, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
1237                dynasm!(self ; fcvt.d.l F(dst), X(src));
1238            }
1239
1240            (false, Size::S32, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
1241                dynasm!(self ; fcvt.s.wu F(dst), X(src));
1242            }
1243            (false, Size::S64, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
1244                dynasm!(self ; fcvt.s.lu F(dst), X(src));
1245            }
1246            (false, Size::S32, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
1247                dynasm!(self ; fcvt.d.wu F(dst), X(src));
1248            }
1249            (false, Size::S64, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
1250                dynasm!(self ; fcvt.d.lu F(dst), X(src));
1251            }
1252            _ => codegen_error!(
1253                "singlepass can't emit FCVT {:?} {:?} {:?} {:?}",
1254                sz_in,
1255                src,
1256                sz_out,
1257                dst
1258            ),
1259        }
1260        Ok(())
1261    }
1262
1263    fn emit_fcvt_with_rounding(
1264        &mut self,
1265        rounding: RoundingMode,
1266        size: Size,
1267        src: Location,
1268        dst: Location,
1269        tmp: GPR,
1270    ) -> Result<(), CompileError> {
1271        let (Location::SIMD(src), Location::SIMD(dst)) = (src, dst) else {
1272            codegen_error!(
1273                "singlepass can't emit FCVT with rounding for non-register operands: {:?} {:?} {:?} {:?}",
1274                src,
1275                dst,
1276                size,
1277                rounding
1278            )
1279        };
1280
1281        match (size, rounding) {
1282            (Size::S32, RoundingMode::Rne) => {
1283                dynasm!(self
1284                    ; fcvt.w.s X(tmp), F(src), rne
1285                    ; fcvt.s.w F(dst), X(tmp), rne
1286                    ; fsgnj.s F(dst), F(dst), F(src));
1287            }
1288            (Size::S32, RoundingMode::Rtz) => {
1289                dynasm!(self
1290                    ; fcvt.w.s X(tmp), F(src), rtz
1291                    ; fcvt.s.w F(dst), X(tmp), rtz
1292                    ; fsgnj.s F(dst), F(dst), F(src));
1293            }
1294            (Size::S32, RoundingMode::Rdn) => {
1295                dynasm!(self
1296                    ; fcvt.w.s X(tmp), F(src), rdn
1297                    ; fcvt.s.w F(dst), X(tmp), rdn
1298                    ; fsgnj.s F(dst), F(dst), F(src));
1299            }
1300            (Size::S32, RoundingMode::Rup) => {
1301                dynasm!(self
1302                    ; fcvt.w.s X(tmp), F(src), rup
1303                    ; fcvt.s.w F(dst), X(tmp), rup
1304                    ; fsgnj.s F(dst), F(dst), F(src));
1305            }
1306
1307            (Size::S64, RoundingMode::Rne) => {
1308                dynasm!(self
1309                    ; fcvt.l.d X(tmp), F(src), rne
1310                    ; fcvt.d.l F(dst), X(tmp), rne
1311                    ; fsgnj.d F(dst), F(dst), F(src));
1312            }
1313            (Size::S64, RoundingMode::Rtz) => {
1314                dynasm!(self
1315                    ; fcvt.l.d X(tmp), F(src), rtz
1316                    ; fcvt.d.l F(dst), X(tmp), rtz
1317                    ; fsgnj.d F(dst), F(dst), F(src));
1318            }
1319            (Size::S64, RoundingMode::Rdn) => {
1320                dynasm!(self
1321                    ; fcvt.l.d X(tmp), F(src), rdn
1322                    ; fcvt.d.l F(dst), X(tmp), rdn
1323                    ; fsgnj.d F(dst), F(dst), F(src));
1324            }
1325            (Size::S64, RoundingMode::Rup) => {
1326                dynasm!(self
1327                    ; fcvt.l.d X(tmp), F(src), rup
1328                    ; fcvt.d.l F(dst), X(tmp), rup
1329                    ; fsgnj.d F(dst), F(dst), F(src));
1330            }
1331            _ => codegen_error!(
1332                "singlepass can't emit FCVT with rounding {:?} {:?}",
1333                size,
1334                rounding
1335            ),
1336        }
1337
1338        Ok(())
1339    }
1340
1341    // Swap `source` register with FSCR.
1342    fn emit_swap_fscr(&mut self, reg: GPR) -> Result<(), CompileError> {
1343        dynasm!(self ; fscsr X(reg), X(reg));
1344        Ok(())
1345    }
1346
1347    fn emit_cmp(
1348        &mut self,
1349        c: Condition,
1350        loc_a: Location,
1351        loc_b: Location,
1352        ret: Location,
1353    ) -> Result<(), CompileError> {
1354        let (Location::GPR(loc_a), Location::GPR(loc_b), Location::GPR(ret)) = (loc_a, loc_b, ret)
1355        else {
1356            codegen_error!(
1357                "singlepass can't emit CMP {:?} {:?} {:?}",
1358                loc_a,
1359                loc_b,
1360                ret
1361            );
1362        };
1363
1364        match c {
1365            // signed comparison operations
1366            Condition::Lt => {
1367                dynasm!(self; slt X(ret), X(loc_a), X(loc_b));
1368            }
1369            Condition::Le => {
1370                dynasm!(self
1371                    ; slt X(ret), X(loc_b), X(loc_a)
1372                    ; xori X(ret), X(ret), 1);
1373            }
1374            Condition::Gt => {
1375                dynasm!(self; slt X(ret), X(loc_b), X(loc_a));
1376            }
1377            Condition::Ge => {
1378                dynasm!(self
1379                    ; slt X(ret), X(loc_a), X(loc_b)
1380                    ; xori X(ret), X(ret), 1);
1381            }
1382            Condition::Eq => {
1383                dynasm!(self
1384                    ; xor X(ret), X(loc_a), X(loc_b)
1385                    ; seqz X(ret), X(ret));
1386            }
1387            Condition::Ne => {
1388                dynasm!(self
1389                    ; xor X(ret), X(loc_a), X(loc_b)
1390                    ; snez X(ret), X(ret));
1391            }
1392            // unsigned comparison operations
1393            Condition::Ltu => {
1394                dynasm!(self; sltu X(ret), X(loc_a), X(loc_b));
1395            }
1396            Condition::Leu => {
1397                dynasm!(self
1398                    ; sltu X(ret), X(loc_b), X(loc_a)
1399                    ; xori X(ret), X(ret), 1);
1400            }
1401            Condition::Gtu => {
1402                dynasm!(self; sltu X(ret), X(loc_b), X(loc_a));
1403            }
1404            Condition::Geu => {
1405                dynasm!(self
1406                    ; sltu X(ret), X(loc_a), X(loc_b)
1407                    ; xori X(ret), X(ret), 1);
1408            }
1409        }
1410
1411        Ok(())
1412    }
1413
1414    fn emit_fcmp(
1415        &mut self,
1416        c: Condition,
1417        size: Size,
1418        loc_a: Location,
1419        loc_b: Location,
1420        ret: Location,
1421    ) -> Result<(), CompileError> {
1422        let (Location::SIMD(loc_a), Location::SIMD(loc_b), Location::GPR(ret)) =
1423            (loc_a, loc_b, ret)
1424        else {
1425            codegen_error!(
1426                "singlepass can't emit FCMP {:?} {:?} {:?}",
1427                loc_a,
1428                loc_b,
1429                ret
1430            );
1431        };
1432
1433        match (size, c) {
1434            (Size::S32, Condition::Lt) => {
1435                dynasm!(self ; flt.s X(ret), F(loc_a), F(loc_b));
1436            }
1437            (Size::S32, Condition::Le) => {
1438                dynasm!(self ; fle.s X(ret), F(loc_a), F(loc_b));
1439            }
1440            (Size::S32, Condition::Gt) => {
1441                dynasm!(self ; flt.s X(ret), F(loc_b), F(loc_a));
1442            }
1443            (Size::S32, Condition::Ge) => {
1444                dynasm!(self ; fle.s X(ret), F(loc_b), F(loc_a));
1445            }
1446            (Size::S32, Condition::Eq) => {
1447                dynasm!(self ; feq.s X(ret), F(loc_a), F(loc_b));
1448            }
1449            (Size::S32, Condition::Ne) => {
1450                dynasm!(self
1451                    ; feq.s X(ret), F(loc_a), F(loc_b)
1452                    ; xori X(ret), X(ret), 1);
1453            }
1454
1455            (Size::S64, Condition::Lt) => {
1456                dynasm!(self ; flt.d X(ret), F(loc_a), F(loc_b));
1457            }
1458            (Size::S64, Condition::Le) => {
1459                dynasm!(self ; fle.d X(ret), F(loc_a), F(loc_b));
1460            }
1461            (Size::S64, Condition::Gt) => {
1462                dynasm!(self ; flt.d X(ret), F(loc_b), F(loc_a));
1463            }
1464            (Size::S64, Condition::Ge) => {
1465                dynasm!(self; fle.d X(ret), F(loc_b), F(loc_a));
1466            }
1467            (Size::S64, Condition::Eq) => {
1468                dynasm!(self ; feq.d X(ret), F(loc_a), F(loc_b));
1469            }
1470            (Size::S64, Condition::Ne) => {
1471                dynasm!(self
1472                    ; feq.d X(ret), F(loc_a), F(loc_b)
1473                    ; xori X(ret), X(ret), 1);
1474            }
1475            _ => codegen_error!(
1476                "singlepass can't emit FCMP {:?} {:?} {:?}",
1477                loc_a,
1478                loc_b,
1479                ret
1480            ),
1481        }
1482
1483        Ok(())
1484    }
1485
1486    fn emit_jmp_on_condition(
1487        &mut self,
1488        cond: Condition,
1489        loc_a: Location,
1490        loc_b: Location,
1491        label: Label,
1492        tmp: GPR,
1493    ) -> Result<(), CompileError> {
1494        let (Location::GPR(loc_a), Location::GPR(loc_b)) = (loc_a, loc_b) else {
1495            codegen_error!("singlepass can't emit JMP_ON_COND {:?} {:?} ", loc_a, loc_b,);
1496        };
1497
1498        let jump: Label = self.get_label();
1499        let cont: Label = self.get_label();
1500
1501        match cond {
1502            Condition::Eq => {
1503                dynasm!(self; beq X(loc_a), X(loc_b), => jump);
1504            }
1505            Condition::Ne => {
1506                dynasm!(self; bne X(loc_a), X(loc_b), => jump);
1507            }
1508            Condition::Ltu => {
1509                dynasm!(self; bltu X(loc_a), X(loc_b), => jump);
1510            }
1511            Condition::Leu => {
1512                dynasm!(self; bleu X(loc_a), X(loc_b), => jump);
1513            }
1514            Condition::Gtu => {
1515                dynasm!(self; bgtu X(loc_a), X(loc_b), => jump);
1516            }
1517            Condition::Geu => {
1518                dynasm!(self; bgeu X(loc_a), X(loc_b), => jump);
1519            }
1520            _ => codegen_error!(
1521                "singlepass can't emit jump on conditional branch {:?}",
1522                cond
1523            ),
1524        }
1525
1526        self.emit_j_label(cont, Some(tmp))?;
1527        self.emit_label(jump)?;
1528        self.emit_j_label(label, Some(tmp))?;
1529
1530        self.emit_label(cont)?;
1531
1532        Ok(())
1533    }
1534
1535    fn emit_on_false_label(
1536        &mut self,
1537        cond: Location,
1538        label: Label,
1539        tmp: GPR,
1540    ) -> Result<(), CompileError> {
1541        match cond {
1542            Location::GPR(cond) => {
1543                dynasm!(self; beqz X(cond), => label);
1544            }
1545            _ if cond.is_imm() => {
1546                let imm = cond.imm_value_scalar().unwrap();
1547                if imm == 0 {
1548                    return self.emit_j_label(label, Some(tmp));
1549                }
1550            }
1551            _ => codegen_error!("singlepass can't emit jump to false branch {:?}", cond),
1552        }
1553        Ok(())
1554    }
1555    fn emit_on_false_label_far(
1556        &mut self,
1557        cond: Location,
1558        label: Label,
1559        tmp: GPR,
1560    ) -> Result<(), CompileError> {
1561        let cont: Label = self.get_label();
1562        match cond {
1563            Location::GPR(cond) => {
1564                // Use the negative condition to jump after the "j" instruction that will
1565                // go to the requested `label`.
1566                dynasm!(self; bnez X(cond), => cont);
1567            }
1568            _ if cond.is_imm() => {
1569                let imm = cond.imm_value_scalar().unwrap();
1570                if imm == 0 {
1571                    return self.emit_j_label(label, Some(tmp));
1572                } else {
1573                    self.emit_j_label(cont, Some(tmp))?;
1574                }
1575            }
1576            _ => codegen_error!("singlepass can't emit jump to false branch {:?}", cond),
1577        }
1578
1579        self.emit_j_label(label, Some(tmp))?;
1580        self.emit_label(cont)?;
1581        Ok(())
1582    }
1583    fn emit_on_true_label(
1584        &mut self,
1585        cond: Location,
1586        label: Label,
1587        tmp: GPR,
1588    ) -> Result<(), CompileError> {
1589        match cond {
1590            Location::GPR(cond) => {
1591                dynasm!(self; bnez X(cond), => label);
1592            }
1593            _ if cond.is_imm() => {
1594                let imm = cond.imm_value_scalar().unwrap();
1595                if imm != 0 {
1596                    return self.emit_j_label(label, Some(tmp));
1597                }
1598            }
1599            _ => codegen_error!("singlepass can't emit jump to true branch {:?}", cond),
1600        }
1601        Ok(())
1602    }
1603    fn emit_on_true_label_far(
1604        &mut self,
1605        cond: Location,
1606        label: Label,
1607        tmp: GPR,
1608    ) -> Result<(), CompileError> {
1609        let cont: Label = self.get_label();
1610        let jump_label: Label = self.get_label();
1611        match cond {
1612            Location::GPR(cond) => {
1613                dynasm!(self; bnez X(cond), => jump_label);
1614            }
1615            _ if cond.is_imm() => {
1616                let imm = cond.imm_value_scalar().unwrap();
1617                if imm == 1 {
1618                    return self.emit_j_label(jump_label, Some(tmp));
1619                }
1620            }
1621            _ => codegen_error!("singlepass can't emit jump to false branch {:?}", cond),
1622        }
1623
1624        // skip over the jump to target
1625        self.emit_j_label(cont, Some(tmp))?;
1626
1627        self.emit_label(jump_label)?;
1628        self.emit_j_label(label, Some(tmp))?;
1629
1630        self.emit_label(cont)?;
1631
1632        Ok(())
1633    }
1634
1635    fn emit_j_label(&mut self, label: Label, reg: Option<GPR>) -> Result<(), CompileError> {
1636        if let Some(reg) = reg {
1637            dynasm!(self ; jump => label, X(reg));
1638        } else {
1639            dynasm!(self ; j => label);
1640        }
1641        Ok(())
1642    }
1643
1644    fn emit_j_register(&mut self, reg: GPR) -> Result<(), CompileError> {
1645        dynasm!(self ; jalr zero, X(reg), 0);
1646        Ok(())
1647    }
1648
1649    fn emit_call_label(&mut self, label: Label) -> Result<(), CompileError> {
1650        dynasm!(self ; call =>label);
1651        Ok(())
1652    }
1653
1654    fn emit_call_register(&mut self, reg: GPR) -> Result<(), CompileError> {
1655        dynasm!(self ; jalr ra, X(reg), 0);
1656        Ok(())
1657    }
1658
1659    fn emit_load_label(&mut self, reg: GPR, label: Label) -> Result<(), CompileError> {
1660        dynasm!(self ; la X(reg), => label);
1661        Ok(())
1662    }
1663
1664    fn emit_rwfence(&mut self) -> Result<(), CompileError> {
1665        dynasm!(self ; fence rw, rw);
1666        Ok(())
1667    }
1668
1669    fn emit_atomic_binop(
1670        &mut self,
1671        op: AtomicBinaryOp,
1672        size: Size,
1673        dest: GPR,
1674        addr: GPR,
1675        source: GPR,
1676    ) -> Result<(), CompileError> {
1677        match (size, op) {
1678            (Size::S64, AtomicBinaryOp::Add) => {
1679                dynasm!(self ; amoadd.d.aqrl X(dest), X(source), [X(addr)])
1680            }
1681            (Size::S64, AtomicBinaryOp::Or) => {
1682                dynasm!(self ; amoor.d.aqrl X(dest), X(source), [X(addr)])
1683            }
1684            (Size::S64, AtomicBinaryOp::Xor) => {
1685                dynasm!(self ; amoxor.d.aqrl X(dest), X(source), [X(addr)])
1686            }
1687            (Size::S64, AtomicBinaryOp::And) => {
1688                dynasm!(self ; amoand.d.aqrl X(dest), X(source), [X(addr)])
1689            }
1690            (Size::S64, AtomicBinaryOp::Exchange) => {
1691                dynasm!(self ; amoswap.d.aqrl X(dest), X(source), [X(addr)])
1692            }
1693
1694            (Size::S32, AtomicBinaryOp::Add) => {
1695                dynasm!(self ; amoadd.w.aqrl X(dest), X(source), [X(addr)])
1696            }
1697            (Size::S32, AtomicBinaryOp::Or) => {
1698                dynasm!(self ; amoor.w.aqrl X(dest), X(source), [X(addr)])
1699            }
1700            (Size::S32, AtomicBinaryOp::Xor) => {
1701                dynasm!(self ; amoxor.w.aqrl X(dest), X(source), [X(addr)])
1702            }
1703            (Size::S32, AtomicBinaryOp::And) => {
1704                dynasm!(self ; amoand.w.aqrl X(dest), X(source), [X(addr)])
1705            }
1706            (Size::S32, AtomicBinaryOp::Exchange) => {
1707                dynasm!(self ; amoswap.w.aqrl X(dest), X(source), [X(addr)])
1708            }
1709            _ => codegen_error!("singlepass can't emit atomic binop {:?} {:?}", size, op),
1710        }
1711
1712        Ok(())
1713    }
1714
1715    fn emit_reserved_ld(&mut self, size: Size, reg: GPR, addr: GPR) -> Result<(), CompileError> {
1716        match size {
1717            Size::S32 => {
1718                dynasm!(self ; lr.w.aqrl X(reg), [X(addr)])
1719            }
1720            Size::S64 => {
1721                dynasm!(self ; lr.d.aqrl X(reg), [X(addr)])
1722            }
1723            _ => codegen_error!("singlepass can't emit atomic reserved ld {:?}", size),
1724        }
1725
1726        Ok(())
1727    }
1728
1729    fn emit_reserved_sd(
1730        &mut self,
1731        size: Size,
1732        dest: GPR,
1733        addr: GPR,
1734        source: GPR,
1735    ) -> Result<(), CompileError> {
1736        match size {
1737            Size::S32 => {
1738                dynasm!(self ; sc.w.rl X(dest), X(source), [X(addr)])
1739            }
1740            Size::S64 => {
1741                dynasm!(self ; sc.d.rl X(dest), X(source), [X(addr)])
1742            }
1743            _ => codegen_error!("singlepass can't emit atomic reserved sd {:?}", size),
1744        }
1745
1746        Ok(())
1747    }
1748}
1749
1750pub fn gen_std_trampoline_riscv(
1751    sig: &FunctionType,
1752    _calling_convention: CallingConvention,
1753    progress_callback: Option<&CompilationProgressCallback>,
1754) -> Result<FunctionBody, CompileError> {
1755    let mut a = Assembler::new(0);
1756    let mut output_reporter = ChunkedOutputReporter::new(progress_callback);
1757
1758    // Callee-save registers must be used.
1759    let fptr = GPR::X26;
1760    let args = GPR::X27;
1761
1762    dynasm!(a
1763        ; addi sp, sp, -32
1764        ; sd s0, [sp,24]
1765        ; sd ra, [sp,16]
1766        ; sd X(fptr), [sp, 8]
1767        ; sd X(args), [sp, 0]
1768        ; mv s0, sp // use frame-pointer register for later restore
1769        ; mv X(fptr), a1
1770        ; mv X(args), a2
1771    );
1772
1773    let stack_args = sig.params().len().saturating_sub(7); //1st arg is ctx, not an actual arg
1774    let stack_return_slots = sig
1775        .results()
1776        .len()
1777        .saturating_sub(RISCV_RETURN_VALUE_REGISTERS.len());
1778    let mut stack_offset = (stack_args + stack_return_slots) as u32 * 8;
1779    if stack_offset > 0 {
1780        stack_offset = stack_offset.next_multiple_of(16);
1781        if ImmType::Bits12Subtraction.compatible_imm(stack_offset as _) {
1782            dynasm!(a ; addi sp, sp, -(stack_offset as i32));
1783        } else {
1784            a.emit_mov_imm(Location::GPR(SCRATCH_REG), stack_offset as _)?;
1785            dynasm!(a ; sub sp, sp, X(SCRATCH_REG));
1786        }
1787    }
1788
1789    // Move arguments to their locations.
1790    // `callee_vmctx` is already in the first argument register, so no need to move.
1791    let mut caller_stack_offset: i32 = stack_return_slots as i32 * 8;
1792    for (i, param) in sig.params().iter().enumerate() {
1793        let sz = match *param {
1794            Type::I32 | Type::F32 => Size::S32,
1795            Type::I64 | Type::F64 => Size::S64,
1796            Type::ExternRef => Size::S64,
1797            Type::FuncRef => Size::S64,
1798            _ => codegen_error!(
1799                "singlepass unsupported param type for trampoline {:?}",
1800                *param
1801            ),
1802        };
1803        match i {
1804            0..=6 => {
1805                a.emit_ld(
1806                    sz,
1807                    false,
1808                    Location::GPR(GPR::from_index(i + 10 + 1).unwrap()),
1809                    Location::Memory(args, (i * 16) as i32),
1810                )?;
1811            }
1812            _ => {
1813                let args_offset = (i * 16) as i32;
1814                let arg_location = if ImmType::Bits12.compatible_imm(args_offset as i64) {
1815                    Location::Memory(args, args_offset)
1816                } else {
1817                    a.emit_mov_imm(Location::GPR(SCRATCH_REG), args_offset as i64)?;
1818                    a.emit_add(
1819                        Size::S64,
1820                        Location::GPR(args),
1821                        Location::GPR(SCRATCH_REG),
1822                        Location::GPR(SCRATCH_REG),
1823                    )?;
1824                    Location::Memory(SCRATCH_REG, 0)
1825                };
1826                a.emit_ld(sz, false, Location::GPR(SCRATCH_REG), arg_location)?;
1827
1828                let arg_dest_location =
1829                    if ImmType::Bits12.compatible_imm(caller_stack_offset as i64) {
1830                        Location::Memory(GPR::Sp, caller_stack_offset)
1831                    } else {
1832                        a.emit_mov_imm(Location::GPR(GPR::X29), caller_stack_offset as i64)?;
1833                        a.emit_add(
1834                            Size::S64,
1835                            Location::GPR(GPR::X29),
1836                            Location::GPR(GPR::Sp),
1837                            Location::GPR(GPR::X29),
1838                        )?;
1839                        Location::Memory(GPR::X29, 0)
1840                    };
1841                a.emit_sd(sz, Location::GPR(SCRATCH_REG), arg_dest_location)?;
1842                caller_stack_offset += 8;
1843            }
1844        }
1845        output_reporter.check(a.offset().0)?;
1846    }
1847
1848    dynasm!(a
1849        ; jalr ra, X(fptr), 0);
1850
1851    // Write return values.
1852    let mut n_stack_return_slots: usize = 0;
1853    for i in 0..sig.results().len() {
1854        let src = if let Some(&reg) = RISCV_RETURN_VALUE_REGISTERS.get(i) {
1855            reg
1856        } else {
1857            a.emit_ld(
1858                Size::S64,
1859                false,
1860                Location::GPR(SCRATCH_REG),
1861                Location::Memory(GPR::Sp, (n_stack_return_slots as u32 * 8) as _),
1862            )?;
1863            n_stack_return_slots += 1;
1864            SCRATCH_REG
1865        };
1866        a.emit_sd(
1867            Size::S64,
1868            Location::GPR(src),
1869            Location::Memory(args, (i * 16) as _),
1870        )?;
1871        output_reporter.check(a.offset().0)?;
1872    }
1873
1874    // Restore stack.
1875    dynasm!(a
1876        ; ld X(args), [s0,0]
1877        ; ld X(fptr), [s0,8]
1878        ; ld ra, [s0,16]
1879        ; ld s0, [s0,24]
1880    );
1881    let restored_stack_offset = 32 + stack_offset;
1882    if ImmType::Bits12.compatible_imm(restored_stack_offset as _) {
1883        dynasm!(a; addi sp, sp, restored_stack_offset as _);
1884    } else {
1885        a.emit_mov_imm(Location::GPR(SCRATCH_REG), restored_stack_offset as _)?;
1886        dynasm!(a; add sp, sp, X(SCRATCH_REG));
1887    }
1888    dynasm!(a; ret);
1889
1890    let mut body = a.finalize().unwrap();
1891
1892    body.shrink_to_fit();
1893    output_reporter.finish(body.len())?;
1894    Ok(FunctionBody {
1895        body,
1896        unwind_info: None,
1897    })
1898}
1899
1900/// Generates dynamic import function call trampoline for a function type.
1901pub fn gen_std_dynamic_import_trampoline_riscv(
1902    vmoffsets: &VMOffsets,
1903    sig: &FunctionType,
1904    progress_callback: Option<&CompilationProgressCallback>,
1905) -> Result<FunctionBody, CompileError> {
1906    let mut a = Assembler::new(0);
1907    let mut output_reporter = ChunkedOutputReporter::new(progress_callback);
1908    // Allocate argument array.
1909    let stack_offset: usize = 16 * std::cmp::max(sig.params().len(), sig.results().len());
1910
1911    // Save RA and the scratch register
1912    a.emit_push(Size::S64, Location::GPR(GPR::X1))?;
1913    a.emit_push(Size::S64, Location::GPR(SCRATCH_REG))?;
1914
1915    if stack_offset != 0 {
1916        if ImmType::Bits12Subtraction.compatible_imm(stack_offset as _) {
1917            a.emit_sub(
1918                Size::S64,
1919                Location::GPR(GPR::Sp),
1920                Location::Imm64(stack_offset as _),
1921                Location::GPR(GPR::Sp),
1922            )?;
1923        } else {
1924            a.emit_mov_imm(Location::GPR(SCRATCH_REG), stack_offset as _)?;
1925            a.emit_sub(
1926                Size::S64,
1927                Location::GPR(GPR::Sp),
1928                Location::GPR(SCRATCH_REG),
1929                Location::GPR(GPR::Sp),
1930            )?;
1931        }
1932    }
1933
1934    // Copy arguments.
1935    if !sig.params().is_empty() {
1936        let mut argalloc = ArgumentRegisterAllocator::default();
1937        argalloc.next(Type::I64).unwrap(); // skip VMContext
1938
1939        let mut stack_param_count: usize = 0;
1940
1941        for (i, ty) in sig.params().iter().enumerate() {
1942            let source_loc = match argalloc.next(*ty)? {
1943                Some(RiscvRegister::GPR(gpr)) => Location::GPR(gpr),
1944                Some(RiscvRegister::FPR(fpr)) => Location::SIMD(fpr),
1945                None => {
1946                    a.emit_ld(
1947                        Size::S64,
1948                        false,
1949                        Location::GPR(SCRATCH_REG),
1950                        Location::Memory(GPR::Sp, (stack_offset + 16 + stack_param_count) as _),
1951                    )?;
1952                    stack_param_count += 8;
1953                    Location::GPR(SCRATCH_REG)
1954                }
1955            };
1956            a.emit_sd(
1957                Size::S64,
1958                source_loc,
1959                Location::Memory(GPR::Sp, (i * 16) as _),
1960            )?;
1961            // Zero upper 64 bits.
1962            a.emit_sd(
1963                Size::S64,
1964                Location::GPR(GPR::XZero),
1965                Location::Memory(GPR::Sp, (i * 16 + 8) as _),
1966            )?;
1967            output_reporter.check(a.offset().0)?;
1968        }
1969    }
1970
1971    // Load target address.
1972    let offset = vmoffsets.vmdynamicfunction_import_context_address();
1973    a.emit_ld(
1974        Size::S64,
1975        false,
1976        Location::GPR(SCRATCH_REG),
1977        Location::Memory(GPR::X10, offset as i32),
1978    )?;
1979    // Load values array.
1980    a.emit_add(
1981        Size::S64,
1982        Location::GPR(GPR::Sp),
1983        Location::Imm64(0),
1984        Location::GPR(GPR::X11),
1985    )?;
1986
1987    // Call target.
1988    a.emit_call_register(SCRATCH_REG)?;
1989
1990    // Fetch return value.
1991    if !sig.results().is_empty() {
1992        assert_eq!(sig.results().len(), 1);
1993        a.emit_ld(
1994            Size::S64,
1995            false,
1996            Location::GPR(GPR::X10),
1997            Location::Memory(GPR::Sp, 0),
1998        )?;
1999    }
2000
2001    // Release values array.
2002    if stack_offset != 0 {
2003        if ImmType::Bits12.compatible_imm(stack_offset as _) {
2004            a.emit_add(
2005                Size::S64,
2006                Location::GPR(GPR::Sp),
2007                Location::Imm64(stack_offset as _),
2008                Location::GPR(GPR::Sp),
2009            )?;
2010        } else {
2011            a.emit_mov_imm(Location::GPR(SCRATCH_REG), stack_offset as _)?;
2012            a.emit_add(
2013                Size::S64,
2014                Location::GPR(GPR::Sp),
2015                Location::GPR(SCRATCH_REG),
2016                Location::GPR(GPR::Sp),
2017            )?;
2018        }
2019    }
2020    a.emit_pop(Size::S64, Location::GPR(SCRATCH_REG))?;
2021    a.emit_pop(Size::S64, Location::GPR(GPR::X1))?;
2022
2023    // Return.
2024    a.emit_ret()?;
2025
2026    let mut body = a.finalize().unwrap();
2027    body.shrink_to_fit();
2028    output_reporter.finish(body.len())?;
2029    Ok(FunctionBody {
2030        body,
2031        unwind_info: None,
2032    })
2033}
2034
2035// Singlepass calls import functions through a trampoline.
2036pub fn gen_import_call_trampoline_riscv(
2037    vmoffsets: &VMOffsets,
2038    index: FunctionIndex,
2039    sig: &FunctionType,
2040    _calling_convention: CallingConvention,
2041    progress_callback: Option<&CompilationProgressCallback>,
2042) -> Result<CustomSection, CompileError> {
2043    let mut a = Assembler::new(0);
2044    let mut output_reporter = ChunkedOutputReporter::new(progress_callback);
2045
2046    // Singlepass internally treats all arguments as integers
2047    // For the standard System V calling convention requires
2048    //  floating point arguments to be passed in FPR registers.
2049    //  Translation is expensive, so only do it if needed.
2050    if sig
2051        .params()
2052        .iter()
2053        .any(|&x| x == Type::F32 || x == Type::F64)
2054    {
2055        const PARAM_REGS: &[GPR] = &[
2056            // X10 is skipped intentionally
2057            GPR::X11,
2058            GPR::X12,
2059            GPR::X13,
2060            GPR::X14,
2061            GPR::X15,
2062            GPR::X16,
2063            GPR::X17,
2064        ];
2065        const PARAM_REGS_COUNT: usize = PARAM_REGS.len();
2066
2067        // Allocate stack space for arguments.
2068        let stack_offset: i32 = if sig.params().len() > PARAM_REGS_COUNT {
2069            (PARAM_REGS_COUNT * 8) as i32
2070        } else {
2071            (sig.params().len() as i32) * 8
2072        };
2073        if stack_offset > 0 {
2074            if ImmType::Bits12Subtraction.compatible_imm(stack_offset as _) {
2075                a.emit_sub(
2076                    Size::S64,
2077                    Location::GPR(GPR::Sp),
2078                    Location::Imm64(stack_offset as _),
2079                    Location::GPR(GPR::Sp),
2080                )?;
2081            } else {
2082                a.emit_mov_imm(Location::GPR(SCRATCH_REG), stack_offset as _)?;
2083                a.emit_sub(
2084                    Size::S64,
2085                    Location::GPR(GPR::Sp),
2086                    Location::GPR(SCRATCH_REG),
2087                    Location::GPR(GPR::Sp),
2088                )?;
2089            }
2090        }
2091
2092        // Store all arguments to the stack to prevent overwrite.
2093
2094        let mut param_locations = vec![];
2095        /* Clippy is wrong about using `i` to index `PARAM_REGS` here. */
2096        #[allow(clippy::needless_range_loop)]
2097        for i in 0..sig.params().len() {
2098            let loc = match i {
2099                0..PARAM_REGS_COUNT => {
2100                    let loc = Location::Memory(GPR::Sp, (i * 8) as i32);
2101                    a.emit_sd(Size::S64, Location::GPR(PARAM_REGS[i]), loc)?;
2102                    loc
2103                }
2104                _ => Location::Memory(GPR::Sp, stack_offset + ((i - PARAM_REGS_COUNT) * 8) as i32),
2105            };
2106            param_locations.push(loc);
2107            output_reporter.check(a.offset().0)?;
2108        }
2109
2110        // Copy arguments.
2111        let mut caller_stack_offset: i32 = 0;
2112        let mut argalloc = ArgumentRegisterAllocator::default();
2113        argalloc.next(Type::I64).unwrap(); // skip VMContext
2114        for (i, ty) in sig.params().iter().enumerate() {
2115            let prev_loc = param_locations[i];
2116            let targ = match argalloc.next(*ty)? {
2117                Some(RiscvRegister::GPR(gpr)) => Location::GPR(gpr),
2118                Some(RiscvRegister::FPR(neon)) => Location::SIMD(neon),
2119                None => {
2120                    // No register can be allocated. Put this argument on the stack.
2121                    a.emit_ld(Size::S64, false, Location::GPR(SCRATCH_REG), prev_loc)?;
2122                    a.emit_sd(
2123                        Size::S64,
2124                        Location::GPR(SCRATCH_REG),
2125                        Location::Memory(GPR::Sp, stack_offset + caller_stack_offset),
2126                    )?;
2127                    caller_stack_offset += 8;
2128                    output_reporter.check(a.offset().0)?;
2129                    continue;
2130                }
2131            };
2132            a.emit_ld(Size::S64, false, targ, prev_loc)?;
2133            output_reporter.check(a.offset().0)?;
2134        }
2135
2136        // Restore stack pointer.
2137        if stack_offset > 0 {
2138            if ImmType::Bits12.compatible_imm(stack_offset as _) {
2139                a.emit_add(
2140                    Size::S64,
2141                    Location::GPR(GPR::Sp),
2142                    Location::Imm64(stack_offset as _),
2143                    Location::GPR(GPR::Sp),
2144                )?;
2145            } else {
2146                a.emit_mov_imm(Location::GPR(SCRATCH_REG), stack_offset as _)?;
2147                a.emit_add(
2148                    Size::S64,
2149                    Location::GPR(GPR::Sp),
2150                    Location::GPR(SCRATCH_REG),
2151                    Location::GPR(GPR::Sp),
2152                )?;
2153            }
2154        }
2155    }
2156
2157    // Emits a tail call trampoline that loads the address of the target import function
2158    // from Ctx and jumps to it.
2159
2160    let offset = vmoffsets.vmctx_vmfunction_import(index);
2161    let ptr_arg_offset = vmctx_offset(offset)?;
2162    let vmctx_arg_offset = vmctx_offset(offset.saturating_add(8))?;
2163
2164    a.emit_ld(
2165        Size::S64,
2166        false,
2167        Location::GPR(SCRATCH_REG),
2168        Location::Memory(GPR::X10, ptr_arg_offset), // function pointer
2169    )?;
2170    a.emit_ld(
2171        Size::S64,
2172        false,
2173        Location::GPR(GPR::X10),
2174        Location::Memory(GPR::X10, vmctx_arg_offset), // target vmctx
2175    )?;
2176
2177    a.emit_j_register(SCRATCH_REG)?;
2178
2179    let mut contents = a.finalize().unwrap();
2180    contents.shrink_to_fit();
2181    output_reporter.finish(contents.len())?;
2182    let section_body = SectionBody::new_with_vec(contents);
2183
2184    Ok(CustomSection {
2185        protection: CustomSectionProtection::ReadExecute,
2186        alignment: None,
2187        bytes: section_body,
2188        relocations: vec![],
2189    })
2190}