wasmer_compiler_singlepass/
machine_riscv.rs

1//! RISC-V machine scaffolding.
2
3use dynasmrt::{DynasmError, VecAssembler, riscv::RiscvRelocation};
4use fixedbitset::FixedBitSet;
5#[cfg(feature = "unwind")]
6use gimli::{RiscV, write::CallFrameInstruction};
7
8use wasmer_compiler::{
9    CANONICAL_NAN_F32, CANONICAL_NAN_F64,
10    types::{
11        address_map::InstructionAddressMap,
12        function::FunctionBody,
13        relocation::{Relocation, RelocationKind, RelocationTarget},
14        section::CustomSection,
15    },
16    wasmparser::MemArg,
17};
18use wasmer_types::{
19    CompilationProgressCallback, CompileError, FunctionIndex, FunctionType, SourceLoc, TrapCode,
20    TrapInformation, VMOffsets,
21    target::{CallingConvention, Target},
22};
23
24use crate::{
25    codegen_error,
26    common_decl::*,
27    emitter_riscv::*,
28    location::{Location as AbstractLocation, Reg},
29    machine::*,
30    riscv_decl::{FPR, GPR},
31    unwind::{UnwindInstructions, UnwindOps, UnwindRegister},
32};
33
34type Assembler = VecAssembler<RiscvRelocation>;
35type Location = AbstractLocation<GPR, FPR>;
36
37use std::{
38    collections::HashMap,
39    ops::{Deref, DerefMut},
40};
41/// The RISC-V assembler wrapper, providing FPU feature tracking and a dynasmrt assembler.
42pub struct AssemblerRiscv {
43    /// Inner dynasm assembler.
44    pub inner: Assembler,
45}
46
47impl AssemblerRiscv {
48    /// Create a new RISC-V assembler.
49    pub fn new(base_addr: usize, _target: Option<Target>) -> Result<Self, CompileError> {
50        // TODO: detect RISC-V FPU extensions (e.g., F/D)
51        Ok(Self {
52            inner: Assembler::new(base_addr),
53        })
54    }
55
56    /// Finalize to machine code bytes.
57    pub fn finalize(self) -> Result<Vec<u8>, DynasmError> {
58        self.inner.finalize()
59    }
60}
61
62impl Deref for AssemblerRiscv {
63    type Target = Assembler;
64    fn deref(&self) -> &Self::Target {
65        &self.inner
66    }
67}
68
69impl DerefMut for AssemblerRiscv {
70    fn deref_mut(&mut self) -> &mut Self::Target {
71        &mut self.inner
72    }
73}
74
75/// The RISC-V machine state and code emitter.
76pub struct MachineRiscv {
77    assembler: AssemblerRiscv,
78    allow_unaligned_memory_accesses: bool,
79    used_gprs: FixedBitSet,
80    used_fprs: FixedBitSet,
81    trap_table: TrapTable,
82    /// Map from byte offset into wasm function to range of native instructions.
83    /// Ordered by increasing InstructionAddressMap::srcloc.
84    instructions_address_map: Vec<InstructionAddressMap>,
85    /// The source location for the current operator.
86    src_loc: u32,
87    /// Vector of unwind operations with offset.
88    unwind_ops: Vec<(usize, UnwindOps<GPR, FPR>)>,
89}
90
91const SCRATCH_REG: GPR = GPR::X28;
92
93impl MachineRiscv {
94    /// Creates a new RISC-V machine for code generation.
95    pub fn new(
96        target: Option<Target>,
97        allow_unaligned_memory_accesses: bool,
98    ) -> Result<Self, CompileError> {
99        // TODO: for now always require FPU
100        Ok(MachineRiscv {
101            assembler: AssemblerRiscv::new(0, target)?,
102            allow_unaligned_memory_accesses,
103            used_gprs: FixedBitSet::with_capacity(32),
104            used_fprs: FixedBitSet::with_capacity(32),
105            trap_table: TrapTable::default(),
106            instructions_address_map: vec![],
107            src_loc: 0,
108            unwind_ops: vec![],
109        })
110    }
111
112    fn used_gprs_contains(&self, r: &GPR) -> bool {
113        self.used_gprs.contains(r.into_index())
114    }
115    fn used_gprs_insert(&mut self, r: GPR) {
116        self.used_gprs.insert(r.into_index());
117    }
118    fn used_gprs_remove(&mut self, r: &GPR) -> bool {
119        let ret = self.used_gprs_contains(r);
120        self.used_gprs.set(r.into_index(), false);
121        ret
122    }
123
124    fn used_fp_contains(&self, r: &FPR) -> bool {
125        self.used_fprs.contains(r.into_index())
126    }
127    fn used_fprs_insert(&mut self, r: FPR) {
128        self.used_fprs.insert(r.into_index());
129    }
130    fn used_fprs_remove(&mut self, r: &FPR) -> bool {
131        let ret = self.used_fp_contains(r);
132        self.used_fprs.set(r.into_index(), false);
133        ret
134    }
135
136    fn location_to_reg(
137        &mut self,
138        sz: Size,
139        src: Location,
140        temps: &mut Vec<GPR>,
141        allow_imm: ImmType,
142        read_val: bool,
143        wanted: Option<GPR>,
144    ) -> Result<Location, CompileError> {
145        match src {
146            Location::GPR(_) | Location::SIMD(_) => Ok(src),
147            Location::Memory(reg, val) => {
148                let tmp = if let Some(wanted) = wanted {
149                    wanted
150                } else {
151                    let tmp = self.acquire_temp_gpr().ok_or_else(|| {
152                        CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
153                    })?;
154                    temps.push(tmp);
155                    tmp
156                };
157                if read_val {
158                    if ImmType::Bits12.compatible_imm(val as _) {
159                        self.assembler.emit_ld(sz, false, Location::GPR(tmp), src)?;
160                    } else {
161                        if reg == tmp {
162                            codegen_error!("singlepass reg == tmp unreachable");
163                        }
164                        self.assembler.emit_mov_imm(Location::GPR(tmp), val as _)?;
165                        self.assembler.emit_add(
166                            Size::S64,
167                            Location::GPR(reg),
168                            Location::GPR(tmp),
169                            Location::GPR(tmp),
170                        )?;
171                        self.assembler.emit_ld(
172                            sz,
173                            false,
174                            Location::GPR(tmp),
175                            Location::Memory(tmp, 0),
176                        )?;
177                    }
178                }
179                Ok(Location::GPR(tmp))
180            }
181            _ if src.is_imm() => {
182                let imm = src.imm_value_scalar().unwrap();
183                if imm == 0 {
184                    Ok(Location::GPR(GPR::XZero))
185                } else if allow_imm.compatible_imm(imm) {
186                    Ok(src)
187                } else {
188                    let tmp = if let Some(wanted) = wanted {
189                        wanted
190                    } else {
191                        let tmp = self.acquire_temp_gpr().ok_or_else(|| {
192                            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
193                        })?;
194                        temps.push(tmp);
195                        tmp
196                    };
197                    self.assembler.emit_mov_imm(Location::GPR(tmp), imm as _)?;
198                    Ok(Location::GPR(tmp))
199                }
200            }
201            _ => todo!("unsupported location"),
202        }
203    }
204
205    fn location_to_fpr(
206        &mut self,
207        sz: Size,
208        src: Location,
209        temps: &mut Vec<FPR>,
210        allow_imm: ImmType,
211        read_val: bool,
212    ) -> Result<Location, CompileError> {
213        match src {
214            Location::SIMD(_) => Ok(src),
215            Location::GPR(_) => {
216                let tmp = self.acquire_temp_simd().ok_or_else(|| {
217                    CompileError::Codegen("singlepass cannot acquire temp fpr".to_owned())
218                })?;
219                temps.push(tmp);
220                if read_val {
221                    self.assembler.emit_mov(sz, src, Location::SIMD(tmp))?;
222                }
223                Ok(Location::SIMD(tmp))
224            }
225            Location::Memory(_, _) => {
226                let tmp = self.acquire_temp_simd().ok_or_else(|| {
227                    CompileError::Codegen("singlepass cannot acquire temp fpr".to_owned())
228                })?;
229                temps.push(tmp);
230                if read_val {
231                    self.assembler
232                        .emit_ld(sz, false, Location::SIMD(tmp), src)?;
233                }
234                Ok(Location::SIMD(tmp))
235            }
236            _ if src.is_imm() => {
237                let tmp = self.acquire_temp_simd().ok_or_else(|| {
238                    CompileError::Codegen("singlepass cannot acquire temp fpr".to_owned())
239                })?;
240                temps.push(tmp);
241
242                let mut gpr_temps = vec![];
243                let dst =
244                    self.location_to_reg(sz, src, &mut gpr_temps, allow_imm, read_val, None)?;
245                self.assembler.emit_mov(sz, dst, Location::SIMD(tmp))?;
246                for r in gpr_temps {
247                    self.release_gpr(r);
248                }
249
250                Ok(Location::SIMD(tmp))
251            }
252            _ => todo!("unsupported location"),
253        }
254    }
255
256    fn emit_relaxed_binop(
257        &mut self,
258        op: fn(&mut Assembler, Size, Location, Location) -> Result<(), CompileError>,
259        sz: Size,
260        src: Location,
261        dst: Location,
262    ) -> Result<(), CompileError> {
263        let mut temps = vec![];
264        let src = self.location_to_reg(sz, src, &mut temps, ImmType::None, true, None)?;
265        let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
266        op(&mut self.assembler, sz, src, dest)?;
267        if dst != dest {
268            self.move_location(sz, dest, dst)?;
269        }
270        for r in temps {
271            self.release_gpr(r);
272        }
273        Ok(())
274    }
275
276    fn emit_relaxed_binop_fp(
277        &mut self,
278        op: fn(&mut Assembler, Size, Location, Location) -> Result<(), CompileError>,
279        sz: Size,
280        src: Location,
281        dst: Location,
282        putback: bool,
283    ) -> Result<(), CompileError> {
284        let mut temps = vec![];
285        let src = self.location_to_fpr(sz, src, &mut temps, ImmType::None, true)?;
286        let dest = self.location_to_fpr(sz, dst, &mut temps, ImmType::None, !putback)?;
287        op(&mut self.assembler, sz, src, dest)?;
288        if dst != dest && putback {
289            self.move_location(sz, dest, dst)?;
290        }
291        for r in temps {
292            self.release_simd(r);
293        }
294        Ok(())
295    }
296
297    fn emit_relaxed_binop3(
298        &mut self,
299        op: fn(&mut Assembler, Size, Location, Location, Location) -> Result<(), CompileError>,
300        sz: Size,
301        src1: Location,
302        src2: Location,
303        dst: Location,
304        allow_imm: ImmType,
305    ) -> Result<(), CompileError> {
306        let mut temps = vec![];
307        let src1 = self.location_to_reg(sz, src1, &mut temps, ImmType::None, true, None)?;
308        let src2 = self.location_to_reg(sz, src2, &mut temps, allow_imm, true, None)?;
309        let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
310        op(&mut self.assembler, sz, src1, src2, dest)?;
311        if dst != dest {
312            self.move_location(sz, dest, dst)?;
313        }
314        for r in temps {
315            self.release_gpr(r);
316        }
317        Ok(())
318    }
319
320    fn emit_relaxed_atomic_binop3(
321        &mut self,
322        op: AtomicBinaryOp,
323        sz: Size,
324        dst: Location,
325        addr: GPR,
326        src: Location,
327    ) -> Result<(), CompileError> {
328        let mut temps = vec![];
329        let source = self.location_to_reg(sz, src, &mut temps, ImmType::None, false, None)?;
330        let dest = self.location_to_reg(Size::S64, dst, &mut temps, ImmType::None, false, None)?;
331        let (Location::GPR(source), Location::GPR(dest)) = (source, dest) else {
332            panic!("emit_relaxed_atomic_binop3 expects locations in registers");
333        };
334
335        // RISC-V does not provide atomic operations for binary operations for S8 and S16 types.
336        // And so we must rely on 32-bit atomic operations with a proper masking.
337        match sz {
338            Size::S32 | Size::S64 => {
339                if op == AtomicBinaryOp::Sub {
340                    self.assembler.emit_neg(
341                        Size::S64,
342                        Location::GPR(source),
343                        Location::GPR(source),
344                    )?;
345                    self.assembler.emit_atomic_binop(
346                        AtomicBinaryOp::Add,
347                        sz,
348                        dest,
349                        addr,
350                        source,
351                    )?;
352                } else {
353                    self.assembler
354                        .emit_atomic_binop(op, sz, dest, addr, source)?;
355                }
356                self.assembler.emit_rwfence()?;
357            }
358            Size::S8 | Size::S16 => {
359                let aligned_addr = self.acquire_temp_gpr().ok_or_else(|| {
360                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
361                })?;
362                temps.push(aligned_addr);
363                let bit_offset = self.acquire_temp_gpr().ok_or_else(|| {
364                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
365                })?;
366                temps.push(bit_offset);
367                let bit_mask = self.acquire_temp_gpr().ok_or_else(|| {
368                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
369                })?;
370                temps.push(bit_mask);
371
372                self.assembler.emit_and(
373                    Size::S64,
374                    Location::GPR(addr),
375                    Location::Imm64(-4i64 as _),
376                    Location::GPR(aligned_addr),
377                )?;
378                self.assembler.emit_and(
379                    Size::S64,
380                    Location::GPR(addr),
381                    Location::Imm64(3),
382                    Location::GPR(bit_offset),
383                )?;
384                self.assembler.emit_sll(
385                    Size::S64,
386                    Location::GPR(bit_offset),
387                    Location::Imm64(3),
388                    Location::GPR(bit_offset),
389                )?;
390                self.assembler.emit_mov_imm(
391                    Location::GPR(bit_mask),
392                    if sz == Size::S8 {
393                        u8::MAX as _
394                    } else {
395                        u16::MAX as _
396                    },
397                )?;
398                self.assembler.emit_and(
399                    Size::S32,
400                    Location::GPR(source),
401                    Location::GPR(bit_mask),
402                    Location::GPR(source),
403                )?;
404                self.assembler.emit_sll(
405                    Size::S64,
406                    Location::GPR(bit_mask),
407                    Location::GPR(bit_offset),
408                    Location::GPR(bit_mask),
409                )?;
410                self.assembler.emit_sll(
411                    Size::S64,
412                    Location::GPR(source),
413                    Location::GPR(bit_offset),
414                    Location::GPR(source),
415                )?;
416
417                match op {
418                    AtomicBinaryOp::Add | AtomicBinaryOp::Sub | AtomicBinaryOp::Exchange => {
419                        let loaded_value = self.acquire_temp_gpr().ok_or_else(|| {
420                            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
421                        })?;
422                        temps.push(loaded_value);
423                        let tmp = self.acquire_temp_gpr().ok_or_else(|| {
424                            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
425                        })?;
426                        temps.push(tmp);
427
428                        // Loop
429                        let label_retry = self.get_label();
430                        self.emit_label(label_retry)?;
431
432                        self.assembler
433                            .emit_reserved_ld(Size::S32, loaded_value, aligned_addr)?;
434
435                        match op {
436                            AtomicBinaryOp::Add => self.assembler.emit_add(
437                                Size::S64,
438                                Location::GPR(loaded_value),
439                                Location::GPR(source),
440                                Location::GPR(tmp),
441                            )?,
442                            AtomicBinaryOp::Sub => self.assembler.emit_sub(
443                                Size::S64,
444                                Location::GPR(loaded_value),
445                                Location::GPR(source),
446                                Location::GPR(tmp),
447                            )?,
448                            AtomicBinaryOp::Exchange => self.assembler.emit_mov(
449                                Size::S64,
450                                Location::GPR(source),
451                                Location::GPR(tmp),
452                            )?,
453                            _ => unreachable!(),
454                        }
455
456                        self.assembler.emit_xor(
457                            Size::S64,
458                            Location::GPR(tmp),
459                            Location::GPR(loaded_value),
460                            Location::GPR(tmp),
461                        )?;
462                        self.assembler.emit_and(
463                            Size::S64,
464                            Location::GPR(tmp),
465                            Location::GPR(bit_mask),
466                            Location::GPR(tmp),
467                        )?;
468                        self.assembler.emit_xor(
469                            Size::S64,
470                            Location::GPR(tmp),
471                            Location::GPR(loaded_value),
472                            Location::GPR(tmp),
473                        )?;
474                        self.assembler
475                            .emit_reserved_sd(Size::S32, tmp, aligned_addr, tmp)?;
476                        let tmp2 = self.acquire_temp_gpr().ok_or_else(|| {
477                            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
478                        })?;
479                        temps.push(tmp2);
480                        self.assembler
481                            .emit_on_true_label(Location::GPR(tmp), label_retry, tmp2)?;
482
483                        self.assembler.emit_rwfence()?;
484
485                        // Return the previous value
486                        self.assembler.emit_and(
487                            Size::S32,
488                            Location::GPR(loaded_value),
489                            Location::GPR(bit_mask),
490                            Location::GPR(dest),
491                        )?;
492                        self.assembler.emit_srl(
493                            Size::S32,
494                            Location::GPR(dest),
495                            Location::GPR(bit_offset),
496                            Location::GPR(dest),
497                        )?;
498                    }
499                    AtomicBinaryOp::Or | AtomicBinaryOp::Xor => {
500                        self.assembler.emit_atomic_binop(
501                            op,
502                            Size::S32,
503                            dest,
504                            aligned_addr,
505                            source,
506                        )?;
507                        self.assembler.emit_rwfence()?;
508                        self.assembler.emit_and(
509                            Size::S32,
510                            Location::GPR(dest),
511                            Location::GPR(bit_mask),
512                            Location::GPR(dest),
513                        )?;
514                        self.assembler.emit_srl(
515                            Size::S32,
516                            Location::GPR(dest),
517                            Location::GPR(bit_offset),
518                            Location::GPR(dest),
519                        )?;
520                    }
521                    AtomicBinaryOp::And => {
522                        self.assembler.emit_not(
523                            Size::S64,
524                            Location::GPR(bit_mask),
525                            Location::GPR(bit_mask),
526                        )?;
527                        self.assembler.emit_or(
528                            Size::S64,
529                            Location::GPR(bit_mask),
530                            Location::GPR(source),
531                            Location::GPR(source),
532                        )?;
533                        self.assembler.emit_not(
534                            Size::S64,
535                            Location::GPR(bit_mask),
536                            Location::GPR(bit_mask),
537                        )?;
538                        self.assembler.emit_atomic_binop(
539                            op,
540                            Size::S32,
541                            dest,
542                            aligned_addr,
543                            source,
544                        )?;
545                        self.assembler.emit_rwfence()?;
546                        self.assembler.emit_and(
547                            Size::S32,
548                            Location::GPR(dest),
549                            Location::GPR(bit_mask),
550                            Location::GPR(dest),
551                        )?;
552                        self.assembler.emit_srl(
553                            Size::S32,
554                            Location::GPR(dest),
555                            Location::GPR(bit_offset),
556                            Location::GPR(dest),
557                        )?;
558                    }
559                }
560            }
561        }
562
563        if dst != Location::GPR(dest) {
564            self.move_location(sz, Location::GPR(dest), dst)?;
565        }
566
567        for r in temps {
568            self.release_gpr(r);
569        }
570        Ok(())
571    }
572
573    fn emit_relaxed_atomic_cmpxchg(
574        &mut self,
575        size: Size,
576        dst: Location,
577        addr: GPR,
578        new: Location,
579        cmp: Location,
580    ) -> Result<(), CompileError> {
581        let mut temps = vec![];
582        let cmp = self.location_to_reg(size, cmp, &mut temps, ImmType::None, true, None)?;
583        let new = self.location_to_reg(size, new, &mut temps, ImmType::None, true, None)?;
584        let (Location::GPR(cmp), Location::GPR(new)) = (cmp, new) else {
585            panic!("emit_relaxed_atomic_cmpxchg expects locations in registers");
586        };
587
588        let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
589            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
590        })?;
591        temps.push(jmp_tmp);
592
593        match size {
594            Size::S32 | Size::S64 => {
595                let value = self.acquire_temp_gpr().ok_or_else(|| {
596                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
597                })?;
598                temps.push(value);
599                let cond = self.acquire_temp_gpr().ok_or_else(|| {
600                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
601                })?;
602                temps.push(cond);
603
604                // main re-try loop
605                let label_retry = self.get_label();
606                let label_after_retry = self.get_label();
607                self.emit_label(label_retry)?;
608
609                self.assembler.emit_reserved_ld(size, value, addr)?;
610                self.assembler.emit_cmp(
611                    Condition::Eq,
612                    Location::GPR(value),
613                    Location::GPR(cmp),
614                    Location::GPR(cond),
615                )?;
616                self.assembler.emit_on_false_label(
617                    Location::GPR(cond),
618                    label_after_retry,
619                    jmp_tmp,
620                )?;
621                self.assembler.emit_reserved_sd(size, cond, addr, new)?;
622                self.assembler
623                    .emit_on_true_label(Location::GPR(cond), label_retry, jmp_tmp)?;
624
625                // after re-try get the previous value
626                self.assembler.emit_rwfence()?;
627                self.emit_label(label_after_retry)?;
628
629                self.assembler.emit_mov(size, Location::GPR(value), dst)?;
630            }
631            Size::S8 | Size::S16 => {
632                let value = self.acquire_temp_gpr().ok_or_else(|| {
633                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
634                })?;
635                temps.push(value);
636                let tmp = self.acquire_temp_gpr().ok_or_else(|| {
637                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
638                })?;
639                temps.push(tmp);
640                let bit_offset = self.acquire_temp_gpr().ok_or_else(|| {
641                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
642                })?;
643                temps.push(bit_offset);
644                let bit_mask = self.acquire_temp_gpr().ok_or_else(|| {
645                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
646                })?;
647                temps.push(bit_mask);
648                let cond = self.acquire_temp_gpr().ok_or_else(|| {
649                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
650                })?;
651                temps.push(cond);
652
653                // before the loop
654                self.assembler.emit_and(
655                    Size::S64,
656                    Location::GPR(addr),
657                    Location::Imm64(3),
658                    Location::GPR(bit_offset),
659                )?;
660                self.assembler.emit_and(
661                    Size::S64,
662                    Location::GPR(addr),
663                    Location::Imm64(-4i64 as _),
664                    Location::GPR(addr),
665                )?;
666                self.assembler.emit_sll(
667                    Size::S64,
668                    Location::GPR(bit_offset),
669                    Location::Imm64(3),
670                    Location::GPR(bit_offset),
671                )?;
672                self.assembler.emit_mov_imm(
673                    Location::GPR(bit_mask),
674                    if size == Size::S8 {
675                        u8::MAX as _
676                    } else {
677                        u16::MAX as _
678                    },
679                )?;
680                self.assembler.emit_and(
681                    Size::S32,
682                    Location::GPR(new),
683                    Location::GPR(bit_mask),
684                    Location::GPR(new),
685                )?;
686                self.assembler.emit_sll(
687                    Size::S64,
688                    Location::GPR(bit_mask),
689                    Location::GPR(bit_offset),
690                    Location::GPR(bit_mask),
691                )?;
692                self.assembler.emit_sll(
693                    Size::S64,
694                    Location::GPR(new),
695                    Location::GPR(bit_offset),
696                    Location::GPR(new),
697                )?;
698                self.assembler.emit_sll(
699                    Size::S64,
700                    Location::GPR(cmp),
701                    Location::GPR(bit_offset),
702                    Location::GPR(cmp),
703                )?;
704
705                // main re-try loop
706                let label_retry = self.get_label();
707                let label_after_retry = self.get_label();
708                self.emit_label(label_retry)?;
709
710                self.assembler.emit_reserved_ld(Size::S32, value, addr)?;
711                self.assembler.emit_and(
712                    Size::S32,
713                    Location::GPR(value),
714                    Location::GPR(bit_mask),
715                    Location::GPR(tmp),
716                )?;
717
718                self.assembler.emit_cmp(
719                    Condition::Eq,
720                    Location::GPR(tmp),
721                    Location::GPR(cmp),
722                    Location::GPR(cond),
723                )?;
724                self.assembler.emit_on_false_label(
725                    Location::GPR(cond),
726                    label_after_retry,
727                    jmp_tmp,
728                )?;
729
730                // mask new to the 4B word
731                self.assembler.emit_xor(
732                    Size::S32,
733                    Location::GPR(value),
734                    Location::GPR(new),
735                    Location::GPR(tmp),
736                )?;
737                self.assembler.emit_and(
738                    Size::S32,
739                    Location::GPR(tmp),
740                    Location::GPR(bit_mask),
741                    Location::GPR(tmp),
742                )?;
743                self.assembler.emit_xor(
744                    Size::S32,
745                    Location::GPR(tmp),
746                    Location::GPR(value),
747                    Location::GPR(tmp),
748                )?;
749                self.assembler
750                    .emit_reserved_sd(Size::S32, cond, addr, tmp)?;
751                self.assembler
752                    .emit_on_true_label(Location::GPR(cond), label_retry, jmp_tmp)?;
753
754                // After re-try get the previous value
755                self.assembler.emit_rwfence()?;
756                self.emit_label(label_after_retry)?;
757
758                self.assembler.emit_and(
759                    Size::S32,
760                    Location::GPR(value),
761                    Location::GPR(bit_mask),
762                    Location::GPR(tmp),
763                )?;
764                self.assembler.emit_srl(
765                    Size::S32,
766                    Location::GPR(tmp),
767                    Location::GPR(bit_offset),
768                    Location::GPR(tmp),
769                )?;
770                self.assembler
771                    .emit_mov(Size::S32, Location::GPR(tmp), dst)?;
772            }
773        }
774
775        for r in temps {
776            self.release_gpr(r);
777        }
778        Ok(())
779    }
780
781    #[allow(clippy::too_many_arguments)]
782    fn emit_relaxed_binop3_fp(
783        &mut self,
784        op: fn(&mut Assembler, Size, Location, Location, Location) -> Result<(), CompileError>,
785        sz: Size,
786        src1: Location,
787        src2: Location,
788        dst: Location,
789        allow_imm: ImmType,
790        return_nan_if_present: bool,
791    ) -> Result<(), CompileError> {
792        let mut temps = vec![];
793        let mut gprs = vec![];
794        let src1 = self.location_to_fpr(sz, src1, &mut temps, ImmType::None, true)?;
795        let src2 = self.location_to_fpr(sz, src2, &mut temps, allow_imm, true)?;
796        let dest = self.location_to_fpr(sz, dst, &mut temps, ImmType::None, false)?;
797
798        let label_after = self.get_label();
799        if return_nan_if_present {
800            let tmp = self.acquire_temp_gpr().ok_or_else(|| {
801                CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
802            })?;
803            gprs.push(tmp);
804            let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
805                CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
806            })?;
807            gprs.push(jmp_tmp);
808
809            // Return an ArithmeticNan if either src1 or (and) src2 have a NaN value.
810            let canonical_nan = match sz {
811                Size::S32 => CANONICAL_NAN_F32 as u64,
812                Size::S64 => CANONICAL_NAN_F64,
813                _ => unreachable!(),
814            };
815
816            self.assembler
817                .emit_mov_imm(Location::GPR(tmp), canonical_nan as _)?;
818            self.assembler.emit_mov(sz, Location::GPR(tmp), dest)?;
819
820            self.assembler
821                .emit_fcmp(Condition::Eq, sz, src1, src1, Location::GPR(tmp))?;
822            self.assembler
823                .emit_on_false_label(Location::GPR(tmp), label_after, jmp_tmp)?;
824
825            self.assembler
826                .emit_fcmp(Condition::Eq, sz, src2, src2, Location::GPR(tmp))?;
827            self.assembler
828                .emit_on_false_label(Location::GPR(tmp), label_after, jmp_tmp)?;
829        }
830
831        op(&mut self.assembler, sz, src1, src2, dest)?;
832        self.emit_label(label_after)?;
833
834        if dst != dest {
835            self.move_location(sz, dest, dst)?;
836        }
837        for r in temps {
838            self.release_simd(r);
839        }
840        for r in gprs {
841            self.release_gpr(r);
842        }
843        Ok(())
844    }
845
846    fn emit_relaxed_cmp(
847        &mut self,
848        c: Condition,
849        loc_a: Location,
850        loc_b: Location,
851        ret: Location,
852        sz: Size,
853        signed: bool,
854    ) -> Result<(), CompileError> {
855        // TODO: add support for immediate operations where some instructions (like `slti`) can be used
856        let mut temps = vec![];
857        let loc_a = self.location_to_reg(sz, loc_a, &mut temps, ImmType::None, true, None)?;
858        let loc_b = self.location_to_reg(sz, loc_b, &mut temps, ImmType::None, true, None)?;
859
860        if sz != Size::S64 {
861            self.assembler.emit_extend(sz, signed, loc_a, loc_a)?;
862            self.assembler.emit_extend(sz, signed, loc_b, loc_b)?;
863        }
864
865        let dest = self.location_to_reg(sz, ret, &mut temps, ImmType::None, false, None)?;
866        self.assembler.emit_cmp(c, loc_a, loc_b, dest)?;
867        if ret != dest {
868            self.move_location(sz, dest, ret)?;
869        }
870        for r in temps {
871            self.release_gpr(r);
872        }
873        Ok(())
874    }
875
876    /// I32 comparison with.
877    fn emit_cmpop_i32_dynamic_b(
878        &mut self,
879        c: Condition,
880        loc_a: Location,
881        loc_b: Location,
882        ret: Location,
883        signed: bool,
884    ) -> Result<(), CompileError> {
885        self.emit_relaxed_cmp(c, loc_a, loc_b, ret, Size::S32, signed)
886    }
887
888    /// I64 comparison with.
889    fn emit_cmpop_i64_dynamic_b(
890        &mut self,
891        c: Condition,
892        loc_a: Location,
893        loc_b: Location,
894        ret: Location,
895    ) -> Result<(), CompileError> {
896        self.emit_relaxed_cmp(c, loc_a, loc_b, ret, Size::S64, false)
897    }
898
899    /// NOTE: As observed on the VisionFive 2 board, when an unaligned memory write happens to write out of bounds (and thus triggers SIGSEGV),
900    /// the memory is partially modified and observable for a subsequent memory read operations.
901    /// Thus, we always check the boundaries.
902    #[allow(clippy::too_many_arguments)]
903    fn memory_op<F: FnOnce(&mut Self, GPR) -> Result<(), CompileError>>(
904        &mut self,
905        addr: Location,
906        memarg: &MemArg,
907        check_alignment: bool,
908        value_size: usize,
909        imported_memories: bool,
910        offset: i32,
911        heap_access_oob: Label,
912        unaligned_atomic: Label,
913        cb: F,
914    ) -> Result<(), CompileError> {
915        let value_size = value_size as i64;
916        let tmp_addr = self.acquire_temp_gpr().ok_or_else(|| {
917            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
918        })?;
919
920        // Reusing `tmp_addr` for temporary indirection here, since it's not used before the last reference to `{base,bound}_loc`.
921        let (base_loc, bound_loc) = if imported_memories {
922            // Imported memories require one level of indirection.
923            self.emit_relaxed_binop(
924                Assembler::emit_mov,
925                Size::S64,
926                Location::Memory(self.get_vmctx_reg(), offset),
927                Location::GPR(tmp_addr),
928            )?;
929            (Location::Memory(tmp_addr, 0), Location::Memory(tmp_addr, 8))
930        } else {
931            (
932                Location::Memory(self.get_vmctx_reg(), offset),
933                Location::Memory(self.get_vmctx_reg(), offset + 8),
934            )
935        };
936
937        let tmp_base = self.acquire_temp_gpr().ok_or_else(|| {
938            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
939        })?;
940        let tmp_bound = self.acquire_temp_gpr().ok_or_else(|| {
941            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
942        })?;
943
944        // Load base into temporary register.
945        self.emit_relaxed_load(Size::S64, false, Location::GPR(tmp_base), base_loc)?;
946
947        // Load bound into temporary register.
948        self.emit_relaxed_load(Size::S64, false, Location::GPR(tmp_bound), bound_loc)?;
949
950        // Wasm -> Effective.
951        // Assuming we never underflow - should always be true on Linux/macOS and Windows >=8,
952        // since the first page from 0x0 to 0x1000 is not accepted by mmap.
953        self.assembler.emit_add(
954            Size::S64,
955            Location::GPR(tmp_bound),
956            Location::GPR(tmp_base),
957            Location::GPR(tmp_bound),
958        )?;
959        self.assembler.emit_sub(
960            Size::S64,
961            Location::GPR(tmp_bound),
962            Location::Imm64(value_size as _),
963            Location::GPR(tmp_bound),
964        )?;
965
966        // Load effective address.
967        // `base_loc` and `bound_loc` becomes INVALID after this line, because `tmp_addr`
968        // might be reused.
969        self.move_location(Size::S32, addr, Location::GPR(tmp_addr))?;
970
971        let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
972            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
973        })?;
974
975        // Add offset to memory address.
976        if memarg.offset != 0 {
977            if ImmType::Bits12.compatible_imm(memarg.offset as _) {
978                self.assembler.emit_add(
979                    Size::S64,
980                    Location::Imm64(memarg.offset),
981                    Location::GPR(tmp_addr),
982                    Location::GPR(tmp_addr),
983                )?;
984            } else {
985                let tmp = self.acquire_temp_gpr().ok_or_else(|| {
986                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
987                })?;
988                self.assembler
989                    .emit_mov_imm(Location::GPR(tmp), memarg.offset as _)?;
990                self.assembler.emit_add(
991                    Size::S64,
992                    Location::GPR(tmp_addr),
993                    Location::GPR(tmp),
994                    Location::GPR(tmp_addr),
995                )?;
996                self.release_gpr(tmp);
997            }
998
999            // Trap if offset calculation overflowed in 32-bits by checking
1000            // the upper half of the 64-bit register.
1001            let tmp = self.acquire_temp_gpr().ok_or_else(|| {
1002                CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1003            })?;
1004            self.assembler.emit_srl(
1005                Size::S64,
1006                Location::GPR(tmp_addr),
1007                Location::Imm64(32),
1008                Location::GPR(tmp),
1009            )?;
1010            self.assembler
1011                .emit_on_true_label_far(Location::GPR(tmp), heap_access_oob, jmp_tmp)?;
1012            self.release_gpr(tmp);
1013        }
1014
1015        // Wasm linear memory -> real memory
1016        self.assembler.emit_add(
1017            Size::S64,
1018            Location::GPR(tmp_base),
1019            Location::GPR(tmp_addr),
1020            Location::GPR(tmp_addr),
1021        )?;
1022
1023        // tmp_base is already unused
1024        let cond = tmp_base;
1025
1026        // Trap if the end address of the requested area is above that of the linear memory.
1027        self.assembler.emit_cmp(
1028            Condition::Le,
1029            Location::GPR(tmp_addr),
1030            Location::GPR(tmp_bound),
1031            Location::GPR(cond),
1032        )?;
1033
1034        // `tmp_bound` is inclusive. So trap only if `tmp_addr > tmp_bound`.
1035        self.assembler
1036            .emit_on_false_label_far(Location::GPR(cond), heap_access_oob, jmp_tmp)?;
1037
1038        self.release_gpr(tmp_bound);
1039        self.release_gpr(cond);
1040
1041        let align = value_size as u32;
1042        if check_alignment && align != 1 {
1043            self.assembler.emit_and(
1044                Size::S64,
1045                Location::GPR(tmp_addr),
1046                Location::Imm64((align - 1) as u64),
1047                Location::GPR(cond),
1048            )?;
1049            self.assembler.emit_on_true_label_far(
1050                Location::GPR(cond),
1051                unaligned_atomic,
1052                jmp_tmp,
1053            )?;
1054        }
1055        let begin = self.assembler.get_offset().0;
1056        cb(self, tmp_addr)?;
1057        let end = self.assembler.get_offset().0;
1058        self.mark_address_range_with_trap_code(TrapCode::HeapAccessOutOfBounds, begin, end);
1059
1060        self.release_gpr(jmp_tmp);
1061        self.release_gpr(tmp_addr);
1062        Ok(())
1063    }
1064
1065    fn emit_relaxed_load(
1066        &mut self,
1067        sz: Size,
1068        signed: bool,
1069        dst: Location,
1070        src: Location,
1071    ) -> Result<(), CompileError> {
1072        let mut temps = vec![];
1073        let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
1074        match src {
1075            Location::Memory(addr, offset) => {
1076                if ImmType::Bits12.compatible_imm(offset as i64) {
1077                    self.assembler.emit_ld(sz, signed, dest, src)?;
1078                } else {
1079                    let tmp = self.acquire_temp_gpr().ok_or_else(|| {
1080                        CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1081                    })?;
1082                    self.assembler
1083                        .emit_mov_imm(Location::GPR(tmp), offset as i64)?;
1084                    self.assembler.emit_add(
1085                        Size::S64,
1086                        Location::GPR(addr),
1087                        Location::GPR(tmp),
1088                        Location::GPR(tmp),
1089                    )?;
1090                    self.assembler
1091                        .emit_ld(sz, signed, dest, Location::Memory(tmp, 0))?;
1092                    temps.push(tmp);
1093                }
1094            }
1095            _ => codegen_error!("singlepass emit_relaxed_load unreachable"),
1096        }
1097        if dst != dest {
1098            // Memory location is used for a local, save the entire register!
1099            self.move_location(Size::S64, dest, dst)?;
1100        }
1101        for r in temps {
1102            self.release_gpr(r);
1103        }
1104        Ok(())
1105    }
1106
1107    fn emit_maybe_unaligned_load(
1108        &mut self,
1109        sz: Size,
1110        signed: bool,
1111        dst: Location,
1112        src: GPR,
1113    ) -> Result<(), CompileError> {
1114        if !self.allow_unaligned_memory_accesses {
1115            return self.emit_relaxed_load(sz, signed, dst, Location::Memory(src, 0));
1116        }
1117
1118        if let Size::S8 = sz {
1119            return self.emit_relaxed_load(sz, signed, dst, Location::Memory(src, 0));
1120        }
1121
1122        let label_unaligned = self.get_label();
1123        let label_completed = self.get_label();
1124        let mut temps = vec![];
1125        let tmp_cond = self.acquire_temp_gpr().ok_or_else(|| {
1126            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1127        })?;
1128        temps.push(tmp_cond);
1129        let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
1130        self.assembler.emit_and(
1131            Size::S64,
1132            Location::GPR(src),
1133            Location::Imm64((sz.bytes() - 1) as u64),
1134            Location::GPR(tmp_cond),
1135        )?;
1136        let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
1137            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1138        })?;
1139        temps.push(jmp_tmp);
1140        self.assembler
1141            .emit_on_true_label(Location::GPR(tmp_cond), label_unaligned, jmp_tmp)?;
1142        // Aligned load
1143        self.assembler
1144            .emit_ld(sz, signed, dest, Location::Memory(src, 0))?;
1145        self.jmp_unconditional(label_completed)?;
1146        self.emit_label(label_unaligned)?;
1147
1148        // Unaligned load
1149        let tmp_value = tmp_cond;
1150
1151        // We assume little-endian for now
1152        self.assembler
1153            .emit_ld(Size::S8, false, dest, Location::Memory(src, 0))?;
1154        for i in 1..sz.bytes() {
1155            self.assembler.emit_ld(
1156                Size::S8,
1157                if i == sz.bytes() - 1 { signed } else { false },
1158                Location::GPR(tmp_value),
1159                Location::Memory(src, i as _),
1160            )?;
1161            self.assembler.emit_sll(
1162                Size::S64,
1163                Location::GPR(tmp_value),
1164                Location::Imm64(8 * i as u64),
1165                Location::GPR(tmp_value),
1166            )?;
1167            self.assembler
1168                .emit_or(Size::S64, dest, Location::GPR(tmp_value), dest)?;
1169        }
1170
1171        // Load completed
1172        self.emit_label(label_completed)?;
1173        if dst != dest {
1174            // Memory location is used for a local, save the entire register!
1175            self.move_location(Size::S64, dest, dst)?;
1176        }
1177
1178        for tmp in temps {
1179            self.release_gpr(tmp);
1180        }
1181        Ok(())
1182    }
1183
1184    fn emit_relaxed_store(
1185        &mut self,
1186        sz: Size,
1187        dst: Location,
1188        src: Location,
1189    ) -> Result<(), CompileError> {
1190        let mut temps = vec![];
1191        let dest = self.location_to_reg(Size::S64, dst, &mut temps, ImmType::None, true, None)?;
1192        match src {
1193            Location::Memory(addr, offset) => {
1194                if ImmType::Bits12.compatible_imm(offset as i64) {
1195                    self.assembler.emit_sd(sz, dest, src)?;
1196                } else {
1197                    let tmp = self.acquire_temp_gpr().ok_or_else(|| {
1198                        CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1199                    })?;
1200                    self.assembler
1201                        .emit_mov_imm(Location::GPR(tmp), offset as i64)?;
1202                    self.assembler.emit_add(
1203                        Size::S64,
1204                        Location::GPR(addr),
1205                        Location::GPR(tmp),
1206                        Location::GPR(tmp),
1207                    )?;
1208                    self.assembler.emit_sd(sz, dest, Location::Memory(tmp, 0))?;
1209                    temps.push(tmp);
1210                }
1211            }
1212            _ => codegen_error!("singlepass emit_relaxed_store unreachable"),
1213        }
1214        for r in temps {
1215            self.release_gpr(r);
1216        }
1217        Ok(())
1218    }
1219
1220    fn emit_maybe_unaligned_store(
1221        &mut self,
1222        sz: Size,
1223        src: Location,
1224        dst: GPR,
1225    ) -> Result<(), CompileError> {
1226        if !self.allow_unaligned_memory_accesses {
1227            return self.emit_relaxed_store(sz, src, Location::Memory(dst, 0));
1228        }
1229
1230        if let Size::S8 = sz {
1231            // `emit_relaxed_store` uses wrong order of src and dst.
1232            // The `src` parameter of `emit_relaxed_store` actually stores
1233            // the destination address, while the `dst` parameter of
1234            // `emit_relaxed_store` actually contains the source register.
1235            return self.emit_relaxed_store(sz, src, Location::Memory(dst, 0));
1236        }
1237
1238        let label_unaligned = self.get_label();
1239        let label_completed = self.get_label();
1240        let mut temps = vec![];
1241        let tmp_cond = self.acquire_temp_gpr().ok_or_else(|| {
1242            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1243        })?;
1244        temps.push(tmp_cond);
1245        let src_value =
1246            self.location_to_reg(Size::S64, src, &mut temps, ImmType::None, true, None)?;
1247        self.assembler.emit_and(
1248            Size::S64,
1249            Location::GPR(dst),
1250            Location::Imm64((sz.bytes() - 1) as u64),
1251            Location::GPR(tmp_cond),
1252        )?;
1253        let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
1254            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1255        })?;
1256        temps.push(jmp_tmp);
1257        self.assembler
1258            .emit_on_true_label(Location::GPR(tmp_cond), label_unaligned, jmp_tmp)?;
1259        // Aligned store
1260        self.assembler
1261            .emit_sd(sz, src_value, Location::Memory(dst, 0))?;
1262        self.jmp_unconditional(label_completed)?;
1263        self.emit_label(label_unaligned)?;
1264
1265        // Unaligned store
1266        let size_bytes = sz.bytes();
1267        // We assume little-endian for now
1268        for i in 0..size_bytes {
1269            self.assembler
1270                .emit_sd(Size::S8, src_value, Location::Memory(dst, i as _))?;
1271            if i != size_bytes - 1 {
1272                self.assembler
1273                    .emit_srl(Size::S64, src_value, Location::Imm64(8), src_value)?;
1274            }
1275        }
1276
1277        // Store completed
1278        self.emit_label(label_completed)?;
1279
1280        for tmp in temps {
1281            self.release_gpr(tmp);
1282        }
1283        Ok(())
1284    }
1285
1286    fn emit_rol(
1287        &mut self,
1288        sz: Size,
1289        loc_a: Location,
1290        loc_b: Location,
1291        ret: Location,
1292        allow_imm: ImmType,
1293    ) -> Result<(), CompileError> {
1294        let mut temps = vec![];
1295        let size_bits = sz.bits();
1296
1297        let src2 = if let Some(imm) = loc_b.imm_value_scalar() {
1298            Location::Imm32(size_bits - (imm as u32) % size_bits)
1299        } else {
1300            let tmp1 = self.location_to_reg(
1301                sz,
1302                Location::Imm32(size_bits),
1303                &mut temps,
1304                ImmType::None,
1305                true,
1306                None,
1307            )?;
1308            let tmp2 = self.location_to_reg(sz, loc_b, &mut temps, ImmType::None, true, None)?;
1309            self.assembler.emit_sub(sz, tmp1, tmp2, tmp1)?;
1310            tmp1
1311        };
1312
1313        self.emit_ror(sz, loc_a, src2, ret, allow_imm)?;
1314
1315        for r in temps {
1316            self.release_gpr(r);
1317        }
1318        Ok(())
1319    }
1320
1321    fn emit_ror(
1322        &mut self,
1323        sz: Size,
1324        loc_a: Location,
1325        loc_b: Location,
1326        ret: Location,
1327        allow_imm: ImmType,
1328    ) -> Result<(), CompileError> {
1329        let mut temps = vec![];
1330
1331        let imm = match sz {
1332            Size::S32 | Size::S64 => Location::Imm32(sz.bits() as u32),
1333            _ => codegen_error!("singlepass emit_ror unreachable"),
1334        };
1335        let imm = self.location_to_reg(sz, imm, &mut temps, ImmType::None, false, None)?;
1336
1337        let tmp1 = self.acquire_temp_gpr().ok_or_else(|| {
1338            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1339        })?;
1340        self.emit_relaxed_binop3(
1341            Assembler::emit_srl,
1342            sz,
1343            loc_a,
1344            loc_b,
1345            Location::GPR(tmp1),
1346            allow_imm,
1347        )?;
1348
1349        let tmp2 = self.acquire_temp_gpr().ok_or_else(|| {
1350            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1351        })?;
1352        self.emit_relaxed_binop3(
1353            Assembler::emit_sub,
1354            sz,
1355            imm,
1356            loc_b,
1357            Location::GPR(tmp2),
1358            allow_imm,
1359        )?;
1360        self.emit_relaxed_binop3(
1361            Assembler::emit_sll,
1362            sz,
1363            loc_a,
1364            Location::GPR(tmp2),
1365            Location::GPR(tmp2),
1366            ImmType::Bits12,
1367        )?;
1368        self.assembler.emit_or(
1369            sz,
1370            Location::GPR(tmp1),
1371            Location::GPR(tmp2),
1372            Location::GPR(tmp1),
1373        )?;
1374
1375        self.move_location(sz, Location::GPR(tmp1), ret)?;
1376        self.release_gpr(tmp1);
1377        self.release_gpr(tmp2);
1378        for r in temps {
1379            self.release_gpr(r);
1380        }
1381        Ok(())
1382    }
1383
1384    fn emit_popcnt(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1385        let arg = self.acquire_temp_gpr().ok_or_else(|| {
1386            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1387        })?;
1388        let cnt = self.acquire_temp_gpr().ok_or_else(|| {
1389            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1390        })?;
1391        let temp = self.acquire_temp_gpr().ok_or_else(|| {
1392            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1393        })?;
1394        self.move_location(sz, src, Location::GPR(arg))?;
1395
1396        self.move_location(sz, Location::Imm32(0), Location::GPR(cnt))?;
1397        let one_imm = match sz {
1398            Size::S32 => Location::Imm32(1),
1399            Size::S64 => Location::Imm64(1),
1400            _ => codegen_error!("singlepass emit_popcnt unreachable"),
1401        };
1402
1403        let label_loop = self.assembler.get_label();
1404        let label_exit = self.assembler.get_label();
1405
1406        self.assembler.emit_label(label_loop)?; // loop:
1407        self.assembler
1408            .emit_and(sz, Location::GPR(arg), one_imm, Location::GPR(temp))?;
1409        self.assembler.emit_add(
1410            sz,
1411            Location::GPR(cnt),
1412            Location::GPR(temp),
1413            Location::GPR(cnt),
1414        )?;
1415        self.assembler
1416            .emit_srl(sz, Location::GPR(arg), one_imm, Location::GPR(arg))?;
1417        let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
1418            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1419        })?;
1420        self.assembler
1421            .emit_on_false_label(Location::GPR(arg), label_exit, jmp_tmp)?;
1422        self.release_gpr(jmp_tmp);
1423        self.jmp_unconditional(label_loop)?;
1424
1425        self.assembler.emit_label(label_exit)?; // exit:
1426
1427        self.move_location(sz, Location::GPR(cnt), dst)?;
1428
1429        self.release_gpr(arg);
1430        self.release_gpr(cnt);
1431        self.release_gpr(temp);
1432        Ok(())
1433    }
1434
1435    fn emit_ctz(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1436        let size_bits = sz.bits();
1437        let arg = self.acquire_temp_gpr().ok_or_else(|| {
1438            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1439        })?;
1440        let cnt = self.acquire_temp_gpr().ok_or_else(|| {
1441            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1442        })?;
1443        let temp = self.acquire_temp_gpr().ok_or_else(|| {
1444            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1445        })?;
1446        self.move_location(sz, src, Location::GPR(arg))?;
1447
1448        let one_imm = match sz {
1449            Size::S32 => Location::Imm32(1),
1450            Size::S64 => Location::Imm64(1),
1451            _ => codegen_error!("singlepass emit_ctz unreachable"),
1452        };
1453
1454        let label_loop = self.assembler.get_label();
1455        let label_exit = self.assembler.get_label();
1456
1457        let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
1458            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1459        })?;
1460
1461        // if the value is zero, return size_bits
1462        self.move_location(sz, Location::Imm32(size_bits), Location::GPR(cnt))?;
1463        self.assembler
1464            .emit_on_false_label(Location::GPR(arg), label_exit, jmp_tmp)?;
1465
1466        self.move_location(sz, Location::Imm32(0), Location::GPR(cnt))?;
1467
1468        self.assembler.emit_label(label_loop)?; // loop:
1469        self.assembler
1470            .emit_and(sz, Location::GPR(arg), one_imm, Location::GPR(temp))?;
1471        self.assembler
1472            .emit_on_true_label(Location::GPR(temp), label_exit, jmp_tmp)?;
1473
1474        self.release_gpr(jmp_tmp);
1475
1476        self.assembler
1477            .emit_add(sz, Location::GPR(cnt), one_imm, Location::GPR(cnt))?;
1478        self.assembler
1479            .emit_srl(sz, Location::GPR(arg), one_imm, Location::GPR(arg))?;
1480        self.jmp_unconditional(label_loop)?;
1481
1482        self.assembler.emit_label(label_exit)?; // exit:
1483
1484        self.move_location(sz, Location::GPR(cnt), dst)?;
1485
1486        self.release_gpr(arg);
1487        self.release_gpr(cnt);
1488        self.release_gpr(temp);
1489        Ok(())
1490    }
1491
1492    fn emit_clz(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1493        let size_bits = sz.bits();
1494        let arg = self.acquire_temp_gpr().ok_or_else(|| {
1495            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1496        })?;
1497        let cnt = self.acquire_temp_gpr().ok_or_else(|| {
1498            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1499        })?;
1500        let tmp = self.acquire_temp_gpr().ok_or_else(|| {
1501            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1502        })?;
1503        self.move_location(sz, src, Location::GPR(arg))?;
1504
1505        let one_imm = match sz {
1506            Size::S32 => Location::Imm32(1),
1507            Size::S64 => Location::Imm64(1),
1508            _ => codegen_error!("singlepass emit_ctz unreachable"),
1509        };
1510
1511        let label_loop = self.assembler.get_label();
1512        let label_exit = self.assembler.get_label();
1513
1514        let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
1515            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1516        })?;
1517
1518        // if the value is zero, return size_bits
1519        self.move_location(sz, Location::Imm32(size_bits), Location::GPR(cnt))?;
1520        self.assembler
1521            .emit_on_false_label(Location::GPR(arg), label_exit, jmp_tmp)?;
1522
1523        self.move_location(sz, Location::Imm32(0), Location::GPR(cnt))?;
1524
1525        // loop:
1526        self.assembler.emit_label(label_loop)?;
1527        // Shift the argument by (bit_size - cnt - 1) and test if it's one
1528        self.move_location(
1529            Size::S32,
1530            Location::Imm32(size_bits - 1),
1531            Location::GPR(tmp),
1532        )?;
1533        self.assembler.emit_sub(
1534            Size::S32,
1535            Location::GPR(tmp),
1536            Location::GPR(cnt),
1537            Location::GPR(tmp),
1538        )?;
1539        self.assembler.emit_srl(
1540            sz,
1541            Location::GPR(arg),
1542            Location::GPR(tmp),
1543            Location::GPR(tmp),
1544        )?;
1545        self.assembler
1546            .emit_on_true_label(Location::GPR(tmp), label_exit, jmp_tmp)?;
1547
1548        self.assembler
1549            .emit_add(sz, Location::GPR(cnt), one_imm, Location::GPR(cnt))?;
1550        self.release_gpr(jmp_tmp);
1551        self.jmp_unconditional(label_loop)?;
1552
1553        self.assembler.emit_label(label_exit)?; // exit:i
1554
1555        self.move_location(sz, Location::GPR(cnt), dst)?;
1556
1557        self.release_gpr(arg);
1558        self.release_gpr(cnt);
1559        self.release_gpr(tmp);
1560        Ok(())
1561    }
1562
1563    fn convert_float_to_int(
1564        &mut self,
1565        loc: Location,
1566        size_in: Size,
1567        ret: Location,
1568        size_out: Size,
1569        signed: bool,
1570        sat: bool,
1571    ) -> Result<(), CompileError> {
1572        let mut gprs = vec![];
1573        let mut fprs = vec![];
1574        let src = self.location_to_fpr(size_in, loc, &mut fprs, ImmType::None, true)?;
1575        let dest = self.location_to_reg(size_out, ret, &mut gprs, ImmType::None, false, None)?;
1576
1577        if sat {
1578            // On RISC-V, if the input value is any NaN, the output of the operation is i32::MAX and thus we must
1579            // convert it to zero on our own.
1580            let end = self.assembler.get_label();
1581            let cond = self.acquire_temp_gpr().ok_or_else(|| {
1582                CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1583            })?;
1584            self.zero_location(size_out, dest)?;
1585            // if NaN -> skip convert operation
1586            self.assembler
1587                .emit_fcmp(Condition::Eq, size_in, src, src, Location::GPR(cond))?;
1588            let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
1589                CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1590            })?;
1591            self.assembler
1592                .emit_on_false_label(Location::GPR(cond), end, jmp_tmp)?;
1593            self.release_gpr(jmp_tmp);
1594            self.release_gpr(cond);
1595
1596            self.assembler
1597                .emit_fcvt(signed, size_in, src, size_out, dest)?;
1598            self.emit_label(end)?;
1599        } else {
1600            let old_fcsr = self.acquire_temp_gpr().ok_or_else(|| {
1601                CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1602            })?;
1603            self.move_location(
1604                Size::S32,
1605                Location::GPR(GPR::XZero),
1606                Location::GPR(old_fcsr),
1607            )?;
1608            self.assembler.emit_swap_fscr(old_fcsr)?;
1609            self.assembler
1610                .emit_fcvt(signed, size_in, src, size_out, dest)?;
1611            self.trap_float_conversion_errors(size_in, src, old_fcsr, &mut gprs)?;
1612            self.release_gpr(old_fcsr);
1613        }
1614
1615        if ret != dest {
1616            self.move_location(size_out, dest, ret)?;
1617        }
1618        for r in gprs {
1619            self.release_gpr(r);
1620        }
1621        for r in fprs {
1622            self.release_simd(r);
1623        }
1624        Ok(())
1625    }
1626
1627    fn convert_int_to_float(
1628        &mut self,
1629        loc: Location,
1630        size_in: Size,
1631        ret: Location,
1632        size_out: Size,
1633        signed: bool,
1634    ) -> Result<(), CompileError> {
1635        let mut gprs = vec![];
1636        let mut fprs = vec![];
1637        let src = self.location_to_reg(size_in, loc, &mut gprs, ImmType::None, true, None)?;
1638        let dest = self.location_to_fpr(size_out, ret, &mut fprs, ImmType::None, false)?;
1639        self.assembler
1640            .emit_fcvt(signed, size_in, src, size_out, dest)?;
1641        if ret != dest {
1642            self.move_location(Size::S32, dest, ret)?;
1643        }
1644        for r in gprs {
1645            self.release_gpr(r);
1646        }
1647        for r in fprs {
1648            self.release_simd(r);
1649        }
1650        Ok(())
1651    }
1652
1653    fn convert_float_to_float(
1654        &mut self,
1655        loc: Location,
1656        size_in: Size,
1657        ret: Location,
1658        size_out: Size,
1659    ) -> Result<(), CompileError> {
1660        let mut temps = vec![];
1661        let src = self.location_to_fpr(size_in, loc, &mut temps, ImmType::None, true)?;
1662        let dest = self.location_to_fpr(size_out, ret, &mut temps, ImmType::None, false)?;
1663
1664        match (size_in, size_out) {
1665            (Size::S32, Size::S64) => self
1666                .assembler
1667                .emit_fcvt(false, size_in, src, size_out, dest)?,
1668            (Size::S64, Size::S32) => self
1669                .assembler
1670                .emit_fcvt(false, size_in, src, size_out, dest)?,
1671            _ => codegen_error!("singlepass convert_float_to_float unreachable"),
1672        }
1673
1674        if ret != dest {
1675            self.move_location(size_out, dest, ret)?;
1676        }
1677        for r in temps {
1678            self.release_simd(r);
1679        }
1680        Ok(())
1681    }
1682
1683    fn trap_float_conversion_errors(
1684        &mut self,
1685        sz: Size,
1686        f: Location,
1687        old_fcsr: GPR,
1688        temps: &mut Vec<GPR>,
1689    ) -> Result<(), CompileError> {
1690        let fscr = self.acquire_temp_gpr().ok_or_else(|| {
1691            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1692        })?;
1693        self.zero_location(Size::S64, Location::GPR(fscr))?;
1694        temps.push(fscr);
1695
1696        let trap_badconv = self.assembler.get_label();
1697        let end = self.assembler.get_label();
1698
1699        self.assembler.emit_swap_fscr(fscr)?;
1700
1701        // The documentation link connected to the behavior connected to FCSR register: https://five-embeddev.com/riscv-user-isa-manual/Priv-v1.12/f.html.
1702        // clear all fflags bits except NV (1 << 4)
1703        self.assembler.emit_srl(
1704            Size::S32,
1705            Location::GPR(fscr),
1706            Location::Imm32(4),
1707            Location::GPR(fscr),
1708        )?;
1709
1710        let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
1711            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1712        })?;
1713        temps.push(jmp_tmp);
1714
1715        self.assembler
1716            .emit_on_false_label(Location::GPR(fscr), end, jmp_tmp)?;
1717
1718        // now need to check if it's overflow or NaN
1719        let cond = self.acquire_temp_gpr().ok_or_else(|| {
1720            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1721        })?;
1722        temps.push(cond);
1723
1724        self.assembler
1725            .emit_fcmp(Condition::Eq, sz, f, f, Location::GPR(cond))?;
1726        // fallthru: trap_overflow
1727        self.assembler
1728            .emit_on_false_label(Location::GPR(cond), trap_badconv, jmp_tmp)?;
1729        self.emit_illegal_op_internal(TrapCode::IntegerOverflow)?;
1730        self.emit_label(trap_badconv)?;
1731        self.emit_illegal_op_internal(TrapCode::BadConversionToInteger)?;
1732
1733        self.emit_label(end)?;
1734        self.assembler.emit_swap_fscr(old_fcsr)?;
1735
1736        Ok(())
1737    }
1738
1739    fn emit_illegal_op_internal(&mut self, trap: TrapCode) -> Result<(), CompileError> {
1740        self.assembler.emit_udf(trap as u8)
1741    }
1742
1743    fn emit_relaxed_fcmp(
1744        &mut self,
1745        c: Condition,
1746        size: Size,
1747        loc_a: Location,
1748        loc_b: Location,
1749        ret: Location,
1750    ) -> Result<(), CompileError> {
1751        // TODO: add support for immediate operations
1752        let mut fprs = vec![];
1753        let mut gprs = vec![];
1754
1755        let loc_a = self.location_to_fpr(size, loc_a, &mut fprs, ImmType::None, true)?;
1756        let loc_b = self.location_to_fpr(size, loc_b, &mut fprs, ImmType::None, true)?;
1757        let dest = self.location_to_reg(size, ret, &mut gprs, ImmType::None, false, None)?;
1758
1759        self.assembler.emit_fcmp(c, size, loc_a, loc_b, dest)?;
1760        if ret != dest {
1761            self.move_location(size, dest, ret)?;
1762        }
1763        for r in fprs {
1764            self.release_simd(r);
1765        }
1766        for r in gprs {
1767            self.release_gpr(r);
1768        }
1769        Ok(())
1770    }
1771
1772    fn emit_relaxed_fcvt_with_rounding(
1773        &mut self,
1774        rounding: RoundingMode,
1775        size: Size,
1776        loc: Location,
1777        ret: Location,
1778    ) -> Result<(), CompileError> {
1779        // For f64, values ≥ 2^52, the least significant bit of the significand represents 2,
1780        // so you can't represent odd integers or any fractional part.
1781        // Similarly, for f32, values ≥ 2^24 fulfil the same precondition.
1782
1783        let mut fprs = vec![];
1784        let tmp = self.acquire_temp_gpr().ok_or_else(|| {
1785            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1786        })?;
1787
1788        let loc = self.location_to_fpr(size, loc, &mut fprs, ImmType::None, true)?;
1789        let dest = self.location_to_fpr(size, ret, &mut fprs, ImmType::None, false)?;
1790
1791        let cond = self.acquire_temp_gpr().ok_or_else(|| {
1792            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1793        })?;
1794        let tmp1 = self.acquire_temp_simd().ok_or_else(|| {
1795            CompileError::Codegen("singlepass cannot acquire temp fpr".to_owned())
1796        })?;
1797        let tmp2 = self.acquire_temp_simd().ok_or_else(|| {
1798            CompileError::Codegen("singlepass cannot acquire temp fpr".to_owned())
1799        })?;
1800
1801        let label_after = self.get_label();
1802
1803        // Return an ArithmeticNan if either src1 or (and) src2 have a NaN value.
1804        let canonical_nan = match size {
1805            Size::S32 => CANONICAL_NAN_F32 as u64,
1806            Size::S64 => CANONICAL_NAN_F64,
1807            _ => unreachable!(),
1808        };
1809
1810        let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
1811            CompileError::Codegen("singlepass cannot acquire temp fpr".to_owned())
1812        })?;
1813
1814        self.assembler
1815            .emit_mov_imm(Location::GPR(tmp), canonical_nan as _)?;
1816        self.assembler.emit_mov(size, Location::GPR(tmp), dest)?;
1817        self.assembler
1818            .emit_fcmp(Condition::Eq, size, loc, loc, Location::GPR(cond))?;
1819        self.assembler
1820            .emit_on_false_label(Location::GPR(cond), label_after, jmp_tmp)?;
1821
1822        // TODO: refactor the constants
1823        if size == Size::S64 {
1824            self.assembler
1825                .emit_mov_imm(Location::GPR(cond), 0x4330000000000000)?;
1826            self.assembler
1827                .emit_mov(Size::S64, Location::GPR(cond), Location::SIMD(tmp1))?;
1828            self.f64_abs(loc, Location::SIMD(tmp2))?;
1829        } else {
1830            assert!(size == Size::S32);
1831            self.assembler
1832                .emit_mov_imm(Location::GPR(cond), 0x4b000000)?;
1833            self.assembler
1834                .emit_mov(Size::S32, Location::GPR(cond), Location::SIMD(tmp1))?;
1835            self.f32_abs(loc, Location::SIMD(tmp2))?;
1836        }
1837        self.emit_relaxed_fcmp(
1838            Condition::Lt,
1839            size,
1840            Location::SIMD(tmp2),
1841            Location::SIMD(tmp1),
1842            Location::GPR(cond),
1843        )?;
1844
1845        self.assembler.emit_mov(size, loc, dest)?;
1846        self.assembler
1847            .emit_on_false_label(Location::GPR(cond), label_after, jmp_tmp)?;
1848
1849        // Emit the actual conversion operation.
1850        self.assembler
1851            .emit_fcvt_with_rounding(rounding, size, loc, dest, cond)?;
1852        self.emit_label(label_after)?;
1853
1854        if ret != dest {
1855            self.move_location(size, dest, ret)?;
1856        }
1857
1858        for r in fprs {
1859            self.release_simd(r);
1860        }
1861        self.release_gpr(jmp_tmp);
1862        self.release_gpr(tmp);
1863        self.release_gpr(cond);
1864        self.release_simd(tmp1);
1865        self.release_simd(tmp2);
1866        Ok(())
1867    }
1868
1869    fn emit_unwind_op(&mut self, op: UnwindOps<GPR, FPR>) {
1870        self.unwind_ops.push((self.get_offset().0, op));
1871    }
1872}
1873
1874/// Get registers for first N function return values.
1875/// NOTE: The register set must be disjoint from pick_gpr registers!
1876/// Intentionally omit GPR::X17: it is reserved as a scratch/temporary register
1877/// and is frequently used while materializing/storing return values.
1878pub(crate) const RISCV_RETURN_VALUE_REGISTERS: [GPR; 7] = [
1879    GPR::X10,
1880    GPR::X11,
1881    GPR::X12,
1882    GPR::X13,
1883    GPR::X14,
1884    GPR::X15,
1885    GPR::X16,
1886];
1887
1888#[allow(dead_code)]
1889#[derive(PartialEq, Copy, Clone)]
1890pub(crate) enum ImmType {
1891    None,
1892    Bits12,
1893    // `add(w) X(rd) -imm` is used for subtraction with an immediate, so we need to check
1894    // the range of negated value.
1895    Bits12Subtraction,
1896    Shift32,
1897    Shift64,
1898}
1899
1900impl ImmType {
1901    pub(crate) fn compatible_imm(&self, imm: i64) -> bool {
1902        match self {
1903            ImmType::None => false,
1904            ImmType::Bits12 => (-0x800..0x800).contains(&imm),
1905            ImmType::Bits12Subtraction => (-0x801..0x801).contains(&imm),
1906            ImmType::Shift32 => (0..32).contains(&imm),
1907            ImmType::Shift64 => (0..64).contains(&imm),
1908        }
1909    }
1910}
1911
1912impl Machine for MachineRiscv {
1913    type GPR = GPR;
1914    type SIMD = FPR;
1915
1916    const STACK_ALIGNMENT: usize = 16;
1917
1918    fn assembler_get_offset(&self) -> Offset {
1919        self.assembler.get_offset()
1920    }
1921
1922    fn get_vmctx_reg(&self) -> Self::GPR {
1923        // Must be a callee-save register.
1924        GPR::X27
1925    }
1926
1927    fn pick_gpr(&self) -> Option<Self::GPR> {
1928        use GPR::*;
1929        // Ignore X28 as we use it as a scratch register
1930        static REGS: &[GPR] = &[X5, X6, X7, X29, X30, X31];
1931        for r in REGS {
1932            if !self.used_gprs_contains(r) {
1933                return Some(*r);
1934            }
1935        }
1936        None
1937    }
1938
1939    fn pick_temp_gpr(&self) -> Option<GPR> {
1940        use GPR::*;
1941        // Reserve a few registers for the first locals of a function!
1942        // These registers below are also used for argument passing, so they must
1943        // be saved/moved in the function prologue before being used as temps.
1944        static REGS: &[GPR] = &[X17, X16, X15, X14, X13, X12, X11, X10];
1945        for r in REGS {
1946            if !self.used_gprs_contains(r) {
1947                return Some(*r);
1948            }
1949        }
1950        None
1951    }
1952
1953    fn get_used_gprs(&self) -> Vec<Self::GPR> {
1954        GPR::iterator()
1955            .filter(|x| self.used_gprs.contains(x.into_index()))
1956            .cloned()
1957            .collect()
1958    }
1959
1960    fn get_used_simd(&self) -> Vec<Self::SIMD> {
1961        FPR::iterator()
1962            .filter(|x| self.used_fprs.contains(x.into_index()))
1963            .cloned()
1964            .collect()
1965    }
1966
1967    fn acquire_temp_gpr(&mut self) -> Option<Self::GPR> {
1968        let gpr = self.pick_temp_gpr();
1969        if let Some(x) = gpr {
1970            self.used_gprs_insert(x);
1971        }
1972        gpr
1973    }
1974
1975    fn release_gpr(&mut self, gpr: Self::GPR) {
1976        assert!(self.used_gprs_remove(&gpr));
1977    }
1978
1979    fn reserve_unused_temp_gpr(&mut self, gpr: Self::GPR) -> Self::GPR {
1980        assert!(!self.used_gprs_contains(&gpr));
1981        self.used_gprs_insert(gpr);
1982        gpr
1983    }
1984
1985    fn reserve_gpr(&mut self, gpr: Self::GPR) {
1986        self.used_gprs_insert(gpr);
1987    }
1988
1989    fn push_used_gpr(&mut self, used_gprs: &[Self::GPR]) -> Result<usize, CompileError> {
1990        for r in used_gprs.iter() {
1991            self.assembler.emit_push(Size::S64, Location::GPR(*r))?;
1992        }
1993        Ok(used_gprs.len() * 16)
1994    }
1995
1996    fn pop_used_gpr(&mut self, used_gprs: &[Self::GPR]) -> Result<(), CompileError> {
1997        for r in used_gprs.iter().rev() {
1998            self.emit_pop(Size::S64, Location::GPR(*r))?;
1999        }
2000        Ok(())
2001    }
2002
2003    fn pick_simd(&self) -> Option<Self::SIMD> {
2004        use FPR::*;
2005        static REGS: &[FPR] = &[F0, F1, F2, F3, F4, F5, F6, F7];
2006        for r in REGS {
2007            if !self.used_fp_contains(r) {
2008                return Some(*r);
2009            }
2010        }
2011        None
2012    }
2013
2014    fn pick_temp_simd(&self) -> Option<FPR> {
2015        use FPR::*;
2016        static REGS: &[FPR] = &[F28, F29, F31];
2017        for r in REGS {
2018            if !self.used_fp_contains(r) {
2019                return Some(*r);
2020            }
2021        }
2022        None
2023    }
2024
2025    fn acquire_temp_simd(&mut self) -> Option<Self::SIMD> {
2026        let fpr = self.pick_temp_simd();
2027        if let Some(x) = fpr {
2028            self.used_fprs_insert(x);
2029        }
2030        fpr
2031    }
2032
2033    fn reserve_simd(&mut self, fpr: Self::SIMD) {
2034        self.used_fprs_insert(fpr);
2035    }
2036
2037    fn release_simd(&mut self, fpr: Self::SIMD) {
2038        assert!(self.used_fprs_remove(&fpr));
2039    }
2040
2041    fn push_used_simd(&mut self, used_neons: &[Self::SIMD]) -> Result<usize, CompileError> {
2042        let stack_adjust = (used_neons.len() * 8) as u32;
2043        self.extend_stack(stack_adjust)?;
2044
2045        for (i, r) in used_neons.iter().enumerate() {
2046            self.assembler.emit_sd(
2047                Size::S64,
2048                Location::SIMD(*r),
2049                Location::Memory(GPR::Sp, (i * 8) as i32),
2050            )?;
2051        }
2052        Ok(stack_adjust as usize)
2053    }
2054
2055    fn pop_used_simd(&mut self, used_neons: &[Self::SIMD]) -> Result<(), CompileError> {
2056        for (i, r) in used_neons.iter().enumerate() {
2057            self.assembler.emit_ld(
2058                Size::S64,
2059                false,
2060                Location::SIMD(*r),
2061                Location::Memory(GPR::Sp, (i * 8) as i32),
2062            )?;
2063        }
2064        let stack_adjust = (used_neons.len() * 8) as u32;
2065        self.assembler.emit_add(
2066            Size::S64,
2067            Location::GPR(GPR::Sp),
2068            Location::Imm64(stack_adjust as _),
2069            Location::GPR(GPR::Sp),
2070        )
2071    }
2072
2073    fn set_srcloc(&mut self, offset: u32) {
2074        self.src_loc = offset;
2075    }
2076
2077    fn mark_address_range_with_trap_code(&mut self, code: TrapCode, begin: usize, end: usize) {
2078        for i in begin..end {
2079            self.trap_table.offset_to_code.insert(i, code);
2080        }
2081        self.mark_instruction_address_end(begin);
2082    }
2083
2084    fn mark_address_with_trap_code(&mut self, code: TrapCode) {
2085        let offset = self.assembler.get_offset().0;
2086        self.trap_table.offset_to_code.insert(offset, code);
2087        self.mark_instruction_address_end(offset);
2088    }
2089
2090    fn mark_instruction_with_trap_code(&mut self, code: TrapCode) -> usize {
2091        let offset = self.assembler.get_offset().0;
2092        self.trap_table.offset_to_code.insert(offset, code);
2093        offset
2094    }
2095
2096    fn mark_instruction_address_end(&mut self, begin: usize) {
2097        self.instructions_address_map.push(InstructionAddressMap {
2098            srcloc: SourceLoc::new(self.src_loc),
2099            code_offset: begin,
2100            code_len: self.assembler.get_offset().0 - begin,
2101        });
2102    }
2103
2104    fn insert_stackoverflow(&mut self) {
2105        let offset = 0;
2106        self.trap_table
2107            .offset_to_code
2108            .insert(offset, TrapCode::StackOverflow);
2109        self.mark_instruction_address_end(offset);
2110    }
2111
2112    fn collect_trap_information(&self) -> Vec<TrapInformation> {
2113        self.trap_table
2114            .offset_to_code
2115            .clone()
2116            .into_iter()
2117            .map(|(offset, code)| TrapInformation {
2118                code_offset: offset as u32,
2119                trap_code: code,
2120            })
2121            .collect()
2122    }
2123
2124    fn instructions_address_map(&self) -> Vec<InstructionAddressMap> {
2125        self.instructions_address_map.clone()
2126    }
2127
2128    fn local_on_stack(&mut self, stack_offset: i32) -> Location {
2129        Location::Memory(GPR::Fp, -stack_offset)
2130    }
2131
2132    fn extend_stack(&mut self, delta_stack_offset: u32) -> Result<(), CompileError> {
2133        let delta = if ImmType::Bits12Subtraction.compatible_imm(delta_stack_offset as _) {
2134            Location::Imm64(delta_stack_offset as _)
2135        } else {
2136            self.assembler
2137                .emit_mov_imm(Location::GPR(SCRATCH_REG), delta_stack_offset as _)?;
2138            Location::GPR(SCRATCH_REG)
2139        };
2140        self.assembler.emit_sub(
2141            Size::S64,
2142            Location::GPR(GPR::Sp),
2143            delta,
2144            Location::GPR(GPR::Sp),
2145        )
2146    }
2147
2148    fn truncate_stack(&mut self, delta_stack_offset: u32) -> Result<(), CompileError> {
2149        let delta = if ImmType::Bits12.compatible_imm(delta_stack_offset as _) {
2150            Location::Imm64(delta_stack_offset as _)
2151        } else {
2152            self.assembler
2153                .emit_mov_imm(Location::GPR(SCRATCH_REG), delta_stack_offset as _)?;
2154            Location::GPR(SCRATCH_REG)
2155        };
2156        self.assembler.emit_add(
2157            Size::S64,
2158            Location::GPR(GPR::Sp),
2159            delta,
2160            Location::GPR(GPR::Sp),
2161        )
2162    }
2163
2164    fn zero_location(&mut self, size: Size, location: Location) -> Result<(), CompileError> {
2165        self.move_location(size, Location::GPR(GPR::XZero), location)
2166    }
2167
2168    fn local_pointer(&self) -> Self::GPR {
2169        GPR::Fp
2170    }
2171
2172    fn move_location_for_native(
2173        &mut self,
2174        _size: Size,
2175        loc: Location,
2176        dest: Location,
2177    ) -> Result<(), CompileError> {
2178        match loc {
2179            Location::Imm64(_)
2180            | Location::Imm32(_)
2181            | Location::Imm8(_)
2182            | Location::Memory(_, _) => {
2183                self.move_location(Size::S64, loc, Location::GPR(SCRATCH_REG))?;
2184                self.move_location(Size::S64, Location::GPR(SCRATCH_REG), dest)
2185            }
2186            _ => self.move_location(Size::S64, loc, dest),
2187        }
2188    }
2189
2190    fn is_local_on_stack(&self, idx: usize) -> bool {
2191        idx > 9
2192    }
2193
2194    fn get_local_location(&self, idx: usize, callee_saved_regs_size: usize) -> Location {
2195        // Use callee-saved registers for the first locals.
2196        match idx {
2197            0 => Location::GPR(GPR::X9),
2198            1 => Location::GPR(GPR::X18),
2199            2 => Location::GPR(GPR::X19),
2200            3 => Location::GPR(GPR::X20),
2201            4 => Location::GPR(GPR::X21),
2202            5 => Location::GPR(GPR::X22),
2203            6 => Location::GPR(GPR::X23),
2204            7 => Location::GPR(GPR::X24),
2205            8 => Location::GPR(GPR::X25),
2206            9 => Location::GPR(GPR::X26),
2207            _ => Location::Memory(GPR::Fp, -(((idx - 9) * 8 + callee_saved_regs_size) as i32)),
2208        }
2209    }
2210
2211    fn move_local(&mut self, stack_offset: i32, location: Location) -> Result<(), CompileError> {
2212        self.move_location(
2213            Size::S64,
2214            location,
2215            Location::Memory(GPR::Fp, -stack_offset),
2216        )?;
2217
2218        match location {
2219            Location::GPR(x) => self.emit_unwind_op(UnwindOps::SaveRegister {
2220                reg: UnwindRegister::GPR(x),
2221                bp_neg_offset: stack_offset,
2222            }),
2223            Location::SIMD(x) => self.emit_unwind_op(UnwindOps::SaveRegister {
2224                reg: UnwindRegister::FPR(x),
2225                bp_neg_offset: stack_offset,
2226            }),
2227            _ => (),
2228        }
2229        Ok(())
2230    }
2231
2232    fn get_param_registers(&self) -> &'static [Self::GPR] {
2233        &[
2234            GPR::X10,
2235            GPR::X11,
2236            GPR::X12,
2237            GPR::X13,
2238            GPR::X14,
2239            GPR::X15,
2240            GPR::X16,
2241            GPR::X17,
2242        ]
2243    }
2244
2245    fn get_param_location(
2246        &self,
2247        idx: usize,
2248        _sz: Size,
2249        stack_args: &mut usize,
2250        _calling_convention: CallingConvention,
2251    ) -> Location {
2252        let register_params: &[GPR] = self.get_param_registers();
2253        if let Some(reg) = register_params.get(idx) {
2254            Location::GPR(*reg)
2255        } else {
2256            let loc = Location::Memory(GPR::Sp, *stack_args as i32);
2257            *stack_args += 8;
2258            loc
2259        }
2260    }
2261
2262    fn get_call_param_location(
2263        &self,
2264        return_slots: usize,
2265        idx: usize,
2266        _sz: Size,
2267        stack_args: &mut usize,
2268        _calling_convention: CallingConvention,
2269    ) -> Location {
2270        let return_values_memory_size =
2271            8 * return_slots.saturating_sub(RISCV_RETURN_VALUE_REGISTERS.len()) as i32;
2272        self.get_param_registers().get(idx).map_or_else(
2273            || {
2274                let loc =
2275                    Location::Memory(GPR::Fp, 16 + return_values_memory_size + *stack_args as i32);
2276                *stack_args += 8;
2277                loc
2278            },
2279            |reg| Location::GPR(*reg),
2280        )
2281    }
2282
2283    fn get_simple_param_location(&self, idx: usize) -> Self::GPR {
2284        self.get_param_registers()[idx]
2285    }
2286
2287    fn adjust_gpr_param_location(
2288        &mut self,
2289        register: Self::GPR,
2290        size: Size,
2291    ) -> Result<(), CompileError> {
2292        // https://five-embeddev.com/riscv-user-isa-manual/Priv-v1.12/rv64.html
2293        // > The compiler and calling convention maintain an invariant that all 32-bit values are held in a sign-extended format in 64-bit registers.
2294        // > Even 32-bit unsigned integers extend bit 31 into bits 63 through 32. Consequently, conversion between unsigned and signed 32-bit integers
2295        // > is a no-op, as is conversion from a signed 32-bit integer to a signed 64-bit integer.
2296        match size {
2297            Size::S64 => Ok(()),
2298            Size::S32 => self.assembler.emit_extend(
2299                Size::S32,
2300                true,
2301                Location::GPR(register),
2302                Location::GPR(register),
2303            ),
2304            Size::S8 | Size::S16 => {
2305                codegen_error!("singlepass adjust_gpr_param_location unreachable")
2306            }
2307        }
2308    }
2309
2310    fn get_return_value_location(
2311        &self,
2312        idx: usize,
2313        stack_location: &mut usize,
2314    ) -> AbstractLocation<Self::GPR, Self::SIMD> {
2315        RISCV_RETURN_VALUE_REGISTERS.get(idx).map_or_else(
2316            || {
2317                let loc = Location::Memory(GPR::Sp, *stack_location as i32);
2318                *stack_location += 8;
2319                loc
2320            },
2321            |reg| Location::GPR(*reg),
2322        )
2323    }
2324
2325    fn get_call_return_value_location(
2326        &self,
2327        idx: usize,
2328    ) -> AbstractLocation<Self::GPR, Self::SIMD> {
2329        RISCV_RETURN_VALUE_REGISTERS.get(idx).map_or_else(
2330            || {
2331                Location::Memory(
2332                    GPR::Fp,
2333                    (16 + (idx - RISCV_RETURN_VALUE_REGISTERS.len()) * 8) as i32,
2334                )
2335            },
2336            |reg| Location::GPR(*reg),
2337        )
2338    }
2339
2340    fn move_location(
2341        &mut self,
2342        size: Size,
2343        source: Location,
2344        dest: Location,
2345    ) -> Result<(), CompileError> {
2346        match (source, dest) {
2347            (Location::GPR(_), Location::GPR(_)) => self.assembler.emit_mov(size, source, dest),
2348            (Location::Imm32(_) | Location::Imm64(_), Location::GPR(_)) => self
2349                .assembler
2350                .emit_mov_imm(dest, source.imm_value_scalar().unwrap()),
2351            (Location::GPR(_), Location::Memory(addr, offset)) => {
2352                let addr = if ImmType::Bits12.compatible_imm(offset as _) {
2353                    dest
2354                } else {
2355                    self.assembler
2356                        .emit_mov_imm(Location::GPR(SCRATCH_REG), offset as _)?;
2357                    self.assembler.emit_add(
2358                        Size::S64,
2359                        Location::GPR(addr),
2360                        Location::GPR(SCRATCH_REG),
2361                        Location::GPR(SCRATCH_REG),
2362                    )?;
2363                    Location::Memory(SCRATCH_REG, 0)
2364                };
2365                self.assembler.emit_sd(size, source, addr)
2366            }
2367            (Location::SIMD(_), Location::Memory(..)) => {
2368                let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2369                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2370                })?;
2371                self.move_location(size, source, Location::GPR(tmp))?;
2372                self.move_location(size, Location::GPR(tmp), dest)?;
2373                self.release_gpr(tmp);
2374                Ok(())
2375            }
2376            (Location::Memory(addr, offset), Location::GPR(_)) => {
2377                let addr = if ImmType::Bits12.compatible_imm(offset as _) {
2378                    source
2379                } else {
2380                    self.assembler
2381                        .emit_mov_imm(Location::GPR(SCRATCH_REG), offset as _)?;
2382                    self.assembler.emit_add(
2383                        Size::S64,
2384                        Location::GPR(addr),
2385                        Location::GPR(SCRATCH_REG),
2386                        Location::GPR(SCRATCH_REG),
2387                    )?;
2388                    Location::Memory(SCRATCH_REG, 0)
2389                };
2390                self.assembler.emit_ld(size, false, dest, addr)
2391            }
2392            (Location::GPR(_), Location::SIMD(_)) => self.assembler.emit_mov(size, source, dest),
2393            (Location::SIMD(_), Location::GPR(_)) => self.assembler.emit_mov(size, source, dest),
2394            (Location::SIMD(_), Location::SIMD(_)) => self.assembler.emit_mov(size, source, dest),
2395            _ => todo!("unsupported move: {size:?} {source:?} {dest:?}"),
2396        }
2397    }
2398
2399    fn move_location_extend(
2400        &mut self,
2401        size_val: Size,
2402        signed: bool,
2403        source: Location,
2404        size_op: Size,
2405        dest: Location,
2406    ) -> Result<(), CompileError> {
2407        if size_op != Size::S64 {
2408            codegen_error!("singlepass move_location_extend unreachable");
2409        }
2410        let mut temps = vec![];
2411        let dst = self.location_to_reg(size_op, dest, &mut temps, ImmType::None, false, None)?;
2412        let src = match (size_val, signed, source) {
2413            (Size::S64, _, _) => source,
2414            (_, _, Location::GPR(_)) => {
2415                self.assembler.emit_extend(size_val, signed, source, dst)?;
2416                dst
2417            }
2418            (_, _, Location::Memory(_, _)) => {
2419                self.assembler.emit_ld(size_val, signed, dst, source)?;
2420                dst
2421            }
2422            _ => codegen_error!(
2423                "singlepass can't emit move_location_extend {:?} {:?} {:?} => {:?} {:?}",
2424                size_val,
2425                signed,
2426                source,
2427                size_op,
2428                dest
2429            ),
2430        };
2431        if src != dst {
2432            self.move_location(size_op, src, dst)?;
2433        }
2434        if dst != dest {
2435            self.move_location(size_op, dst, dest)?;
2436        }
2437        for r in temps {
2438            self.release_gpr(r);
2439        }
2440        Ok(())
2441    }
2442
2443    fn init_stack_loc(
2444        &mut self,
2445        init_stack_loc_cnt: u64,
2446        last_stack_loc: Location,
2447    ) -> Result<(), CompileError> {
2448        let label = self.assembler.get_label();
2449        let mut temps = vec![];
2450        let dest = self.acquire_temp_gpr().ok_or_else(|| {
2451            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2452        })?;
2453        temps.push(dest);
2454        let cnt = self.location_to_reg(
2455            Size::S64,
2456            Location::Imm64(init_stack_loc_cnt),
2457            &mut temps,
2458            ImmType::None,
2459            true,
2460            None,
2461        )?;
2462        match last_stack_loc {
2463            Location::GPR(_) => codegen_error!("singlepass init_stack_loc unreachable"),
2464            Location::SIMD(_) => codegen_error!("singlepass init_stack_loc unreachable"),
2465            Location::Memory(reg, offset) => {
2466                if ImmType::Bits12.compatible_imm(offset as _) {
2467                    self.assembler.emit_add(
2468                        Size::S64,
2469                        Location::GPR(reg),
2470                        Location::Imm64(offset as _),
2471                        Location::GPR(dest),
2472                    )?;
2473                } else {
2474                    let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2475                        CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2476                    })?;
2477                    self.assembler
2478                        .emit_mov_imm(Location::GPR(tmp), offset as _)?;
2479                    self.assembler.emit_add(
2480                        Size::S64,
2481                        Location::GPR(reg),
2482                        Location::GPR(tmp),
2483                        Location::GPR(dest),
2484                    )?;
2485                    temps.push(tmp);
2486                }
2487            }
2488            _ => codegen_error!("singlepass can't emit init_stack_loc {:?}", last_stack_loc),
2489        };
2490        self.assembler.emit_label(label)?;
2491        self.assembler.emit_sd(
2492            Size::S64,
2493            Location::GPR(GPR::XZero),
2494            Location::Memory(dest, 0),
2495        )?;
2496        self.assembler
2497            .emit_sub(Size::S64, cnt, Location::Imm64(1), cnt)?;
2498        self.assembler.emit_add(
2499            Size::S64,
2500            Location::GPR(dest),
2501            Location::Imm64(8),
2502            Location::GPR(dest),
2503        )?;
2504        let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
2505            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2506        })?;
2507        temps.push(jmp_tmp);
2508        self.assembler.emit_on_true_label(cnt, label, jmp_tmp)?;
2509        for r in temps {
2510            self.release_gpr(r);
2511        }
2512        Ok(())
2513    }
2514
2515    fn restore_saved_area(&mut self, saved_area_offset: i32) -> Result<(), CompileError> {
2516        self.assembler.emit_sub(
2517            Size::S64,
2518            Location::GPR(GPR::Fp),
2519            Location::Imm64(saved_area_offset as _),
2520            Location::GPR(GPR::Sp),
2521        )
2522    }
2523
2524    fn pop_location(&mut self, location: Location) -> Result<(), CompileError> {
2525        self.emit_pop(Size::S64, location)
2526    }
2527
2528    fn assembler_finalize(
2529        self,
2530        assembly_comments: HashMap<usize, AssemblyComment>,
2531    ) -> Result<FinalizedAssembly, CompileError> {
2532        Ok(FinalizedAssembly {
2533            body: self.assembler.finalize().map_err(|e| {
2534                CompileError::Codegen(format!("Assembler failed finalization with: {e:?}"))
2535            })?,
2536            assembly_comments,
2537        })
2538    }
2539
2540    fn get_offset(&self) -> Offset {
2541        self.assembler.get_offset()
2542    }
2543
2544    fn finalize_function(&mut self) -> Result<(), CompileError> {
2545        self.assembler.finalize_function()?;
2546        Ok(())
2547    }
2548
2549    fn emit_function_prolog(&mut self) -> Result<(), CompileError> {
2550        self.assembler.emit_sub(
2551            Size::S64,
2552            Location::GPR(GPR::Sp),
2553            Location::Imm64(16),
2554            Location::GPR(GPR::Sp),
2555        )?;
2556        self.emit_unwind_op(UnwindOps::SubtractFP { up_to_sp: 16 });
2557
2558        self.assembler.emit_sd(
2559            Size::S64,
2560            Location::GPR(GPR::X1), // return address register
2561            Location::Memory(GPR::Sp, 8),
2562        )?;
2563        self.assembler.emit_sd(
2564            Size::S64,
2565            Location::GPR(GPR::Fp),
2566            Location::Memory(GPR::Sp, 0),
2567        )?;
2568        self.emit_unwind_op(UnwindOps::SaveRegister {
2569            reg: UnwindRegister::GPR(GPR::X1),
2570            bp_neg_offset: 8,
2571        });
2572        self.emit_unwind_op(UnwindOps::SaveRegister {
2573            reg: UnwindRegister::GPR(GPR::Fp),
2574            bp_neg_offset: 16,
2575        });
2576
2577        self.assembler
2578            .emit_mov(Size::S64, Location::GPR(GPR::Sp), Location::GPR(GPR::Fp))?;
2579        self.emit_unwind_op(UnwindOps::DefineNewFrame);
2580        Ok(())
2581    }
2582
2583    fn emit_function_epilog(&mut self) -> Result<(), CompileError> {
2584        self.assembler
2585            .emit_mov(Size::S64, Location::GPR(GPR::Fp), Location::GPR(GPR::Sp))?;
2586        self.assembler.emit_ld(
2587            Size::S64,
2588            false,
2589            Location::GPR(GPR::X1), // return address register
2590            Location::Memory(GPR::Sp, 8),
2591        )?;
2592        self.assembler.emit_ld(
2593            Size::S64,
2594            false,
2595            Location::GPR(GPR::Fp),
2596            Location::Memory(GPR::Sp, 0),
2597        )?;
2598        self.assembler.emit_add(
2599            Size::S64,
2600            Location::GPR(GPR::Sp),
2601            Location::Imm64(16),
2602            Location::GPR(GPR::Sp),
2603        )?;
2604
2605        Ok(())
2606    }
2607
2608    fn emit_function_return_float(&mut self) -> Result<(), CompileError> {
2609        self.assembler
2610            .emit_mov(Size::S64, Location::GPR(GPR::X10), Location::SIMD(FPR::F10))
2611    }
2612
2613    fn canonicalize_nan(
2614        &mut self,
2615        sz: Size,
2616        input: Location,
2617        output: Location,
2618    ) -> Result<(), CompileError> {
2619        let mut temps = vec![];
2620        // use FMAX (input, input) => output to automatically normalize the NaN
2621        match (sz, input, output) {
2622            (Size::S32, Location::SIMD(_), Location::SIMD(_)) => {
2623                self.assembler.emit_fmax(sz, input, input, output)?;
2624            }
2625            (Size::S64, Location::SIMD(_), Location::SIMD(_)) => {
2626                self.assembler.emit_fmax(sz, input, input, output)?;
2627            }
2628            (Size::S32, Location::SIMD(_), _) | (Size::S64, Location::SIMD(_), _) => {
2629                let tmp = self.location_to_fpr(sz, output, &mut temps, ImmType::None, false)?;
2630                self.assembler.emit_fmax(sz, input, input, tmp)?;
2631                self.move_location(sz, tmp, output)?;
2632            }
2633            (Size::S32, Location::Memory(_, _), _) | (Size::S64, Location::Memory(_, _), _) => {
2634                let src = self.location_to_fpr(sz, input, &mut temps, ImmType::None, true)?;
2635                let tmp = self.location_to_fpr(sz, output, &mut temps, ImmType::None, false)?;
2636                self.assembler.emit_fmax(sz, src, src, tmp)?;
2637                if tmp != output {
2638                    self.move_location(sz, tmp, output)?;
2639                }
2640            }
2641            _ => codegen_error!(
2642                "singlepass can't emit canonicalize_nan {:?} {:?} {:?}",
2643                sz,
2644                input,
2645                output
2646            ),
2647        }
2648
2649        for r in temps {
2650            self.release_simd(r);
2651        }
2652        Ok(())
2653    }
2654
2655    fn emit_illegal_op(&mut self, trap: TrapCode) -> Result<(), CompileError> {
2656        let offset = self.assembler.get_offset().0;
2657        self.assembler.emit_udf(trap as u8)?;
2658        self.mark_instruction_address_end(offset);
2659        Ok(())
2660    }
2661
2662    fn get_label(&mut self) -> Label {
2663        self.assembler.new_dynamic_label()
2664    }
2665
2666    fn emit_label(&mut self, label: Label) -> Result<(), CompileError> {
2667        self.assembler.emit_label(label)
2668    }
2669
2670    fn get_gpr_for_call(&self) -> Self::GPR {
2671        GPR::X1
2672    }
2673
2674    fn emit_call_register(&mut self, register: Self::GPR) -> Result<(), CompileError> {
2675        self.assembler.emit_call_register(register)
2676    }
2677
2678    fn emit_call_label(&mut self, label: Label) -> Result<(), CompileError> {
2679        self.assembler.emit_call_label(label)
2680    }
2681
2682    fn arch_emit_indirect_call_with_trampoline(
2683        &mut self,
2684        _location: Location,
2685    ) -> Result<(), CompileError> {
2686        codegen_error!("singlepass arch_emit_indirect_call_with_trampoline unimplemented")
2687    }
2688
2689    fn emit_call_location(&mut self, location: Location) -> Result<(), CompileError> {
2690        let mut temps = vec![];
2691        let loc = self.location_to_reg(
2692            Size::S64,
2693            location,
2694            &mut temps,
2695            ImmType::None,
2696            true,
2697            Some(self.get_gpr_for_call()),
2698        )?;
2699        match loc {
2700            Location::GPR(reg) => self.assembler.emit_call_register(reg),
2701            _ => codegen_error!("singlepass can't emit CALL Location"),
2702        }?;
2703        for r in temps {
2704            self.release_gpr(r);
2705        }
2706        Ok(())
2707    }
2708
2709    fn emit_debug_breakpoint(&mut self) -> Result<(), CompileError> {
2710        self.assembler.emit_brk()
2711    }
2712
2713    fn location_add(
2714        &mut self,
2715        size: Size,
2716        source: Location,
2717        dest: Location,
2718        _flags: bool,
2719    ) -> Result<(), CompileError> {
2720        let mut temps = vec![];
2721        let src = self.location_to_reg(size, source, &mut temps, ImmType::Bits12, true, None)?;
2722        let dst = self.location_to_reg(size, dest, &mut temps, ImmType::None, true, None)?;
2723        self.assembler.emit_add(size, dst, src, dst)?;
2724        if dst != dest {
2725            self.move_location(size, dst, dest)?;
2726        }
2727        for r in temps {
2728            self.release_gpr(r);
2729        }
2730        Ok(())
2731    }
2732
2733    fn location_cmp(
2734        &mut self,
2735        _size: Size,
2736        _source: Location,
2737        _dest: Location,
2738    ) -> Result<(), CompileError> {
2739        codegen_error!("singlepass location_cmp not implemented")
2740    }
2741
2742    fn jmp_unconditional(&mut self, label: Label) -> Result<(), CompileError> {
2743        let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2744            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2745        })?;
2746        self.assembler.emit_j_label(label, Some(tmp))?;
2747        self.release_gpr(tmp);
2748        Ok(())
2749    }
2750
2751    fn jmp_on_condition(
2752        &mut self,
2753        cond: UnsignedCondition,
2754        size: Size,
2755        loc_a: AbstractLocation<Self::GPR, Self::SIMD>,
2756        loc_b: AbstractLocation<Self::GPR, Self::SIMD>,
2757        label: Label,
2758    ) -> Result<(), CompileError> {
2759        let c = match cond {
2760            UnsignedCondition::Equal => Condition::Eq,
2761            UnsignedCondition::NotEqual => Condition::Ne,
2762            UnsignedCondition::Above => Condition::Gtu,
2763            UnsignedCondition::AboveEqual => Condition::Geu,
2764            UnsignedCondition::Below => Condition::Ltu,
2765            UnsignedCondition::BelowEqual => Condition::Leu,
2766        };
2767
2768        let mut temps = vec![];
2769        let loc_a = self.location_to_reg(size, loc_a, &mut temps, ImmType::None, true, None)?;
2770        let loc_b = self.location_to_reg(size, loc_b, &mut temps, ImmType::None, true, None)?;
2771
2772        if size != Size::S64 {
2773            self.assembler.emit_extend(size, false, loc_a, loc_a)?;
2774            self.assembler.emit_extend(size, false, loc_b, loc_b)?;
2775        }
2776
2777        let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2778            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2779        })?;
2780        temps.push(tmp);
2781        self.assembler
2782            .emit_jmp_on_condition(c, loc_a, loc_b, label, tmp)?;
2783
2784        for r in temps {
2785            self.release_gpr(r);
2786        }
2787        Ok(())
2788    }
2789
2790    fn emit_jmp_to_jumptable(&mut self, label: Label, cond: Location) -> Result<(), CompileError> {
2791        let tmp1 = self.acquire_temp_gpr().ok_or_else(|| {
2792            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2793        })?;
2794        let tmp2 = self.acquire_temp_gpr().ok_or_else(|| {
2795            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2796        })?;
2797
2798        self.assembler.emit_load_label(tmp1, label)?;
2799        self.move_location(Size::S32, cond, Location::GPR(tmp2))?;
2800
2801        // Multiply by 8, for RISC-V, a jump is made via 2 instructions:
2802        // auipc and j. This way we can navigate greater code regions.
2803        self.assembler.emit_sll(
2804            Size::S32,
2805            Location::GPR(tmp2),
2806            Location::Imm32(3),
2807            Location::GPR(tmp2),
2808        )?;
2809        self.assembler.emit_add(
2810            Size::S64,
2811            Location::GPR(tmp1),
2812            Location::GPR(tmp2),
2813            Location::GPR(tmp2),
2814        )?;
2815        self.assembler.emit_j_register(tmp2)?;
2816        self.release_gpr(tmp2);
2817        self.release_gpr(tmp1);
2818        Ok(())
2819    }
2820
2821    fn align_for_loop(&mut self) -> Result<(), CompileError> {
2822        // nothing to do on RISC-V
2823        Ok(())
2824    }
2825
2826    fn emit_ret(&mut self) -> Result<(), CompileError> {
2827        self.assembler.emit_ret()
2828    }
2829
2830    fn emit_push(&mut self, size: Size, loc: Location) -> Result<(), CompileError> {
2831        self.assembler.emit_push(size, loc)
2832    }
2833
2834    fn emit_pop(&mut self, size: Size, loc: Location) -> Result<(), CompileError> {
2835        self.assembler.emit_pop(size, loc)
2836    }
2837
2838    fn emit_relaxed_mov(
2839        &mut self,
2840        sz: Size,
2841        src: Location,
2842        dst: Location,
2843    ) -> Result<(), CompileError> {
2844        self.emit_relaxed_binop(Assembler::emit_mov, sz, src, dst)
2845    }
2846
2847    fn emit_relaxed_cmp(
2848        &mut self,
2849        _sz: Size,
2850        _src: Location,
2851        _dst: Location,
2852    ) -> Result<(), CompileError> {
2853        todo!();
2854    }
2855
2856    fn emit_memory_fence(&mut self) -> Result<(), CompileError> {
2857        self.assembler.emit_rwfence()
2858    }
2859
2860    fn emit_relaxed_sign_extension(
2861        &mut self,
2862        sz_src: Size,
2863        src: Location,
2864        sz_dst: Size,
2865        dst: Location,
2866    ) -> Result<(), CompileError> {
2867        let mut temps = vec![];
2868
2869        match (src, dst) {
2870            (Location::Memory(reg, offset), Location::GPR(_)) => {
2871                let src = if ImmType::Bits12.compatible_imm(offset as _) {
2872                    src
2873                } else {
2874                    let tmp =
2875                        self.location_to_reg(sz_src, src, &mut temps, ImmType::None, true, None)?;
2876                    self.assembler.emit_mov_imm(tmp, offset as _)?;
2877                    self.assembler
2878                        .emit_add(Size::S64, Location::GPR(reg), tmp, tmp)?;
2879                    let Location::GPR(tmp) = tmp else {
2880                        unreachable!()
2881                    };
2882                    Location::Memory(tmp, 0)
2883                };
2884                self.assembler.emit_ld(sz_src, true, dst, src)?;
2885            }
2886            _ => {
2887                let src =
2888                    self.location_to_reg(sz_src, src, &mut temps, ImmType::None, true, None)?;
2889                let dest =
2890                    self.location_to_reg(sz_dst, dst, &mut temps, ImmType::None, false, None)?;
2891                self.assembler.emit_extend(sz_src, true, src, dest)?;
2892                if dst != dest {
2893                    self.move_location(sz_dst, dest, dst)?;
2894                }
2895            }
2896        }
2897
2898        for r in temps {
2899            self.release_gpr(r);
2900        }
2901
2902        Ok(())
2903    }
2904
2905    fn emit_imul_imm32(
2906        &mut self,
2907        size: Size,
2908        imm32: u32,
2909        gpr: Self::GPR,
2910    ) -> Result<(), CompileError> {
2911        let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2912            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2913        })?;
2914        self.assembler
2915            .emit_mov_imm(Location::GPR(tmp), imm32 as _)?;
2916        self.assembler.emit_mul(
2917            size,
2918            Location::GPR(gpr),
2919            Location::GPR(tmp),
2920            Location::GPR(gpr),
2921        )?;
2922        self.release_gpr(tmp);
2923        Ok(())
2924    }
2925
2926    fn emit_binop_add32(
2927        &mut self,
2928        loc_a: Location,
2929        loc_b: Location,
2930        ret: Location,
2931    ) -> Result<(), CompileError> {
2932        self.emit_relaxed_binop3(
2933            Assembler::emit_add,
2934            Size::S32,
2935            loc_a,
2936            loc_b,
2937            ret,
2938            ImmType::Bits12,
2939        )
2940    }
2941
2942    fn emit_binop_sub32(
2943        &mut self,
2944        loc_a: Location,
2945        loc_b: Location,
2946        ret: Location,
2947    ) -> Result<(), CompileError> {
2948        self.emit_relaxed_binop3(
2949            Assembler::emit_sub,
2950            Size::S32,
2951            loc_a,
2952            loc_b,
2953            ret,
2954            ImmType::Bits12Subtraction,
2955        )
2956    }
2957
2958    fn emit_binop_mul32(
2959        &mut self,
2960        loc_a: Location,
2961        loc_b: Location,
2962        ret: Location,
2963    ) -> Result<(), CompileError> {
2964        self.emit_relaxed_binop3(
2965            Assembler::emit_mul,
2966            Size::S32,
2967            loc_a,
2968            loc_b,
2969            ret,
2970            ImmType::None,
2971        )
2972    }
2973
2974    fn emit_binop_udiv32(
2975        &mut self,
2976        loc_a: Location,
2977        loc_b: Location,
2978        ret: Location,
2979        integer_division_by_zero: Label,
2980    ) -> Result<usize, CompileError> {
2981        let mut temps = vec![];
2982        let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
2983        let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
2984        let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
2985
2986        let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
2987            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2988        })?;
2989        temps.push(jmp_tmp);
2990
2991        self.assembler
2992            .emit_on_false_label_far(src2, integer_division_by_zero, jmp_tmp)?;
2993        let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
2994        self.assembler.emit_udiv(Size::S32, src1, src2, dest)?;
2995        if ret != dest {
2996            self.move_location(Size::S32, dest, ret)?;
2997        }
2998        for r in temps {
2999            self.release_gpr(r);
3000        }
3001        Ok(offset)
3002    }
3003
3004    fn emit_binop_sdiv32(
3005        &mut self,
3006        loc_a: Location,
3007        loc_b: Location,
3008        ret: Location,
3009        integer_division_by_zero: Label,
3010        integer_overflow: Label,
3011    ) -> Result<usize, CompileError> {
3012        let mut temps = vec![];
3013        let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
3014        let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
3015        let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3016
3017        let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
3018            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3019        })?;
3020        temps.push(jmp_tmp);
3021
3022        self.assembler
3023            .emit_on_false_label_far(src2, integer_division_by_zero, jmp_tmp)?;
3024        let label_nooverflow = self.assembler.get_label();
3025        let tmp = self.location_to_reg(
3026            Size::S32,
3027            Location::Imm32(i32::MIN as u32),
3028            &mut temps,
3029            ImmType::None,
3030            true,
3031            None,
3032        )?;
3033        self.assembler.emit_cmp(Condition::Ne, tmp, src1, tmp)?;
3034        self.assembler
3035            .emit_on_true_label(tmp, label_nooverflow, jmp_tmp)?;
3036        self.move_location(Size::S32, Location::Imm32(-1i32 as _), tmp)?;
3037        self.assembler.emit_cmp(Condition::Eq, tmp, src2, tmp)?;
3038        self.assembler
3039            .emit_on_true_label_far(tmp, integer_overflow, jmp_tmp)?;
3040        let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
3041        self.assembler.emit_label(label_nooverflow)?;
3042        self.assembler.emit_sdiv(Size::S32, src1, src2, dest)?;
3043        if ret != dest {
3044            self.move_location(Size::S32, dest, ret)?;
3045        }
3046        for r in temps {
3047            self.release_gpr(r);
3048        }
3049        Ok(offset)
3050    }
3051
3052    fn emit_binop_urem32(
3053        &mut self,
3054        loc_a: Location,
3055        loc_b: Location,
3056        ret: Location,
3057        integer_division_by_zero: Label,
3058    ) -> Result<usize, CompileError> {
3059        let mut temps = vec![];
3060        let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
3061        let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
3062        let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3063
3064        let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
3065            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3066        })?;
3067        temps.push(jmp_tmp);
3068
3069        self.assembler
3070            .emit_on_false_label_far(src2, integer_division_by_zero, jmp_tmp)?;
3071        let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
3072        self.assembler.emit_urem(Size::S32, src1, src2, dest)?;
3073        if ret != dest {
3074            self.move_location(Size::S32, dest, ret)?;
3075        }
3076        for r in temps {
3077            self.release_gpr(r);
3078        }
3079        Ok(offset)
3080    }
3081
3082    fn emit_binop_srem32(
3083        &mut self,
3084        loc_a: Location,
3085        loc_b: Location,
3086        ret: Location,
3087        integer_division_by_zero: Label,
3088    ) -> Result<usize, CompileError> {
3089        let mut temps = vec![];
3090        let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
3091        let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
3092        let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3093
3094        let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
3095            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3096        })?;
3097        temps.push(jmp_tmp);
3098
3099        self.assembler
3100            .emit_on_false_label_far(src2, integer_division_by_zero, jmp_tmp)?;
3101        let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
3102        self.assembler.emit_srem(Size::S32, src1, src2, dest)?;
3103        if ret != dest {
3104            self.move_location(Size::S32, dest, ret)?;
3105        }
3106        for r in temps {
3107            self.release_gpr(r);
3108        }
3109        Ok(offset)
3110    }
3111
3112    fn emit_binop_and32(
3113        &mut self,
3114        loc_a: Location,
3115        loc_b: Location,
3116        ret: Location,
3117    ) -> Result<(), CompileError> {
3118        self.emit_relaxed_binop3(
3119            Assembler::emit_and,
3120            Size::S32,
3121            loc_a,
3122            loc_b,
3123            ret,
3124            ImmType::Bits12,
3125        )
3126    }
3127
3128    fn emit_binop_or32(
3129        &mut self,
3130        loc_a: Location,
3131        loc_b: Location,
3132        ret: Location,
3133    ) -> Result<(), CompileError> {
3134        self.emit_relaxed_binop3(
3135            Assembler::emit_or,
3136            Size::S32,
3137            loc_a,
3138            loc_b,
3139            ret,
3140            ImmType::Bits12,
3141        )
3142    }
3143
3144    fn emit_binop_xor32(
3145        &mut self,
3146        loc_a: Location,
3147        loc_b: Location,
3148        ret: Location,
3149    ) -> Result<(), CompileError> {
3150        self.emit_relaxed_binop3(
3151            Assembler::emit_xor,
3152            Size::S32,
3153            loc_a,
3154            loc_b,
3155            ret,
3156            ImmType::Bits12,
3157        )
3158    }
3159
3160    fn i32_cmp_ge_s(
3161        &mut self,
3162        loc_a: Location,
3163        loc_b: Location,
3164        ret: Location,
3165    ) -> Result<(), CompileError> {
3166        self.emit_cmpop_i32_dynamic_b(Condition::Ge, loc_a, loc_b, ret, true)
3167    }
3168
3169    fn i32_cmp_gt_s(
3170        &mut self,
3171        loc_a: Location,
3172        loc_b: Location,
3173        ret: Location,
3174    ) -> Result<(), CompileError> {
3175        self.emit_cmpop_i32_dynamic_b(Condition::Gt, loc_a, loc_b, ret, true)
3176    }
3177
3178    fn i32_cmp_le_s(
3179        &mut self,
3180        loc_a: Location,
3181        loc_b: Location,
3182        ret: Location,
3183    ) -> Result<(), CompileError> {
3184        self.emit_cmpop_i32_dynamic_b(Condition::Le, loc_a, loc_b, ret, true)
3185    }
3186
3187    fn i32_cmp_lt_s(
3188        &mut self,
3189        loc_a: Location,
3190        loc_b: Location,
3191        ret: Location,
3192    ) -> Result<(), CompileError> {
3193        self.emit_cmpop_i32_dynamic_b(Condition::Lt, loc_a, loc_b, ret, true)
3194    }
3195
3196    fn i32_cmp_ge_u(
3197        &mut self,
3198        loc_a: Location,
3199        loc_b: Location,
3200        ret: Location,
3201    ) -> Result<(), CompileError> {
3202        self.emit_cmpop_i32_dynamic_b(Condition::Geu, loc_a, loc_b, ret, false)
3203    }
3204
3205    fn i32_cmp_gt_u(
3206        &mut self,
3207        loc_a: Location,
3208        loc_b: Location,
3209        ret: Location,
3210    ) -> Result<(), CompileError> {
3211        self.emit_cmpop_i32_dynamic_b(Condition::Gtu, loc_a, loc_b, ret, false)
3212    }
3213
3214    fn i32_cmp_le_u(
3215        &mut self,
3216        loc_a: Location,
3217        loc_b: Location,
3218        ret: Location,
3219    ) -> Result<(), CompileError> {
3220        self.emit_cmpop_i32_dynamic_b(Condition::Leu, loc_a, loc_b, ret, false)
3221    }
3222
3223    fn i32_cmp_lt_u(
3224        &mut self,
3225        loc_a: Location,
3226        loc_b: Location,
3227        ret: Location,
3228    ) -> Result<(), CompileError> {
3229        self.emit_cmpop_i32_dynamic_b(Condition::Ltu, loc_a, loc_b, ret, false)
3230    }
3231
3232    fn i32_cmp_ne(
3233        &mut self,
3234        loc_a: Location,
3235        loc_b: Location,
3236        ret: Location,
3237    ) -> Result<(), CompileError> {
3238        self.emit_cmpop_i32_dynamic_b(Condition::Ne, loc_a, loc_b, ret, true)
3239    }
3240
3241    fn i32_cmp_eq(
3242        &mut self,
3243        loc_a: Location,
3244        loc_b: Location,
3245        ret: Location,
3246    ) -> Result<(), CompileError> {
3247        self.emit_cmpop_i32_dynamic_b(Condition::Eq, loc_a, loc_b, ret, true)
3248    }
3249
3250    fn i32_clz(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
3251        self.emit_clz(Size::S32, loc, ret)
3252    }
3253
3254    fn i32_ctz(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
3255        self.emit_ctz(Size::S32, loc, ret)
3256    }
3257
3258    fn i32_popcnt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
3259        self.emit_popcnt(Size::S32, loc, ret)
3260    }
3261
3262    fn i32_shl(
3263        &mut self,
3264        loc_a: Location,
3265        loc_b: Location,
3266        ret: Location,
3267    ) -> Result<(), CompileError> {
3268        self.emit_relaxed_binop3(
3269            Assembler::emit_sll,
3270            Size::S32,
3271            loc_a,
3272            loc_b,
3273            ret,
3274            ImmType::Shift32,
3275        )
3276    }
3277
3278    fn i32_shr(
3279        &mut self,
3280        loc_a: Location,
3281        loc_b: Location,
3282        ret: Location,
3283    ) -> Result<(), CompileError> {
3284        self.emit_relaxed_binop3(
3285            Assembler::emit_srl,
3286            Size::S32,
3287            loc_a,
3288            loc_b,
3289            ret,
3290            ImmType::Shift32,
3291        )
3292    }
3293
3294    fn i32_sar(
3295        &mut self,
3296        loc_a: Location,
3297        loc_b: Location,
3298        ret: Location,
3299    ) -> Result<(), CompileError> {
3300        self.emit_relaxed_binop3(
3301            Assembler::emit_sra,
3302            Size::S32,
3303            loc_a,
3304            loc_b,
3305            ret,
3306            ImmType::Shift32,
3307        )
3308    }
3309
3310    fn i32_rol(
3311        &mut self,
3312        loc_a: Location,
3313        loc_b: Location,
3314        ret: Location,
3315    ) -> Result<(), CompileError> {
3316        self.emit_rol(Size::S32, loc_a, loc_b, ret, ImmType::Shift32)
3317    }
3318
3319    fn i32_ror(
3320        &mut self,
3321        loc_a: Location,
3322        loc_b: Location,
3323        ret: Location,
3324    ) -> Result<(), CompileError> {
3325        self.emit_ror(Size::S32, loc_a, loc_b, ret, ImmType::Shift32)
3326    }
3327
3328    fn i32_load(
3329        &mut self,
3330        addr: Location,
3331        memarg: &MemArg,
3332        ret: Location,
3333        _need_check: bool,
3334        imported_memories: bool,
3335        offset: i32,
3336        heap_access_oob: Label,
3337        unaligned_atomic: Label,
3338    ) -> Result<(), CompileError> {
3339        self.memory_op(
3340            addr,
3341            memarg,
3342            false,
3343            4,
3344            imported_memories,
3345            offset,
3346            heap_access_oob,
3347            unaligned_atomic,
3348            |this, addr| this.emit_maybe_unaligned_load(Size::S32, true, ret, addr),
3349        )
3350    }
3351
3352    fn i32_load_8u(
3353        &mut self,
3354        addr: Location,
3355        memarg: &MemArg,
3356        ret: Location,
3357        _need_check: bool,
3358        imported_memories: bool,
3359        offset: i32,
3360        heap_access_oob: Label,
3361        unaligned_atomic: Label,
3362    ) -> Result<(), CompileError> {
3363        self.memory_op(
3364            addr,
3365            memarg,
3366            false,
3367            1,
3368            imported_memories,
3369            offset,
3370            heap_access_oob,
3371            unaligned_atomic,
3372            |this, addr| this.emit_maybe_unaligned_load(Size::S8, false, ret, addr),
3373        )
3374    }
3375
3376    fn i32_load_8s(
3377        &mut self,
3378        addr: Location,
3379        memarg: &MemArg,
3380        ret: Location,
3381        _need_check: bool,
3382        imported_memories: bool,
3383        offset: i32,
3384        heap_access_oob: Label,
3385        unaligned_atomic: Label,
3386    ) -> Result<(), CompileError> {
3387        self.memory_op(
3388            addr,
3389            memarg,
3390            false,
3391            1,
3392            imported_memories,
3393            offset,
3394            heap_access_oob,
3395            unaligned_atomic,
3396            |this, addr| this.emit_maybe_unaligned_load(Size::S8, true, ret, addr),
3397        )
3398    }
3399
3400    fn i32_load_16u(
3401        &mut self,
3402        addr: Location,
3403        memarg: &MemArg,
3404        ret: Location,
3405        _need_check: bool,
3406        imported_memories: bool,
3407        offset: i32,
3408        heap_access_oob: Label,
3409        unaligned_atomic: Label,
3410    ) -> Result<(), CompileError> {
3411        self.memory_op(
3412            addr,
3413            memarg,
3414            false,
3415            2,
3416            imported_memories,
3417            offset,
3418            heap_access_oob,
3419            unaligned_atomic,
3420            |this, addr| this.emit_maybe_unaligned_load(Size::S16, false, ret, addr),
3421        )
3422    }
3423
3424    fn i32_load_16s(
3425        &mut self,
3426        addr: Location,
3427        memarg: &MemArg,
3428        ret: Location,
3429        _need_check: bool,
3430        imported_memories: bool,
3431        offset: i32,
3432        heap_access_oob: Label,
3433        unaligned_atomic: Label,
3434    ) -> Result<(), CompileError> {
3435        self.memory_op(
3436            addr,
3437            memarg,
3438            false,
3439            2,
3440            imported_memories,
3441            offset,
3442            heap_access_oob,
3443            unaligned_atomic,
3444            |this, addr| this.emit_maybe_unaligned_load(Size::S16, true, ret, addr),
3445        )
3446    }
3447
3448    fn i32_atomic_load(
3449        &mut self,
3450        addr: Location,
3451        memarg: &MemArg,
3452        ret: Location,
3453        _need_check: bool,
3454        imported_memories: bool,
3455        offset: i32,
3456        heap_access_oob: Label,
3457        unaligned_atomic: Label,
3458    ) -> Result<(), CompileError> {
3459        self.memory_op(
3460            addr,
3461            memarg,
3462            true,
3463            4,
3464            imported_memories,
3465            offset,
3466            heap_access_oob,
3467            unaligned_atomic,
3468            |this, addr| this.emit_relaxed_load(Size::S32, true, ret, Location::Memory(addr, 0)),
3469        )
3470    }
3471
3472    fn i32_atomic_load_8u(
3473        &mut self,
3474        addr: Location,
3475        memarg: &MemArg,
3476        ret: Location,
3477        _need_check: bool,
3478        imported_memories: bool,
3479        offset: i32,
3480        heap_access_oob: Label,
3481        unaligned_atomic: Label,
3482    ) -> Result<(), CompileError> {
3483        self.memory_op(
3484            addr,
3485            memarg,
3486            true,
3487            1,
3488            imported_memories,
3489            offset,
3490            heap_access_oob,
3491            unaligned_atomic,
3492            |this, addr| this.emit_relaxed_load(Size::S8, false, ret, Location::Memory(addr, 0)),
3493        )
3494    }
3495
3496    fn i32_atomic_load_16u(
3497        &mut self,
3498        addr: Location,
3499        memarg: &MemArg,
3500        ret: Location,
3501        _need_check: bool,
3502        imported_memories: bool,
3503        offset: i32,
3504        heap_access_oob: Label,
3505        unaligned_atomic: Label,
3506    ) -> Result<(), CompileError> {
3507        self.memory_op(
3508            addr,
3509            memarg,
3510            true,
3511            2,
3512            imported_memories,
3513            offset,
3514            heap_access_oob,
3515            unaligned_atomic,
3516            |this, addr| this.emit_relaxed_load(Size::S16, false, ret, Location::Memory(addr, 0)),
3517        )
3518    }
3519
3520    fn i32_save(
3521        &mut self,
3522        value: Location,
3523        memarg: &MemArg,
3524        addr: Location,
3525        _need_check: bool,
3526        imported_memories: bool,
3527        offset: i32,
3528        heap_access_oob: Label,
3529        unaligned_atomic: Label,
3530    ) -> Result<(), CompileError> {
3531        self.memory_op(
3532            addr,
3533            memarg,
3534            false,
3535            4,
3536            imported_memories,
3537            offset,
3538            heap_access_oob,
3539            unaligned_atomic,
3540            |this, addr| this.emit_maybe_unaligned_store(Size::S32, value, addr),
3541        )
3542    }
3543
3544    fn i32_save_8(
3545        &mut self,
3546        value: Location,
3547        memarg: &MemArg,
3548        addr: Location,
3549        _need_check: bool,
3550        imported_memories: bool,
3551        offset: i32,
3552        heap_access_oob: Label,
3553        unaligned_atomic: Label,
3554    ) -> Result<(), CompileError> {
3555        self.memory_op(
3556            addr,
3557            memarg,
3558            false,
3559            1,
3560            imported_memories,
3561            offset,
3562            heap_access_oob,
3563            unaligned_atomic,
3564            |this, addr| this.emit_maybe_unaligned_store(Size::S8, value, addr),
3565        )
3566    }
3567
3568    fn i32_save_16(
3569        &mut self,
3570        value: Location,
3571        memarg: &MemArg,
3572        addr: Location,
3573        _need_check: bool,
3574        imported_memories: bool,
3575        offset: i32,
3576        heap_access_oob: Label,
3577        unaligned_atomic: Label,
3578    ) -> Result<(), CompileError> {
3579        self.memory_op(
3580            addr,
3581            memarg,
3582            false,
3583            2,
3584            imported_memories,
3585            offset,
3586            heap_access_oob,
3587            unaligned_atomic,
3588            |this, addr| this.emit_maybe_unaligned_store(Size::S16, value, addr),
3589        )
3590    }
3591
3592    fn i32_atomic_save(
3593        &mut self,
3594        value: Location,
3595        memarg: &MemArg,
3596        addr: Location,
3597        _need_check: bool,
3598        imported_memories: bool,
3599        offset: i32,
3600        heap_access_oob: Label,
3601        unaligned_atomic: Label,
3602    ) -> Result<(), CompileError> {
3603        self.memory_op(
3604            addr,
3605            memarg,
3606            true,
3607            4,
3608            imported_memories,
3609            offset,
3610            heap_access_oob,
3611            unaligned_atomic,
3612            |this, addr| this.emit_relaxed_store(Size::S32, value, Location::Memory(addr, 0)),
3613        )?;
3614        self.assembler.emit_rwfence()
3615    }
3616
3617    fn i32_atomic_save_8(
3618        &mut self,
3619        value: Location,
3620        memarg: &MemArg,
3621        addr: Location,
3622        _need_check: bool,
3623        imported_memories: bool,
3624        offset: i32,
3625        heap_access_oob: Label,
3626        unaligned_atomic: Label,
3627    ) -> Result<(), CompileError> {
3628        self.memory_op(
3629            addr,
3630            memarg,
3631            true,
3632            1,
3633            imported_memories,
3634            offset,
3635            heap_access_oob,
3636            unaligned_atomic,
3637            |this, addr| this.emit_relaxed_store(Size::S8, value, Location::Memory(addr, 0)),
3638        )?;
3639        self.assembler.emit_rwfence()
3640    }
3641
3642    fn i32_atomic_save_16(
3643        &mut self,
3644        value: Location,
3645        memarg: &MemArg,
3646        addr: Location,
3647        _need_check: bool,
3648        imported_memories: bool,
3649        offset: i32,
3650        heap_access_oob: Label,
3651        unaligned_atomic: Label,
3652    ) -> Result<(), CompileError> {
3653        self.memory_op(
3654            addr,
3655            memarg,
3656            true,
3657            2,
3658            imported_memories,
3659            offset,
3660            heap_access_oob,
3661            unaligned_atomic,
3662            |this, addr| this.emit_relaxed_store(Size::S16, value, Location::Memory(addr, 0)),
3663        )?;
3664        self.assembler.emit_rwfence()
3665    }
3666
3667    fn i32_atomic_add(
3668        &mut self,
3669        loc: Location,
3670        target: Location,
3671        memarg: &MemArg,
3672        ret: Location,
3673        _need_check: bool,
3674        imported_memories: bool,
3675        offset: i32,
3676        heap_access_oob: Label,
3677        unaligned_atomic: Label,
3678    ) -> Result<(), CompileError> {
3679        self.memory_op(
3680            target,
3681            memarg,
3682            true,
3683            4,
3684            imported_memories,
3685            offset,
3686            heap_access_oob,
3687            unaligned_atomic,
3688            |this, addr| {
3689                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Add, Size::S32, ret, addr, loc)
3690            },
3691        )
3692    }
3693
3694    fn i32_atomic_add_8u(
3695        &mut self,
3696        loc: Location,
3697        target: Location,
3698        memarg: &MemArg,
3699        ret: Location,
3700        _need_check: bool,
3701        imported_memories: bool,
3702        offset: i32,
3703        heap_access_oob: Label,
3704        unaligned_atomic: Label,
3705    ) -> Result<(), CompileError> {
3706        self.memory_op(
3707            target,
3708            memarg,
3709            true,
3710            1,
3711            imported_memories,
3712            offset,
3713            heap_access_oob,
3714            unaligned_atomic,
3715            |this, addr| {
3716                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Add, Size::S8, ret, addr, loc)
3717            },
3718        )
3719    }
3720
3721    fn i32_atomic_add_16u(
3722        &mut self,
3723        loc: Location,
3724        target: Location,
3725        memarg: &MemArg,
3726        ret: Location,
3727        _need_check: bool,
3728        imported_memories: bool,
3729        offset: i32,
3730        heap_access_oob: Label,
3731        unaligned_atomic: Label,
3732    ) -> Result<(), CompileError> {
3733        self.memory_op(
3734            target,
3735            memarg,
3736            true,
3737            2,
3738            imported_memories,
3739            offset,
3740            heap_access_oob,
3741            unaligned_atomic,
3742            |this, addr| {
3743                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Add, Size::S16, ret, addr, loc)
3744            },
3745        )
3746    }
3747
3748    fn i32_atomic_sub(
3749        &mut self,
3750        loc: Location,
3751        target: Location,
3752        memarg: &MemArg,
3753        ret: Location,
3754        _need_check: bool,
3755        imported_memories: bool,
3756        offset: i32,
3757        heap_access_oob: Label,
3758        unaligned_atomic: Label,
3759    ) -> Result<(), CompileError> {
3760        self.memory_op(
3761            target,
3762            memarg,
3763            true,
3764            4,
3765            imported_memories,
3766            offset,
3767            heap_access_oob,
3768            unaligned_atomic,
3769            |this, addr| {
3770                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Sub, Size::S32, ret, addr, loc)
3771            },
3772        )
3773    }
3774
3775    fn i32_atomic_sub_8u(
3776        &mut self,
3777        loc: Location,
3778        target: Location,
3779        memarg: &MemArg,
3780        ret: Location,
3781        _need_check: bool,
3782        imported_memories: bool,
3783        offset: i32,
3784        heap_access_oob: Label,
3785        unaligned_atomic: Label,
3786    ) -> Result<(), CompileError> {
3787        self.memory_op(
3788            target,
3789            memarg,
3790            true,
3791            1,
3792            imported_memories,
3793            offset,
3794            heap_access_oob,
3795            unaligned_atomic,
3796            |this, addr| {
3797                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Sub, Size::S8, ret, addr, loc)
3798            },
3799        )
3800    }
3801
3802    fn i32_atomic_sub_16u(
3803        &mut self,
3804        loc: Location,
3805        target: Location,
3806        memarg: &MemArg,
3807        ret: Location,
3808        _need_check: bool,
3809        imported_memories: bool,
3810        offset: i32,
3811        heap_access_oob: Label,
3812        unaligned_atomic: Label,
3813    ) -> Result<(), CompileError> {
3814        self.memory_op(
3815            target,
3816            memarg,
3817            true,
3818            2,
3819            imported_memories,
3820            offset,
3821            heap_access_oob,
3822            unaligned_atomic,
3823            |this, addr| {
3824                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Sub, Size::S16, ret, addr, loc)
3825            },
3826        )
3827    }
3828
3829    fn i32_atomic_and(
3830        &mut self,
3831        loc: Location,
3832        target: Location,
3833        memarg: &MemArg,
3834        ret: Location,
3835        _need_check: bool,
3836        imported_memories: bool,
3837        offset: i32,
3838        heap_access_oob: Label,
3839        unaligned_atomic: Label,
3840    ) -> Result<(), CompileError> {
3841        self.memory_op(
3842            target,
3843            memarg,
3844            true,
3845            4,
3846            imported_memories,
3847            offset,
3848            heap_access_oob,
3849            unaligned_atomic,
3850            |this, addr| {
3851                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::And, Size::S32, ret, addr, loc)
3852            },
3853        )
3854    }
3855
3856    fn i32_atomic_and_8u(
3857        &mut self,
3858        loc: Location,
3859        target: Location,
3860        memarg: &MemArg,
3861        ret: Location,
3862        _need_check: bool,
3863        imported_memories: bool,
3864        offset: i32,
3865        heap_access_oob: Label,
3866        unaligned_atomic: Label,
3867    ) -> Result<(), CompileError> {
3868        self.memory_op(
3869            target,
3870            memarg,
3871            true,
3872            1,
3873            imported_memories,
3874            offset,
3875            heap_access_oob,
3876            unaligned_atomic,
3877            |this, addr| {
3878                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::And, Size::S8, ret, addr, loc)
3879            },
3880        )
3881    }
3882
3883    fn i32_atomic_and_16u(
3884        &mut self,
3885        loc: Location,
3886        target: Location,
3887        memarg: &MemArg,
3888        ret: Location,
3889        _need_check: bool,
3890        imported_memories: bool,
3891        offset: i32,
3892        heap_access_oob: Label,
3893        unaligned_atomic: Label,
3894    ) -> Result<(), CompileError> {
3895        self.memory_op(
3896            target,
3897            memarg,
3898            true,
3899            2,
3900            imported_memories,
3901            offset,
3902            heap_access_oob,
3903            unaligned_atomic,
3904            |this, addr| {
3905                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::And, Size::S16, ret, addr, loc)
3906            },
3907        )
3908    }
3909
3910    fn i32_atomic_or(
3911        &mut self,
3912        loc: Location,
3913        target: Location,
3914        memarg: &MemArg,
3915        ret: Location,
3916        _need_check: bool,
3917        imported_memories: bool,
3918        offset: i32,
3919        heap_access_oob: Label,
3920        unaligned_atomic: Label,
3921    ) -> Result<(), CompileError> {
3922        self.memory_op(
3923            target,
3924            memarg,
3925            true,
3926            4,
3927            imported_memories,
3928            offset,
3929            heap_access_oob,
3930            unaligned_atomic,
3931            |this, addr| {
3932                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Or, Size::S32, ret, addr, loc)
3933            },
3934        )
3935    }
3936
3937    fn i32_atomic_or_8u(
3938        &mut self,
3939        loc: Location,
3940        target: Location,
3941        memarg: &MemArg,
3942        ret: Location,
3943        _need_check: bool,
3944        imported_memories: bool,
3945        offset: i32,
3946        heap_access_oob: Label,
3947        unaligned_atomic: Label,
3948    ) -> Result<(), CompileError> {
3949        self.memory_op(
3950            target,
3951            memarg,
3952            true,
3953            1,
3954            imported_memories,
3955            offset,
3956            heap_access_oob,
3957            unaligned_atomic,
3958            |this, addr| {
3959                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Or, Size::S8, ret, addr, loc)
3960            },
3961        )
3962    }
3963
3964    fn i32_atomic_or_16u(
3965        &mut self,
3966        loc: Location,
3967        target: Location,
3968        memarg: &MemArg,
3969        ret: Location,
3970        _need_check: bool,
3971        imported_memories: bool,
3972        offset: i32,
3973        heap_access_oob: Label,
3974        unaligned_atomic: Label,
3975    ) -> Result<(), CompileError> {
3976        self.memory_op(
3977            target,
3978            memarg,
3979            true,
3980            2,
3981            imported_memories,
3982            offset,
3983            heap_access_oob,
3984            unaligned_atomic,
3985            |this, addr| {
3986                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Or, Size::S16, ret, addr, loc)
3987            },
3988        )
3989    }
3990
3991    fn i32_atomic_xor(
3992        &mut self,
3993        loc: Location,
3994        target: Location,
3995        memarg: &MemArg,
3996        ret: Location,
3997        _need_check: bool,
3998        imported_memories: bool,
3999        offset: i32,
4000        heap_access_oob: Label,
4001        unaligned_atomic: Label,
4002    ) -> Result<(), CompileError> {
4003        self.memory_op(
4004            target,
4005            memarg,
4006            true,
4007            4,
4008            imported_memories,
4009            offset,
4010            heap_access_oob,
4011            unaligned_atomic,
4012            |this, addr| {
4013                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Xor, Size::S32, ret, addr, loc)
4014            },
4015        )
4016    }
4017
4018    fn i32_atomic_xor_8u(
4019        &mut self,
4020        loc: Location,
4021        target: Location,
4022        memarg: &MemArg,
4023        ret: Location,
4024        _need_check: bool,
4025        imported_memories: bool,
4026        offset: i32,
4027        heap_access_oob: Label,
4028        unaligned_atomic: Label,
4029    ) -> Result<(), CompileError> {
4030        self.memory_op(
4031            target,
4032            memarg,
4033            true,
4034            1,
4035            imported_memories,
4036            offset,
4037            heap_access_oob,
4038            unaligned_atomic,
4039            |this, addr| {
4040                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Xor, Size::S8, ret, addr, loc)
4041            },
4042        )
4043    }
4044
4045    fn i32_atomic_xor_16u(
4046        &mut self,
4047        loc: Location,
4048        target: Location,
4049        memarg: &MemArg,
4050        ret: Location,
4051        _need_check: bool,
4052        imported_memories: bool,
4053        offset: i32,
4054        heap_access_oob: Label,
4055        unaligned_atomic: Label,
4056    ) -> Result<(), CompileError> {
4057        self.memory_op(
4058            target,
4059            memarg,
4060            true,
4061            2,
4062            imported_memories,
4063            offset,
4064            heap_access_oob,
4065            unaligned_atomic,
4066            |this, addr| {
4067                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Xor, Size::S16, ret, addr, loc)
4068            },
4069        )
4070    }
4071
4072    fn i32_atomic_xchg(
4073        &mut self,
4074        loc: Location,
4075        target: Location,
4076        memarg: &MemArg,
4077        ret: Location,
4078        _need_check: bool,
4079        imported_memories: bool,
4080        offset: i32,
4081        heap_access_oob: Label,
4082        unaligned_atomic: Label,
4083    ) -> Result<(), CompileError> {
4084        self.memory_op(
4085            target,
4086            memarg,
4087            true,
4088            4,
4089            imported_memories,
4090            offset,
4091            heap_access_oob,
4092            unaligned_atomic,
4093            |this, addr| {
4094                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Exchange, Size::S32, ret, addr, loc)
4095            },
4096        )
4097    }
4098
4099    fn i32_atomic_xchg_8u(
4100        &mut self,
4101        loc: Location,
4102        target: Location,
4103        memarg: &MemArg,
4104        ret: Location,
4105        _need_check: bool,
4106        imported_memories: bool,
4107        offset: i32,
4108        heap_access_oob: Label,
4109        unaligned_atomic: Label,
4110    ) -> Result<(), CompileError> {
4111        self.memory_op(
4112            target,
4113            memarg,
4114            true,
4115            1,
4116            imported_memories,
4117            offset,
4118            heap_access_oob,
4119            unaligned_atomic,
4120            |this, addr| {
4121                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Exchange, Size::S8, ret, addr, loc)
4122            },
4123        )
4124    }
4125
4126    fn i32_atomic_xchg_16u(
4127        &mut self,
4128        loc: Location,
4129        target: Location,
4130        memarg: &MemArg,
4131        ret: Location,
4132        _need_check: bool,
4133        imported_memories: bool,
4134        offset: i32,
4135        heap_access_oob: Label,
4136        unaligned_atomic: Label,
4137    ) -> Result<(), CompileError> {
4138        self.memory_op(
4139            target,
4140            memarg,
4141            true,
4142            2,
4143            imported_memories,
4144            offset,
4145            heap_access_oob,
4146            unaligned_atomic,
4147            |this, addr| {
4148                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Exchange, Size::S16, ret, addr, loc)
4149            },
4150        )
4151    }
4152
4153    fn i32_atomic_cmpxchg(
4154        &mut self,
4155        new: Location,
4156        cmp: Location,
4157        target: Location,
4158        memarg: &MemArg,
4159        ret: Location,
4160        _need_check: bool,
4161        imported_memories: bool,
4162        offset: i32,
4163        heap_access_oob: Label,
4164        unaligned_atomic: Label,
4165    ) -> Result<(), CompileError> {
4166        self.memory_op(
4167            target,
4168            memarg,
4169            true,
4170            4,
4171            imported_memories,
4172            offset,
4173            heap_access_oob,
4174            unaligned_atomic,
4175            |this, addr| this.emit_relaxed_atomic_cmpxchg(Size::S32, ret, addr, new, cmp),
4176        )
4177    }
4178
4179    fn i32_atomic_cmpxchg_8u(
4180        &mut self,
4181        new: Location,
4182        cmp: Location,
4183        target: Location,
4184        memarg: &MemArg,
4185        ret: Location,
4186        _need_check: bool,
4187        imported_memories: bool,
4188        offset: i32,
4189        heap_access_oob: Label,
4190        unaligned_atomic: Label,
4191    ) -> Result<(), CompileError> {
4192        self.memory_op(
4193            target,
4194            memarg,
4195            true,
4196            1,
4197            imported_memories,
4198            offset,
4199            heap_access_oob,
4200            unaligned_atomic,
4201            |this, addr| this.emit_relaxed_atomic_cmpxchg(Size::S8, ret, addr, new, cmp),
4202        )
4203    }
4204
4205    fn i32_atomic_cmpxchg_16u(
4206        &mut self,
4207        new: Location,
4208        cmp: Location,
4209        target: Location,
4210        memarg: &MemArg,
4211        ret: Location,
4212        _need_check: bool,
4213        imported_memories: bool,
4214        offset: i32,
4215        heap_access_oob: Label,
4216        unaligned_atomic: Label,
4217    ) -> Result<(), CompileError> {
4218        self.memory_op(
4219            target,
4220            memarg,
4221            true,
4222            2,
4223            imported_memories,
4224            offset,
4225            heap_access_oob,
4226            unaligned_atomic,
4227            |this, addr| this.emit_relaxed_atomic_cmpxchg(Size::S16, ret, addr, new, cmp),
4228        )
4229    }
4230
4231    fn emit_call_with_reloc(
4232        &mut self,
4233        reloc_target: RelocationTarget,
4234    ) -> Result<Vec<Relocation>, CompileError> {
4235        let mut relocations = vec![];
4236        let next = self.get_label();
4237        let reloc_at = self.assembler.get_offset().0;
4238        self.emit_label(next)?; // this is to be sure the current imm26 value is 0
4239        self.assembler.emit_call_label(next)?;
4240        relocations.push(Relocation {
4241            kind: RelocationKind::RiscvCall,
4242            reloc_target,
4243            offset: reloc_at as u32,
4244            addend: 0,
4245        });
4246        Ok(relocations)
4247    }
4248
4249    fn emit_binop_add64(
4250        &mut self,
4251        loc_a: Location,
4252        loc_b: Location,
4253        ret: Location,
4254    ) -> Result<(), CompileError> {
4255        self.emit_relaxed_binop3(
4256            Assembler::emit_add,
4257            Size::S64,
4258            loc_a,
4259            loc_b,
4260            ret,
4261            ImmType::Bits12,
4262        )
4263    }
4264
4265    fn emit_binop_sub64(
4266        &mut self,
4267        loc_a: Location,
4268        loc_b: Location,
4269        ret: Location,
4270    ) -> Result<(), CompileError> {
4271        self.emit_relaxed_binop3(
4272            Assembler::emit_sub,
4273            Size::S64,
4274            loc_a,
4275            loc_b,
4276            ret,
4277            ImmType::Bits12Subtraction,
4278        )
4279    }
4280
4281    fn emit_binop_mul64(
4282        &mut self,
4283        loc_a: Location,
4284        loc_b: Location,
4285        ret: Location,
4286    ) -> Result<(), CompileError> {
4287        self.emit_relaxed_binop3(
4288            Assembler::emit_mul,
4289            Size::S64,
4290            loc_a,
4291            loc_b,
4292            ret,
4293            ImmType::None,
4294        )
4295    }
4296
4297    fn emit_binop_udiv64(
4298        &mut self,
4299        loc_a: Location,
4300        loc_b: Location,
4301        ret: Location,
4302        integer_division_by_zero: Label,
4303    ) -> Result<usize, CompileError> {
4304        let mut temps = vec![];
4305        let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4306        let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4307        let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4308
4309        let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
4310            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4311        })?;
4312        temps.push(jmp_tmp);
4313
4314        self.assembler
4315            .emit_on_false_label_far(src2, integer_division_by_zero, jmp_tmp)?;
4316        let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
4317        self.assembler.emit_udiv(Size::S64, src1, src2, dest)?;
4318        if ret != dest {
4319            self.move_location(Size::S64, dest, ret)?;
4320        }
4321        for r in temps {
4322            self.release_gpr(r);
4323        }
4324        Ok(offset)
4325    }
4326
4327    fn emit_binop_sdiv64(
4328        &mut self,
4329        loc_a: Location,
4330        loc_b: Location,
4331        ret: Location,
4332        integer_division_by_zero: Label,
4333        integer_overflow: Label,
4334    ) -> Result<usize, CompileError> {
4335        let mut temps = vec![];
4336        let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4337        let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4338        let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4339
4340        let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
4341            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4342        })?;
4343        temps.push(jmp_tmp);
4344
4345        self.assembler
4346            .emit_on_false_label_far(src2, integer_division_by_zero, jmp_tmp)?;
4347        let label_nooverflow = self.assembler.get_label();
4348        let tmp = self.location_to_reg(
4349            Size::S64,
4350            Location::Imm64(i64::MIN as u64),
4351            &mut temps,
4352            ImmType::None,
4353            true,
4354            None,
4355        )?;
4356
4357        self.assembler.emit_cmp(Condition::Ne, tmp, src1, tmp)?;
4358        self.assembler
4359            .emit_on_true_label(tmp, label_nooverflow, jmp_tmp)?;
4360        self.move_location(Size::S64, Location::Imm64(-1i64 as _), tmp)?;
4361        self.assembler.emit_cmp(Condition::Eq, tmp, src2, tmp)?;
4362        self.assembler
4363            .emit_on_true_label_far(tmp, integer_overflow, jmp_tmp)?;
4364        let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
4365        self.assembler.emit_label(label_nooverflow)?;
4366        self.assembler.emit_sdiv(Size::S64, src1, src2, dest)?;
4367        if ret != dest {
4368            self.move_location(Size::S64, dest, ret)?;
4369        }
4370        for r in temps {
4371            self.release_gpr(r);
4372        }
4373        Ok(offset)
4374    }
4375
4376    fn emit_binop_urem64(
4377        &mut self,
4378        loc_a: Location,
4379        loc_b: Location,
4380        ret: Location,
4381        integer_division_by_zero: Label,
4382    ) -> Result<usize, CompileError> {
4383        let mut temps = vec![];
4384        let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4385        let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4386        let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4387
4388        let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
4389            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4390        })?;
4391        temps.push(jmp_tmp);
4392
4393        self.assembler
4394            .emit_on_false_label_far(src2, integer_division_by_zero, jmp_tmp)?;
4395        let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
4396        self.assembler.emit_urem(Size::S64, src1, src2, dest)?;
4397        if ret != dest {
4398            self.move_location(Size::S64, dest, ret)?;
4399        }
4400        for r in temps {
4401            self.release_gpr(r);
4402        }
4403        Ok(offset)
4404    }
4405
4406    fn emit_binop_srem64(
4407        &mut self,
4408        loc_a: Location,
4409        loc_b: Location,
4410        ret: Location,
4411        integer_division_by_zero: Label,
4412    ) -> Result<usize, CompileError> {
4413        let mut temps = vec![];
4414        let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4415        let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4416        let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4417
4418        let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
4419            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4420        })?;
4421        temps.push(jmp_tmp);
4422
4423        self.assembler
4424            .emit_on_false_label_far(src2, integer_division_by_zero, jmp_tmp)?;
4425        let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
4426        self.assembler.emit_srem(Size::S64, src1, src2, dest)?;
4427        if ret != dest {
4428            self.move_location(Size::S64, dest, ret)?;
4429        }
4430        for r in temps {
4431            self.release_gpr(r);
4432        }
4433        Ok(offset)
4434    }
4435
4436    fn emit_binop_and64(
4437        &mut self,
4438        loc_a: Location,
4439        loc_b: Location,
4440        ret: Location,
4441    ) -> Result<(), CompileError> {
4442        self.emit_relaxed_binop3(
4443            Assembler::emit_and,
4444            Size::S64,
4445            loc_a,
4446            loc_b,
4447            ret,
4448            ImmType::Bits12,
4449        )
4450    }
4451
4452    fn emit_binop_or64(
4453        &mut self,
4454        loc_a: Location,
4455        loc_b: Location,
4456        ret: Location,
4457    ) -> Result<(), CompileError> {
4458        self.emit_relaxed_binop3(
4459            Assembler::emit_or,
4460            Size::S64,
4461            loc_a,
4462            loc_b,
4463            ret,
4464            ImmType::Bits12,
4465        )
4466    }
4467
4468    fn emit_binop_xor64(
4469        &mut self,
4470        loc_a: Location,
4471        loc_b: Location,
4472        ret: Location,
4473    ) -> Result<(), CompileError> {
4474        self.emit_relaxed_binop3(
4475            Assembler::emit_xor,
4476            Size::S64,
4477            loc_a,
4478            loc_b,
4479            ret,
4480            ImmType::Bits12,
4481        )
4482    }
4483
4484    fn i64_cmp_ge_s(
4485        &mut self,
4486        loc_a: Location,
4487        loc_b: Location,
4488        ret: Location,
4489    ) -> Result<(), CompileError> {
4490        self.emit_cmpop_i64_dynamic_b(Condition::Ge, loc_a, loc_b, ret)
4491    }
4492
4493    fn i64_cmp_gt_s(
4494        &mut self,
4495        loc_a: Location,
4496        loc_b: Location,
4497        ret: Location,
4498    ) -> Result<(), CompileError> {
4499        self.emit_cmpop_i64_dynamic_b(Condition::Gt, loc_a, loc_b, ret)
4500    }
4501
4502    fn i64_cmp_le_s(
4503        &mut self,
4504        loc_a: Location,
4505        loc_b: Location,
4506        ret: Location,
4507    ) -> Result<(), CompileError> {
4508        self.emit_cmpop_i64_dynamic_b(Condition::Le, loc_a, loc_b, ret)
4509    }
4510
4511    fn i64_cmp_lt_s(
4512        &mut self,
4513        loc_a: Location,
4514        loc_b: Location,
4515        ret: Location,
4516    ) -> Result<(), CompileError> {
4517        self.emit_cmpop_i64_dynamic_b(Condition::Lt, loc_a, loc_b, ret)
4518    }
4519
4520    fn i64_cmp_ge_u(
4521        &mut self,
4522        loc_a: Location,
4523        loc_b: Location,
4524        ret: Location,
4525    ) -> Result<(), CompileError> {
4526        self.emit_cmpop_i64_dynamic_b(Condition::Geu, loc_a, loc_b, ret)
4527    }
4528
4529    fn i64_cmp_gt_u(
4530        &mut self,
4531        loc_a: Location,
4532        loc_b: Location,
4533        ret: Location,
4534    ) -> Result<(), CompileError> {
4535        self.emit_cmpop_i64_dynamic_b(Condition::Gtu, loc_a, loc_b, ret)
4536    }
4537
4538    fn i64_cmp_le_u(
4539        &mut self,
4540        loc_a: Location,
4541        loc_b: Location,
4542        ret: Location,
4543    ) -> Result<(), CompileError> {
4544        self.emit_cmpop_i64_dynamic_b(Condition::Leu, loc_a, loc_b, ret)
4545    }
4546
4547    fn i64_cmp_lt_u(
4548        &mut self,
4549        loc_a: Location,
4550        loc_b: Location,
4551        ret: Location,
4552    ) -> Result<(), CompileError> {
4553        self.emit_cmpop_i64_dynamic_b(Condition::Ltu, loc_a, loc_b, ret)
4554    }
4555
4556    fn i64_cmp_ne(
4557        &mut self,
4558        loc_a: Location,
4559        loc_b: Location,
4560        ret: Location,
4561    ) -> Result<(), CompileError> {
4562        self.emit_cmpop_i64_dynamic_b(Condition::Ne, loc_a, loc_b, ret)
4563    }
4564
4565    fn i64_cmp_eq(
4566        &mut self,
4567        loc_a: Location,
4568        loc_b: Location,
4569        ret: Location,
4570    ) -> Result<(), CompileError> {
4571        self.emit_cmpop_i64_dynamic_b(Condition::Eq, loc_a, loc_b, ret)
4572    }
4573
4574    fn i64_clz(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
4575        self.emit_clz(Size::S64, loc, ret)
4576    }
4577
4578    fn i64_ctz(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
4579        self.emit_ctz(Size::S64, loc, ret)
4580    }
4581
4582    fn i64_popcnt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
4583        self.emit_popcnt(Size::S64, loc, ret)
4584    }
4585
4586    fn i64_shl(
4587        &mut self,
4588        loc_a: Location,
4589        loc_b: Location,
4590        ret: Location,
4591    ) -> Result<(), CompileError> {
4592        self.emit_relaxed_binop3(
4593            Assembler::emit_sll,
4594            Size::S64,
4595            loc_a,
4596            loc_b,
4597            ret,
4598            ImmType::Shift64,
4599        )
4600    }
4601
4602    fn i64_shr(
4603        &mut self,
4604        loc_a: Location,
4605        loc_b: Location,
4606        ret: Location,
4607    ) -> Result<(), CompileError> {
4608        self.emit_relaxed_binop3(
4609            Assembler::emit_srl,
4610            Size::S64,
4611            loc_a,
4612            loc_b,
4613            ret,
4614            ImmType::Shift64,
4615        )
4616    }
4617
4618    fn i64_sar(
4619        &mut self,
4620        loc_a: Location,
4621        loc_b: Location,
4622        ret: Location,
4623    ) -> Result<(), CompileError> {
4624        self.emit_relaxed_binop3(
4625            Assembler::emit_sra,
4626            Size::S64,
4627            loc_a,
4628            loc_b,
4629            ret,
4630            ImmType::Shift64,
4631        )
4632    }
4633
4634    fn i64_rol(
4635        &mut self,
4636        loc_a: Location,
4637        loc_b: Location,
4638        ret: Location,
4639    ) -> Result<(), CompileError> {
4640        self.emit_rol(Size::S64, loc_a, loc_b, ret, ImmType::Shift64)
4641    }
4642
4643    fn i64_ror(
4644        &mut self,
4645        loc_a: Location,
4646        loc_b: Location,
4647        ret: Location,
4648    ) -> Result<(), CompileError> {
4649        self.emit_ror(Size::S64, loc_a, loc_b, ret, ImmType::Shift64)
4650    }
4651
4652    fn i64_load(
4653        &mut self,
4654        addr: Location,
4655        memarg: &MemArg,
4656        ret: Location,
4657        _need_check: bool,
4658        imported_memories: bool,
4659        offset: i32,
4660        heap_access_oob: Label,
4661        unaligned_atomic: Label,
4662    ) -> Result<(), CompileError> {
4663        self.memory_op(
4664            addr,
4665            memarg,
4666            false,
4667            8,
4668            imported_memories,
4669            offset,
4670            heap_access_oob,
4671            unaligned_atomic,
4672            |this, addr| this.emit_maybe_unaligned_load(Size::S64, true, ret, addr),
4673        )
4674    }
4675
4676    fn i64_load_8u(
4677        &mut self,
4678        addr: Location,
4679        memarg: &MemArg,
4680        ret: Location,
4681        _need_check: bool,
4682        imported_memories: bool,
4683        offset: i32,
4684        heap_access_oob: Label,
4685        unaligned_atomic: Label,
4686    ) -> Result<(), CompileError> {
4687        self.memory_op(
4688            addr,
4689            memarg,
4690            false,
4691            1,
4692            imported_memories,
4693            offset,
4694            heap_access_oob,
4695            unaligned_atomic,
4696            |this, addr| this.emit_maybe_unaligned_load(Size::S8, false, ret, addr),
4697        )
4698    }
4699
4700    fn i64_load_8s(
4701        &mut self,
4702        addr: Location,
4703        memarg: &MemArg,
4704        ret: Location,
4705        _need_check: bool,
4706        imported_memories: bool,
4707        offset: i32,
4708        heap_access_oob: Label,
4709        unaligned_atomic: Label,
4710    ) -> Result<(), CompileError> {
4711        self.memory_op(
4712            addr,
4713            memarg,
4714            false,
4715            1,
4716            imported_memories,
4717            offset,
4718            heap_access_oob,
4719            unaligned_atomic,
4720            |this, addr| this.emit_maybe_unaligned_load(Size::S8, true, ret, addr),
4721        )
4722    }
4723
4724    fn i64_load_32u(
4725        &mut self,
4726        addr: Location,
4727        memarg: &MemArg,
4728        ret: Location,
4729        _need_check: bool,
4730        imported_memories: bool,
4731        offset: i32,
4732        heap_access_oob: Label,
4733        unaligned_atomic: Label,
4734    ) -> Result<(), CompileError> {
4735        self.memory_op(
4736            addr,
4737            memarg,
4738            false,
4739            4,
4740            imported_memories,
4741            offset,
4742            heap_access_oob,
4743            unaligned_atomic,
4744            |this, addr| this.emit_maybe_unaligned_load(Size::S32, false, ret, addr),
4745        )
4746    }
4747
4748    fn i64_load_32s(
4749        &mut self,
4750        addr: Location,
4751        memarg: &MemArg,
4752        ret: Location,
4753        _need_check: bool,
4754        imported_memories: bool,
4755        offset: i32,
4756        heap_access_oob: Label,
4757        unaligned_atomic: Label,
4758    ) -> Result<(), CompileError> {
4759        self.memory_op(
4760            addr,
4761            memarg,
4762            false,
4763            4,
4764            imported_memories,
4765            offset,
4766            heap_access_oob,
4767            unaligned_atomic,
4768            |this, addr| this.emit_maybe_unaligned_load(Size::S32, true, ret, addr),
4769        )
4770    }
4771
4772    fn i64_load_16u(
4773        &mut self,
4774        addr: Location,
4775        memarg: &MemArg,
4776        ret: Location,
4777        _need_check: bool,
4778        imported_memories: bool,
4779        offset: i32,
4780        heap_access_oob: Label,
4781        unaligned_atomic: Label,
4782    ) -> Result<(), CompileError> {
4783        self.memory_op(
4784            addr,
4785            memarg,
4786            false,
4787            2,
4788            imported_memories,
4789            offset,
4790            heap_access_oob,
4791            unaligned_atomic,
4792            |this, addr| this.emit_maybe_unaligned_load(Size::S16, false, ret, addr),
4793        )
4794    }
4795
4796    fn i64_load_16s(
4797        &mut self,
4798        addr: Location,
4799        memarg: &MemArg,
4800        ret: Location,
4801        _need_check: bool,
4802        imported_memories: bool,
4803        offset: i32,
4804        heap_access_oob: Label,
4805        unaligned_atomic: Label,
4806    ) -> Result<(), CompileError> {
4807        self.memory_op(
4808            addr,
4809            memarg,
4810            false,
4811            2,
4812            imported_memories,
4813            offset,
4814            heap_access_oob,
4815            unaligned_atomic,
4816            |this, addr| this.emit_maybe_unaligned_load(Size::S16, true, ret, addr),
4817        )
4818    }
4819
4820    fn i64_atomic_load(
4821        &mut self,
4822        addr: Location,
4823        memarg: &MemArg,
4824        ret: Location,
4825        _need_check: bool,
4826        imported_memories: bool,
4827        offset: i32,
4828        heap_access_oob: Label,
4829        unaligned_atomic: Label,
4830    ) -> Result<(), CompileError> {
4831        self.memory_op(
4832            addr,
4833            memarg,
4834            true,
4835            8,
4836            imported_memories,
4837            offset,
4838            heap_access_oob,
4839            unaligned_atomic,
4840            |this, addr| this.emit_relaxed_load(Size::S64, true, ret, Location::Memory(addr, 0)),
4841        )
4842    }
4843
4844    fn i64_atomic_load_8u(
4845        &mut self,
4846        addr: Location,
4847        memarg: &MemArg,
4848        ret: Location,
4849        _need_check: bool,
4850        imported_memories: bool,
4851        offset: i32,
4852        heap_access_oob: Label,
4853        unaligned_atomic: Label,
4854    ) -> Result<(), CompileError> {
4855        self.memory_op(
4856            addr,
4857            memarg,
4858            true,
4859            1,
4860            imported_memories,
4861            offset,
4862            heap_access_oob,
4863            unaligned_atomic,
4864            |this, addr| this.emit_relaxed_load(Size::S8, false, ret, Location::Memory(addr, 0)),
4865        )
4866    }
4867
4868    fn i64_atomic_load_16u(
4869        &mut self,
4870        addr: Location,
4871        memarg: &MemArg,
4872        ret: Location,
4873        _need_check: bool,
4874        imported_memories: bool,
4875        offset: i32,
4876        heap_access_oob: Label,
4877        unaligned_atomic: Label,
4878    ) -> Result<(), CompileError> {
4879        self.memory_op(
4880            addr,
4881            memarg,
4882            true,
4883            2,
4884            imported_memories,
4885            offset,
4886            heap_access_oob,
4887            unaligned_atomic,
4888            |this, addr| this.emit_relaxed_load(Size::S16, false, ret, Location::Memory(addr, 0)),
4889        )
4890    }
4891
4892    fn i64_atomic_load_32u(
4893        &mut self,
4894        addr: Location,
4895        memarg: &MemArg,
4896        ret: Location,
4897        _need_check: bool,
4898        imported_memories: bool,
4899        offset: i32,
4900        heap_access_oob: Label,
4901        unaligned_atomic: Label,
4902    ) -> Result<(), CompileError> {
4903        self.memory_op(
4904            addr,
4905            memarg,
4906            true,
4907            4,
4908            imported_memories,
4909            offset,
4910            heap_access_oob,
4911            unaligned_atomic,
4912            |this, addr| this.emit_relaxed_load(Size::S32, false, ret, Location::Memory(addr, 0)),
4913        )
4914    }
4915
4916    fn i64_save(
4917        &mut self,
4918        value: Location,
4919        memarg: &MemArg,
4920        addr: Location,
4921        _need_check: bool,
4922        imported_memories: bool,
4923        offset: i32,
4924        heap_access_oob: Label,
4925        unaligned_atomic: Label,
4926    ) -> Result<(), CompileError> {
4927        self.memory_op(
4928            addr,
4929            memarg,
4930            false,
4931            8,
4932            imported_memories,
4933            offset,
4934            heap_access_oob,
4935            unaligned_atomic,
4936            |this, addr| this.emit_maybe_unaligned_store(Size::S64, value, addr),
4937        )
4938    }
4939
4940    fn i64_save_8(
4941        &mut self,
4942        value: Location,
4943        memarg: &MemArg,
4944        addr: Location,
4945        _need_check: bool,
4946        imported_memories: bool,
4947        offset: i32,
4948        heap_access_oob: Label,
4949        unaligned_atomic: Label,
4950    ) -> Result<(), CompileError> {
4951        self.memory_op(
4952            addr,
4953            memarg,
4954            false,
4955            1,
4956            imported_memories,
4957            offset,
4958            heap_access_oob,
4959            unaligned_atomic,
4960            |this, addr| this.emit_maybe_unaligned_store(Size::S8, value, addr),
4961        )
4962    }
4963
4964    fn i64_save_16(
4965        &mut self,
4966        value: Location,
4967        memarg: &MemArg,
4968        addr: Location,
4969        _need_check: bool,
4970        imported_memories: bool,
4971        offset: i32,
4972        heap_access_oob: Label,
4973        unaligned_atomic: Label,
4974    ) -> Result<(), CompileError> {
4975        self.memory_op(
4976            addr,
4977            memarg,
4978            false,
4979            2,
4980            imported_memories,
4981            offset,
4982            heap_access_oob,
4983            unaligned_atomic,
4984            |this, addr| this.emit_maybe_unaligned_store(Size::S16, value, addr),
4985        )
4986    }
4987
4988    fn i64_save_32(
4989        &mut self,
4990        value: Location,
4991        memarg: &MemArg,
4992        addr: Location,
4993        _need_check: bool,
4994        imported_memories: bool,
4995        offset: i32,
4996        heap_access_oob: Label,
4997        unaligned_atomic: Label,
4998    ) -> Result<(), CompileError> {
4999        self.memory_op(
5000            addr,
5001            memarg,
5002            false,
5003            4,
5004            imported_memories,
5005            offset,
5006            heap_access_oob,
5007            unaligned_atomic,
5008            |this, addr| this.emit_maybe_unaligned_store(Size::S32, value, addr),
5009        )
5010    }
5011
5012    fn i64_atomic_save(
5013        &mut self,
5014        value: Location,
5015        memarg: &MemArg,
5016        addr: Location,
5017        _need_check: bool,
5018        imported_memories: bool,
5019        offset: i32,
5020        heap_access_oob: Label,
5021        unaligned_atomic: Label,
5022    ) -> Result<(), CompileError> {
5023        self.memory_op(
5024            addr,
5025            memarg,
5026            true,
5027            8,
5028            imported_memories,
5029            offset,
5030            heap_access_oob,
5031            unaligned_atomic,
5032            |this, addr| this.emit_relaxed_store(Size::S64, value, Location::Memory(addr, 0)),
5033        )?;
5034        self.assembler.emit_rwfence()
5035    }
5036
5037    fn i64_atomic_save_8(
5038        &mut self,
5039        value: Location,
5040        memarg: &MemArg,
5041        addr: Location,
5042        _need_check: bool,
5043        imported_memories: bool,
5044        offset: i32,
5045        heap_access_oob: Label,
5046        unaligned_atomic: Label,
5047    ) -> Result<(), CompileError> {
5048        self.memory_op(
5049            addr,
5050            memarg,
5051            true,
5052            1,
5053            imported_memories,
5054            offset,
5055            heap_access_oob,
5056            unaligned_atomic,
5057            |this, addr| this.emit_relaxed_store(Size::S8, value, Location::Memory(addr, 0)),
5058        )?;
5059        self.assembler.emit_rwfence()
5060    }
5061
5062    fn i64_atomic_save_16(
5063        &mut self,
5064        value: Location,
5065        memarg: &MemArg,
5066        addr: Location,
5067        _need_check: bool,
5068        imported_memories: bool,
5069        offset: i32,
5070        heap_access_oob: Label,
5071        unaligned_atomic: Label,
5072    ) -> Result<(), CompileError> {
5073        self.memory_op(
5074            addr,
5075            memarg,
5076            true,
5077            2,
5078            imported_memories,
5079            offset,
5080            heap_access_oob,
5081            unaligned_atomic,
5082            |this, addr| this.emit_relaxed_store(Size::S16, value, Location::Memory(addr, 0)),
5083        )?;
5084        self.assembler.emit_rwfence()
5085    }
5086
5087    fn i64_atomic_save_32(
5088        &mut self,
5089        value: Location,
5090        memarg: &MemArg,
5091        addr: Location,
5092        _need_check: bool,
5093        imported_memories: bool,
5094        offset: i32,
5095        heap_access_oob: Label,
5096        unaligned_atomic: Label,
5097    ) -> Result<(), CompileError> {
5098        self.memory_op(
5099            addr,
5100            memarg,
5101            true,
5102            4,
5103            imported_memories,
5104            offset,
5105            heap_access_oob,
5106            unaligned_atomic,
5107            |this, addr| this.emit_relaxed_store(Size::S32, value, Location::Memory(addr, 0)),
5108        )?;
5109        self.assembler.emit_rwfence()
5110    }
5111
5112    fn i64_atomic_add(
5113        &mut self,
5114        loc: Location,
5115        target: Location,
5116        memarg: &MemArg,
5117        ret: Location,
5118        _need_check: bool,
5119        imported_memories: bool,
5120        offset: i32,
5121        heap_access_oob: Label,
5122        unaligned_atomic: Label,
5123    ) -> Result<(), CompileError> {
5124        self.memory_op(
5125            target,
5126            memarg,
5127            true,
5128            8,
5129            imported_memories,
5130            offset,
5131            heap_access_oob,
5132            unaligned_atomic,
5133            |this, addr| {
5134                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Add, Size::S64, ret, addr, loc)
5135            },
5136        )
5137    }
5138
5139    fn i64_atomic_add_8u(
5140        &mut self,
5141        loc: Location,
5142        target: Location,
5143        memarg: &MemArg,
5144        ret: Location,
5145        _need_check: bool,
5146        imported_memories: bool,
5147        offset: i32,
5148        heap_access_oob: Label,
5149        unaligned_atomic: Label,
5150    ) -> Result<(), CompileError> {
5151        self.memory_op(
5152            target,
5153            memarg,
5154            true,
5155            1,
5156            imported_memories,
5157            offset,
5158            heap_access_oob,
5159            unaligned_atomic,
5160            |this, addr| {
5161                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Add, Size::S8, ret, addr, loc)
5162            },
5163        )
5164    }
5165
5166    fn i64_atomic_add_16u(
5167        &mut self,
5168        loc: Location,
5169        target: Location,
5170        memarg: &MemArg,
5171        ret: Location,
5172        _need_check: bool,
5173        imported_memories: bool,
5174        offset: i32,
5175        heap_access_oob: Label,
5176        unaligned_atomic: Label,
5177    ) -> Result<(), CompileError> {
5178        self.memory_op(
5179            target,
5180            memarg,
5181            true,
5182            2,
5183            imported_memories,
5184            offset,
5185            heap_access_oob,
5186            unaligned_atomic,
5187            |this, addr| {
5188                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Add, Size::S16, ret, addr, loc)
5189            },
5190        )
5191    }
5192
5193    fn i64_atomic_add_32u(
5194        &mut self,
5195        loc: Location,
5196        target: Location,
5197        memarg: &MemArg,
5198        ret: Location,
5199        _need_check: bool,
5200        imported_memories: bool,
5201        offset: i32,
5202        heap_access_oob: Label,
5203        unaligned_atomic: Label,
5204    ) -> Result<(), CompileError> {
5205        self.memory_op(
5206            target,
5207            memarg,
5208            true,
5209            4,
5210            imported_memories,
5211            offset,
5212            heap_access_oob,
5213            unaligned_atomic,
5214            |this, addr| {
5215                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Add, Size::S32, ret, addr, loc)
5216            },
5217        )
5218    }
5219
5220    fn i64_atomic_sub(
5221        &mut self,
5222        loc: Location,
5223        target: Location,
5224        memarg: &MemArg,
5225        ret: Location,
5226        _need_check: bool,
5227        imported_memories: bool,
5228        offset: i32,
5229        heap_access_oob: Label,
5230        unaligned_atomic: Label,
5231    ) -> Result<(), CompileError> {
5232        self.memory_op(
5233            target,
5234            memarg,
5235            true,
5236            8,
5237            imported_memories,
5238            offset,
5239            heap_access_oob,
5240            unaligned_atomic,
5241            |this, addr| {
5242                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Sub, Size::S64, ret, addr, loc)
5243            },
5244        )
5245    }
5246
5247    fn i64_atomic_sub_8u(
5248        &mut self,
5249        loc: Location,
5250        target: Location,
5251        memarg: &MemArg,
5252        ret: Location,
5253        _need_check: bool,
5254        imported_memories: bool,
5255        offset: i32,
5256        heap_access_oob: Label,
5257        unaligned_atomic: Label,
5258    ) -> Result<(), CompileError> {
5259        self.memory_op(
5260            target,
5261            memarg,
5262            true,
5263            1,
5264            imported_memories,
5265            offset,
5266            heap_access_oob,
5267            unaligned_atomic,
5268            |this, addr| {
5269                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Sub, Size::S8, ret, addr, loc)
5270            },
5271        )
5272    }
5273
5274    fn i64_atomic_sub_16u(
5275        &mut self,
5276        loc: Location,
5277        target: Location,
5278        memarg: &MemArg,
5279        ret: Location,
5280        _need_check: bool,
5281        imported_memories: bool,
5282        offset: i32,
5283        heap_access_oob: Label,
5284        unaligned_atomic: Label,
5285    ) -> Result<(), CompileError> {
5286        self.memory_op(
5287            target,
5288            memarg,
5289            true,
5290            2,
5291            imported_memories,
5292            offset,
5293            heap_access_oob,
5294            unaligned_atomic,
5295            |this, addr| {
5296                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Sub, Size::S16, ret, addr, loc)
5297            },
5298        )
5299    }
5300
5301    fn i64_atomic_sub_32u(
5302        &mut self,
5303        loc: Location,
5304        target: Location,
5305        memarg: &MemArg,
5306        ret: Location,
5307        _need_check: bool,
5308        imported_memories: bool,
5309        offset: i32,
5310        heap_access_oob: Label,
5311        unaligned_atomic: Label,
5312    ) -> Result<(), CompileError> {
5313        self.memory_op(
5314            target,
5315            memarg,
5316            true,
5317            4,
5318            imported_memories,
5319            offset,
5320            heap_access_oob,
5321            unaligned_atomic,
5322            |this, addr| {
5323                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Sub, Size::S32, ret, addr, loc)
5324            },
5325        )
5326    }
5327
5328    fn i64_atomic_and(
5329        &mut self,
5330        loc: Location,
5331        target: Location,
5332        memarg: &MemArg,
5333        ret: Location,
5334        _need_check: bool,
5335        imported_memories: bool,
5336        offset: i32,
5337        heap_access_oob: Label,
5338        unaligned_atomic: Label,
5339    ) -> Result<(), CompileError> {
5340        self.memory_op(
5341            target,
5342            memarg,
5343            true,
5344            8,
5345            imported_memories,
5346            offset,
5347            heap_access_oob,
5348            unaligned_atomic,
5349            |this, addr| {
5350                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::And, Size::S64, ret, addr, loc)
5351            },
5352        )
5353    }
5354
5355    fn i64_atomic_and_8u(
5356        &mut self,
5357        loc: Location,
5358        target: Location,
5359        memarg: &MemArg,
5360        ret: Location,
5361        _need_check: bool,
5362        imported_memories: bool,
5363        offset: i32,
5364        heap_access_oob: Label,
5365        unaligned_atomic: Label,
5366    ) -> Result<(), CompileError> {
5367        self.memory_op(
5368            target,
5369            memarg,
5370            true,
5371            1,
5372            imported_memories,
5373            offset,
5374            heap_access_oob,
5375            unaligned_atomic,
5376            |this, addr| {
5377                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::And, Size::S8, ret, addr, loc)
5378            },
5379        )
5380    }
5381
5382    fn i64_atomic_and_16u(
5383        &mut self,
5384        loc: Location,
5385        target: Location,
5386        memarg: &MemArg,
5387        ret: Location,
5388        _need_check: bool,
5389        imported_memories: bool,
5390        offset: i32,
5391        heap_access_oob: Label,
5392        unaligned_atomic: Label,
5393    ) -> Result<(), CompileError> {
5394        self.memory_op(
5395            target,
5396            memarg,
5397            true,
5398            2,
5399            imported_memories,
5400            offset,
5401            heap_access_oob,
5402            unaligned_atomic,
5403            |this, addr| {
5404                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::And, Size::S16, ret, addr, loc)
5405            },
5406        )
5407    }
5408
5409    fn i64_atomic_and_32u(
5410        &mut self,
5411        loc: Location,
5412        target: Location,
5413        memarg: &MemArg,
5414        ret: Location,
5415        _need_check: bool,
5416        imported_memories: bool,
5417        offset: i32,
5418        heap_access_oob: Label,
5419        unaligned_atomic: Label,
5420    ) -> Result<(), CompileError> {
5421        self.memory_op(
5422            target,
5423            memarg,
5424            true,
5425            4,
5426            imported_memories,
5427            offset,
5428            heap_access_oob,
5429            unaligned_atomic,
5430            |this, addr| {
5431                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::And, Size::S32, ret, addr, loc)
5432            },
5433        )
5434    }
5435
5436    fn i64_atomic_or(
5437        &mut self,
5438        loc: Location,
5439        target: Location,
5440        memarg: &MemArg,
5441        ret: Location,
5442        _need_check: bool,
5443        imported_memories: bool,
5444        offset: i32,
5445        heap_access_oob: Label,
5446        unaligned_atomic: Label,
5447    ) -> Result<(), CompileError> {
5448        self.memory_op(
5449            target,
5450            memarg,
5451            true,
5452            8,
5453            imported_memories,
5454            offset,
5455            heap_access_oob,
5456            unaligned_atomic,
5457            |this, addr| {
5458                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Or, Size::S64, ret, addr, loc)
5459            },
5460        )
5461    }
5462
5463    fn i64_atomic_or_8u(
5464        &mut self,
5465        loc: Location,
5466        target: Location,
5467        memarg: &MemArg,
5468        ret: Location,
5469        _need_check: bool,
5470        imported_memories: bool,
5471        offset: i32,
5472        heap_access_oob: Label,
5473        unaligned_atomic: Label,
5474    ) -> Result<(), CompileError> {
5475        self.memory_op(
5476            target,
5477            memarg,
5478            true,
5479            1,
5480            imported_memories,
5481            offset,
5482            heap_access_oob,
5483            unaligned_atomic,
5484            |this, addr| {
5485                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Or, Size::S8, ret, addr, loc)
5486            },
5487        )
5488    }
5489
5490    fn i64_atomic_or_16u(
5491        &mut self,
5492        loc: Location,
5493        target: Location,
5494        memarg: &MemArg,
5495        ret: Location,
5496        _need_check: bool,
5497        imported_memories: bool,
5498        offset: i32,
5499        heap_access_oob: Label,
5500        unaligned_atomic: Label,
5501    ) -> Result<(), CompileError> {
5502        self.memory_op(
5503            target,
5504            memarg,
5505            true,
5506            2,
5507            imported_memories,
5508            offset,
5509            heap_access_oob,
5510            unaligned_atomic,
5511            |this, addr| {
5512                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Or, Size::S16, ret, addr, loc)
5513            },
5514        )
5515    }
5516
5517    fn i64_atomic_or_32u(
5518        &mut self,
5519        loc: Location,
5520        target: Location,
5521        memarg: &MemArg,
5522        ret: Location,
5523        _need_check: bool,
5524        imported_memories: bool,
5525        offset: i32,
5526        heap_access_oob: Label,
5527        unaligned_atomic: Label,
5528    ) -> Result<(), CompileError> {
5529        self.memory_op(
5530            target,
5531            memarg,
5532            true,
5533            4,
5534            imported_memories,
5535            offset,
5536            heap_access_oob,
5537            unaligned_atomic,
5538            |this, addr| {
5539                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Or, Size::S32, ret, addr, loc)
5540            },
5541        )
5542    }
5543
5544    fn i64_atomic_xor(
5545        &mut self,
5546        loc: Location,
5547        target: Location,
5548        memarg: &MemArg,
5549        ret: Location,
5550        _need_check: bool,
5551        imported_memories: bool,
5552        offset: i32,
5553        heap_access_oob: Label,
5554        unaligned_atomic: Label,
5555    ) -> Result<(), CompileError> {
5556        self.memory_op(
5557            target,
5558            memarg,
5559            true,
5560            8,
5561            imported_memories,
5562            offset,
5563            heap_access_oob,
5564            unaligned_atomic,
5565            |this, addr| {
5566                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Xor, Size::S64, ret, addr, loc)
5567            },
5568        )
5569    }
5570
5571    fn i64_atomic_xor_8u(
5572        &mut self,
5573        loc: Location,
5574        target: Location,
5575        memarg: &MemArg,
5576        ret: Location,
5577        _need_check: bool,
5578        imported_memories: bool,
5579        offset: i32,
5580        heap_access_oob: Label,
5581        unaligned_atomic: Label,
5582    ) -> Result<(), CompileError> {
5583        self.memory_op(
5584            target,
5585            memarg,
5586            true,
5587            1,
5588            imported_memories,
5589            offset,
5590            heap_access_oob,
5591            unaligned_atomic,
5592            |this, addr| {
5593                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Xor, Size::S8, ret, addr, loc)
5594            },
5595        )
5596    }
5597
5598    fn i64_atomic_xor_16u(
5599        &mut self,
5600        loc: Location,
5601        target: Location,
5602        memarg: &MemArg,
5603        ret: Location,
5604        _need_check: bool,
5605        imported_memories: bool,
5606        offset: i32,
5607        heap_access_oob: Label,
5608        unaligned_atomic: Label,
5609    ) -> Result<(), CompileError> {
5610        self.memory_op(
5611            target,
5612            memarg,
5613            true,
5614            2,
5615            imported_memories,
5616            offset,
5617            heap_access_oob,
5618            unaligned_atomic,
5619            |this, addr| {
5620                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Xor, Size::S16, ret, addr, loc)
5621            },
5622        )
5623    }
5624
5625    fn i64_atomic_xor_32u(
5626        &mut self,
5627        loc: Location,
5628        target: Location,
5629        memarg: &MemArg,
5630        ret: Location,
5631        _need_check: bool,
5632        imported_memories: bool,
5633        offset: i32,
5634        heap_access_oob: Label,
5635        unaligned_atomic: Label,
5636    ) -> Result<(), CompileError> {
5637        self.memory_op(
5638            target,
5639            memarg,
5640            true,
5641            4,
5642            imported_memories,
5643            offset,
5644            heap_access_oob,
5645            unaligned_atomic,
5646            |this, addr| {
5647                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Xor, Size::S32, ret, addr, loc)
5648            },
5649        )
5650    }
5651
5652    fn i64_atomic_xchg(
5653        &mut self,
5654        loc: Location,
5655        target: Location,
5656        memarg: &MemArg,
5657        ret: Location,
5658        _need_check: bool,
5659        imported_memories: bool,
5660        offset: i32,
5661        heap_access_oob: Label,
5662        unaligned_atomic: Label,
5663    ) -> Result<(), CompileError> {
5664        self.memory_op(
5665            target,
5666            memarg,
5667            true,
5668            8,
5669            imported_memories,
5670            offset,
5671            heap_access_oob,
5672            unaligned_atomic,
5673            |this, addr| {
5674                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Exchange, Size::S64, ret, addr, loc)
5675            },
5676        )
5677    }
5678
5679    fn i64_atomic_xchg_8u(
5680        &mut self,
5681        loc: Location,
5682        target: Location,
5683        memarg: &MemArg,
5684        ret: Location,
5685        _need_check: bool,
5686        imported_memories: bool,
5687        offset: i32,
5688        heap_access_oob: Label,
5689        unaligned_atomic: Label,
5690    ) -> Result<(), CompileError> {
5691        self.memory_op(
5692            target,
5693            memarg,
5694            true,
5695            1,
5696            imported_memories,
5697            offset,
5698            heap_access_oob,
5699            unaligned_atomic,
5700            |this, addr| {
5701                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Exchange, Size::S8, ret, addr, loc)
5702            },
5703        )
5704    }
5705
5706    fn i64_atomic_xchg_16u(
5707        &mut self,
5708        loc: Location,
5709        target: Location,
5710        memarg: &MemArg,
5711        ret: Location,
5712        _need_check: bool,
5713        imported_memories: bool,
5714        offset: i32,
5715        heap_access_oob: Label,
5716        unaligned_atomic: Label,
5717    ) -> Result<(), CompileError> {
5718        self.memory_op(
5719            target,
5720            memarg,
5721            true,
5722            2,
5723            imported_memories,
5724            offset,
5725            heap_access_oob,
5726            unaligned_atomic,
5727            |this, addr| {
5728                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Exchange, Size::S16, ret, addr, loc)
5729            },
5730        )
5731    }
5732
5733    fn i64_atomic_xchg_32u(
5734        &mut self,
5735        loc: Location,
5736        target: Location,
5737        memarg: &MemArg,
5738        ret: Location,
5739        _need_check: bool,
5740        imported_memories: bool,
5741        offset: i32,
5742        heap_access_oob: Label,
5743        unaligned_atomic: Label,
5744    ) -> Result<(), CompileError> {
5745        self.memory_op(
5746            target,
5747            memarg,
5748            true,
5749            4,
5750            imported_memories,
5751            offset,
5752            heap_access_oob,
5753            unaligned_atomic,
5754            |this, addr| {
5755                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Exchange, Size::S32, ret, addr, loc)
5756            },
5757        )
5758    }
5759
5760    fn i64_atomic_cmpxchg(
5761        &mut self,
5762        new: Location,
5763        cmp: Location,
5764        target: Location,
5765        memarg: &MemArg,
5766        ret: Location,
5767        _need_check: bool,
5768        imported_memories: bool,
5769        offset: i32,
5770        heap_access_oob: Label,
5771        unaligned_atomic: Label,
5772    ) -> Result<(), CompileError> {
5773        self.memory_op(
5774            target,
5775            memarg,
5776            true,
5777            8,
5778            imported_memories,
5779            offset,
5780            heap_access_oob,
5781            unaligned_atomic,
5782            |this, addr| this.emit_relaxed_atomic_cmpxchg(Size::S64, ret, addr, new, cmp),
5783        )
5784    }
5785
5786    fn i64_atomic_cmpxchg_8u(
5787        &mut self,
5788        new: Location,
5789        cmp: Location,
5790        target: Location,
5791        memarg: &MemArg,
5792        ret: Location,
5793        _need_check: bool,
5794        imported_memories: bool,
5795        offset: i32,
5796        heap_access_oob: Label,
5797        unaligned_atomic: Label,
5798    ) -> Result<(), CompileError> {
5799        self.memory_op(
5800            target,
5801            memarg,
5802            true,
5803            1,
5804            imported_memories,
5805            offset,
5806            heap_access_oob,
5807            unaligned_atomic,
5808            |this, addr| this.emit_relaxed_atomic_cmpxchg(Size::S8, ret, addr, new, cmp),
5809        )
5810    }
5811
5812    fn i64_atomic_cmpxchg_16u(
5813        &mut self,
5814        new: Location,
5815        cmp: Location,
5816        target: Location,
5817        memarg: &MemArg,
5818        ret: Location,
5819        _need_check: bool,
5820        imported_memories: bool,
5821        offset: i32,
5822        heap_access_oob: Label,
5823        unaligned_atomic: Label,
5824    ) -> Result<(), CompileError> {
5825        self.memory_op(
5826            target,
5827            memarg,
5828            true,
5829            2,
5830            imported_memories,
5831            offset,
5832            heap_access_oob,
5833            unaligned_atomic,
5834            |this, addr| this.emit_relaxed_atomic_cmpxchg(Size::S16, ret, addr, new, cmp),
5835        )
5836    }
5837
5838    fn i64_atomic_cmpxchg_32u(
5839        &mut self,
5840        new: Location,
5841        cmp: Location,
5842        target: Location,
5843        memarg: &MemArg,
5844        ret: Location,
5845        _need_check: bool,
5846        imported_memories: bool,
5847        offset: i32,
5848        heap_access_oob: Label,
5849        unaligned_atomic: Label,
5850    ) -> Result<(), CompileError> {
5851        self.memory_op(
5852            target,
5853            memarg,
5854            true,
5855            4,
5856            imported_memories,
5857            offset,
5858            heap_access_oob,
5859            unaligned_atomic,
5860            |this, addr| this.emit_relaxed_atomic_cmpxchg(Size::S32, ret, addr, new, cmp),
5861        )
5862    }
5863
5864    fn f32_load(
5865        &mut self,
5866        addr: Location,
5867        memarg: &MemArg,
5868        ret: Location,
5869        _need_check: bool,
5870        imported_memories: bool,
5871        offset: i32,
5872        heap_access_oob: Label,
5873        unaligned_atomic: Label,
5874    ) -> Result<(), CompileError> {
5875        self.memory_op(
5876            addr,
5877            memarg,
5878            false,
5879            4,
5880            imported_memories,
5881            offset,
5882            heap_access_oob,
5883            unaligned_atomic,
5884            |this, addr| this.emit_relaxed_load(Size::S32, false, ret, Location::Memory(addr, 0)),
5885        )
5886    }
5887
5888    fn f32_save(
5889        &mut self,
5890        value: Location,
5891        memarg: &MemArg,
5892        addr: Location,
5893        canonicalize: bool,
5894        _need_check: bool,
5895        imported_memories: bool,
5896        offset: i32,
5897        heap_access_oob: Label,
5898        unaligned_atomic: Label,
5899    ) -> Result<(), CompileError> {
5900        self.memory_op(
5901            addr,
5902            memarg,
5903            false,
5904            4,
5905            imported_memories,
5906            offset,
5907            heap_access_oob,
5908            unaligned_atomic,
5909            |this, addr| {
5910                if !canonicalize {
5911                    this.emit_relaxed_store(Size::S32, value, Location::Memory(addr, 0))
5912                } else {
5913                    this.canonicalize_nan(Size::S32, value, Location::Memory(addr, 0))
5914                }
5915            },
5916        )
5917    }
5918
5919    fn f64_load(
5920        &mut self,
5921        addr: Location,
5922        memarg: &MemArg,
5923        ret: Location,
5924        _need_check: bool,
5925        imported_memories: bool,
5926        offset: i32,
5927        heap_access_oob: Label,
5928        unaligned_atomic: Label,
5929    ) -> Result<(), CompileError> {
5930        self.memory_op(
5931            addr,
5932            memarg,
5933            false,
5934            8,
5935            imported_memories,
5936            offset,
5937            heap_access_oob,
5938            unaligned_atomic,
5939            |this, addr| this.emit_relaxed_load(Size::S64, false, ret, Location::Memory(addr, 0)),
5940        )
5941    }
5942
5943    fn f64_save(
5944        &mut self,
5945        value: Location,
5946        memarg: &MemArg,
5947        addr: Location,
5948        canonicalize: bool,
5949        _need_check: bool,
5950        imported_memories: bool,
5951        offset: i32,
5952        heap_access_oob: Label,
5953        unaligned_atomic: Label,
5954    ) -> Result<(), CompileError> {
5955        self.memory_op(
5956            addr,
5957            memarg,
5958            false,
5959            8,
5960            imported_memories,
5961            offset,
5962            heap_access_oob,
5963            unaligned_atomic,
5964            |this, addr| {
5965                if !canonicalize {
5966                    this.emit_relaxed_store(Size::S64, value, Location::Memory(addr, 0))
5967                } else {
5968                    this.canonicalize_nan(Size::S64, value, Location::Memory(addr, 0))
5969                }
5970            },
5971        )
5972    }
5973
5974    fn convert_f64_i64(
5975        &mut self,
5976        loc: Location,
5977        signed: bool,
5978        ret: Location,
5979    ) -> Result<(), CompileError> {
5980        self.convert_int_to_float(loc, Size::S64, ret, Size::S64, signed)
5981    }
5982
5983    fn convert_f64_i32(
5984        &mut self,
5985        loc: Location,
5986        signed: bool,
5987        ret: Location,
5988    ) -> Result<(), CompileError> {
5989        self.convert_int_to_float(loc, Size::S32, ret, Size::S64, signed)
5990    }
5991
5992    fn convert_f32_i64(
5993        &mut self,
5994        loc: Location,
5995        signed: bool,
5996        ret: Location,
5997    ) -> Result<(), CompileError> {
5998        self.convert_int_to_float(loc, Size::S64, ret, Size::S32, signed)
5999    }
6000
6001    fn convert_f32_i32(
6002        &mut self,
6003        loc: Location,
6004        signed: bool,
6005        ret: Location,
6006    ) -> Result<(), CompileError> {
6007        self.convert_int_to_float(loc, Size::S32, ret, Size::S32, signed)
6008    }
6009
6010    fn convert_i64_f64(
6011        &mut self,
6012        loc: Location,
6013        ret: Location,
6014        signed: bool,
6015        sat: bool,
6016    ) -> Result<(), CompileError> {
6017        self.convert_float_to_int(loc, Size::S64, ret, Size::S64, signed, sat)
6018    }
6019
6020    fn convert_i32_f64(
6021        &mut self,
6022        loc: Location,
6023        ret: Location,
6024        signed: bool,
6025        sat: bool,
6026    ) -> Result<(), CompileError> {
6027        self.convert_float_to_int(loc, Size::S64, ret, Size::S32, signed, sat)
6028    }
6029
6030    fn convert_i64_f32(
6031        &mut self,
6032        loc: Location,
6033        ret: Location,
6034        signed: bool,
6035        sat: bool,
6036    ) -> Result<(), CompileError> {
6037        self.convert_float_to_int(loc, Size::S32, ret, Size::S64, signed, sat)
6038    }
6039
6040    fn convert_i32_f32(
6041        &mut self,
6042        loc: Location,
6043        ret: Location,
6044        signed: bool,
6045        sat: bool,
6046    ) -> Result<(), CompileError> {
6047        self.convert_float_to_int(loc, Size::S32, ret, Size::S32, signed, sat)
6048    }
6049
6050    fn convert_f64_f32(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6051        self.convert_float_to_float(loc, Size::S32, ret, Size::S64)
6052    }
6053
6054    fn convert_f32_f64(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6055        self.convert_float_to_float(loc, Size::S64, ret, Size::S32)
6056    }
6057
6058    fn f64_neg(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6059        self.emit_relaxed_binop_fp(Assembler::emit_fneg, Size::S64, loc, ret, true)
6060    }
6061
6062    fn f64_abs(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6063        let tmp = self.acquire_temp_gpr().ok_or_else(|| {
6064            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6065        })?;
6066        let mask = self.acquire_temp_gpr().ok_or_else(|| {
6067            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6068        })?;
6069
6070        self.move_location(Size::S64, loc, Location::GPR(tmp))?;
6071        self.assembler
6072            .emit_mov_imm(Location::GPR(mask), 0x7fffffffffffffffi64)?;
6073        self.assembler.emit_and(
6074            Size::S64,
6075            Location::GPR(tmp),
6076            Location::GPR(mask),
6077            Location::GPR(tmp),
6078        )?;
6079        self.move_location(Size::S64, Location::GPR(tmp), ret)?;
6080
6081        self.release_gpr(tmp);
6082        self.release_gpr(mask);
6083        Ok(())
6084    }
6085
6086    fn emit_i64_copysign(&mut self, tmp1: Self::GPR, tmp2: Self::GPR) -> Result<(), CompileError> {
6087        let mask = self.acquire_temp_gpr().ok_or_else(|| {
6088            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6089        })?;
6090
6091        self.assembler
6092            .emit_mov_imm(Location::GPR(mask), 0x7fffffffffffffffu64 as _)?;
6093        self.assembler.emit_and(
6094            Size::S64,
6095            Location::GPR(tmp1),
6096            Location::GPR(mask),
6097            Location::GPR(tmp1),
6098        )?;
6099
6100        self.assembler
6101            .emit_mov_imm(Location::GPR(mask), 0x8000000000000000u64 as _)?;
6102        self.assembler.emit_and(
6103            Size::S64,
6104            Location::GPR(tmp2),
6105            Location::GPR(mask),
6106            Location::GPR(tmp2),
6107        )?;
6108
6109        self.release_gpr(mask);
6110        self.assembler.emit_or(
6111            Size::S64,
6112            Location::GPR(tmp1),
6113            Location::GPR(tmp2),
6114            Location::GPR(tmp1),
6115        )
6116    }
6117
6118    fn f64_sqrt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6119        self.emit_relaxed_binop_fp(Assembler::emit_fsqrt, Size::S64, loc, ret, true)
6120    }
6121
6122    fn f64_trunc(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6123        self.emit_relaxed_fcvt_with_rounding(RoundingMode::Rtz, Size::S64, loc, ret)
6124    }
6125
6126    fn f64_ceil(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6127        self.emit_relaxed_fcvt_with_rounding(RoundingMode::Rup, Size::S64, loc, ret)
6128    }
6129
6130    fn f64_floor(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6131        self.emit_relaxed_fcvt_with_rounding(RoundingMode::Rdn, Size::S64, loc, ret)
6132    }
6133
6134    fn f64_nearest(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6135        self.emit_relaxed_fcvt_with_rounding(RoundingMode::Rne, Size::S64, loc, ret)
6136    }
6137
6138    fn f64_cmp_ge(
6139        &mut self,
6140        loc_a: Location,
6141        loc_b: Location,
6142        ret: Location,
6143    ) -> Result<(), CompileError> {
6144        self.emit_relaxed_fcmp(Condition::Ge, Size::S64, loc_a, loc_b, ret)
6145    }
6146
6147    fn f64_cmp_gt(
6148        &mut self,
6149        loc_a: Location,
6150        loc_b: Location,
6151        ret: Location,
6152    ) -> Result<(), CompileError> {
6153        self.emit_relaxed_fcmp(Condition::Gt, Size::S64, loc_a, loc_b, ret)
6154    }
6155
6156    fn f64_cmp_le(
6157        &mut self,
6158        loc_a: Location,
6159        loc_b: Location,
6160        ret: Location,
6161    ) -> Result<(), CompileError> {
6162        self.emit_relaxed_fcmp(Condition::Le, Size::S64, loc_a, loc_b, ret)
6163    }
6164
6165    fn f64_cmp_lt(
6166        &mut self,
6167        loc_a: Location,
6168        loc_b: Location,
6169        ret: Location,
6170    ) -> Result<(), CompileError> {
6171        self.emit_relaxed_fcmp(Condition::Lt, Size::S64, loc_a, loc_b, ret)
6172    }
6173
6174    fn f64_cmp_ne(
6175        &mut self,
6176        loc_a: Location,
6177        loc_b: Location,
6178        ret: Location,
6179    ) -> Result<(), CompileError> {
6180        self.emit_relaxed_fcmp(Condition::Ne, Size::S64, loc_a, loc_b, ret)
6181    }
6182
6183    fn f64_cmp_eq(
6184        &mut self,
6185        loc_a: Location,
6186        loc_b: Location,
6187        ret: Location,
6188    ) -> Result<(), CompileError> {
6189        self.emit_relaxed_fcmp(Condition::Eq, Size::S64, loc_a, loc_b, ret)
6190    }
6191
6192    fn f64_min(
6193        &mut self,
6194        loc_a: Location,
6195        loc_b: Location,
6196        ret: Location,
6197    ) -> Result<(), CompileError> {
6198        self.emit_relaxed_binop3_fp(
6199            Assembler::emit_fmin,
6200            Size::S64,
6201            loc_a,
6202            loc_b,
6203            ret,
6204            ImmType::None,
6205            true,
6206        )
6207    }
6208
6209    fn f64_max(
6210        &mut self,
6211        loc_a: Location,
6212        loc_b: Location,
6213        ret: Location,
6214    ) -> Result<(), CompileError> {
6215        self.emit_relaxed_binop3_fp(
6216            Assembler::emit_fmax,
6217            Size::S64,
6218            loc_a,
6219            loc_b,
6220            ret,
6221            ImmType::None,
6222            true,
6223        )
6224    }
6225
6226    fn f64_add(
6227        &mut self,
6228        loc_a: Location,
6229        loc_b: Location,
6230        ret: Location,
6231    ) -> Result<(), CompileError> {
6232        self.emit_relaxed_binop3_fp(
6233            Assembler::emit_add,
6234            Size::S64,
6235            loc_a,
6236            loc_b,
6237            ret,
6238            ImmType::None,
6239            false,
6240        )
6241    }
6242
6243    fn f64_sub(
6244        &mut self,
6245        loc_a: Location,
6246        loc_b: Location,
6247        ret: Location,
6248    ) -> Result<(), CompileError> {
6249        self.emit_relaxed_binop3_fp(
6250            Assembler::emit_sub,
6251            Size::S64,
6252            loc_a,
6253            loc_b,
6254            ret,
6255            ImmType::None,
6256            false,
6257        )
6258    }
6259
6260    fn f64_mul(
6261        &mut self,
6262        loc_a: Location,
6263        loc_b: Location,
6264        ret: Location,
6265    ) -> Result<(), CompileError> {
6266        self.emit_relaxed_binop3_fp(
6267            Assembler::emit_mul,
6268            Size::S64,
6269            loc_a,
6270            loc_b,
6271            ret,
6272            ImmType::None,
6273            false,
6274        )
6275    }
6276
6277    fn f64_div(
6278        &mut self,
6279        loc_a: Location,
6280        loc_b: Location,
6281        ret: Location,
6282    ) -> Result<(), CompileError> {
6283        self.emit_relaxed_binop3_fp(
6284            Assembler::emit_fdiv,
6285            Size::S64,
6286            loc_a,
6287            loc_b,
6288            ret,
6289            ImmType::None,
6290            false,
6291        )
6292    }
6293
6294    fn f32_neg(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6295        self.emit_relaxed_binop_fp(Assembler::emit_fneg, Size::S32, loc, ret, true)
6296    }
6297
6298    fn f32_abs(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6299        let tmp = self.acquire_temp_gpr().ok_or_else(|| {
6300            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6301        })?;
6302        let mask = self.acquire_temp_gpr().ok_or_else(|| {
6303            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6304        })?;
6305
6306        self.move_location(Size::S32, loc, Location::GPR(tmp))?;
6307        self.assembler
6308            .emit_mov_imm(Location::GPR(mask), 0x7fffffffi64)?;
6309        self.assembler.emit_and(
6310            Size::S32,
6311            Location::GPR(tmp),
6312            Location::GPR(mask),
6313            Location::GPR(tmp),
6314        )?;
6315        self.move_location(Size::S32, Location::GPR(tmp), ret)?;
6316
6317        self.release_gpr(tmp);
6318        self.release_gpr(mask);
6319        Ok(())
6320    }
6321
6322    fn emit_i32_copysign(&mut self, tmp1: Self::GPR, tmp2: Self::GPR) -> Result<(), CompileError> {
6323        let mask = self.acquire_temp_gpr().ok_or_else(|| {
6324            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6325        })?;
6326
6327        self.assembler
6328            .emit_mov_imm(Location::GPR(mask), 0x7fffffffu32 as _)?;
6329        self.assembler.emit_and(
6330            Size::S32,
6331            Location::GPR(tmp1),
6332            Location::GPR(mask),
6333            Location::GPR(tmp1),
6334        )?;
6335
6336        self.assembler
6337            .emit_mov_imm(Location::GPR(mask), 0x80000000u32 as _)?;
6338        self.assembler.emit_and(
6339            Size::S32,
6340            Location::GPR(tmp2),
6341            Location::GPR(mask),
6342            Location::GPR(tmp2),
6343        )?;
6344
6345        self.release_gpr(mask);
6346        self.assembler.emit_or(
6347            Size::S32,
6348            Location::GPR(tmp1),
6349            Location::GPR(tmp2),
6350            Location::GPR(tmp1),
6351        )
6352    }
6353
6354    fn f32_sqrt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6355        self.emit_relaxed_binop_fp(Assembler::emit_fsqrt, Size::S32, loc, ret, true)
6356    }
6357
6358    fn f32_trunc(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6359        self.emit_relaxed_fcvt_with_rounding(RoundingMode::Rtz, Size::S32, loc, ret)
6360    }
6361
6362    fn f32_ceil(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6363        self.emit_relaxed_fcvt_with_rounding(RoundingMode::Rup, Size::S32, loc, ret)
6364    }
6365
6366    fn f32_floor(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6367        self.emit_relaxed_fcvt_with_rounding(RoundingMode::Rdn, Size::S32, loc, ret)
6368    }
6369
6370    fn f32_nearest(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6371        self.emit_relaxed_fcvt_with_rounding(RoundingMode::Rne, Size::S32, loc, ret)
6372    }
6373
6374    fn f32_cmp_ge(
6375        &mut self,
6376        loc_a: Location,
6377        loc_b: Location,
6378        ret: Location,
6379    ) -> Result<(), CompileError> {
6380        self.emit_relaxed_fcmp(Condition::Ge, Size::S32, loc_a, loc_b, ret)
6381    }
6382
6383    fn f32_cmp_gt(
6384        &mut self,
6385        loc_a: Location,
6386        loc_b: Location,
6387        ret: Location,
6388    ) -> Result<(), CompileError> {
6389        self.emit_relaxed_fcmp(Condition::Gt, Size::S32, loc_a, loc_b, ret)
6390    }
6391
6392    fn f32_cmp_le(
6393        &mut self,
6394        loc_a: Location,
6395        loc_b: Location,
6396        ret: Location,
6397    ) -> Result<(), CompileError> {
6398        self.emit_relaxed_fcmp(Condition::Le, Size::S32, loc_a, loc_b, ret)
6399    }
6400
6401    fn f32_cmp_lt(
6402        &mut self,
6403        loc_a: Location,
6404        loc_b: Location,
6405        ret: Location,
6406    ) -> Result<(), CompileError> {
6407        self.emit_relaxed_fcmp(Condition::Lt, Size::S32, loc_a, loc_b, ret)
6408    }
6409
6410    fn f32_cmp_ne(
6411        &mut self,
6412        loc_a: Location,
6413        loc_b: Location,
6414        ret: Location,
6415    ) -> Result<(), CompileError> {
6416        self.emit_relaxed_fcmp(Condition::Ne, Size::S32, loc_a, loc_b, ret)
6417    }
6418
6419    fn f32_cmp_eq(
6420        &mut self,
6421        loc_a: Location,
6422        loc_b: Location,
6423        ret: Location,
6424    ) -> Result<(), CompileError> {
6425        self.emit_relaxed_fcmp(Condition::Eq, Size::S32, loc_a, loc_b, ret)
6426    }
6427
6428    fn f32_min(
6429        &mut self,
6430        loc_a: Location,
6431        loc_b: Location,
6432        ret: Location,
6433    ) -> Result<(), CompileError> {
6434        self.emit_relaxed_binop3_fp(
6435            Assembler::emit_fmin,
6436            Size::S32,
6437            loc_a,
6438            loc_b,
6439            ret,
6440            ImmType::None,
6441            true,
6442        )
6443    }
6444
6445    fn f32_max(
6446        &mut self,
6447        loc_a: Location,
6448        loc_b: Location,
6449        ret: Location,
6450    ) -> Result<(), CompileError> {
6451        self.emit_relaxed_binop3_fp(
6452            Assembler::emit_fmax,
6453            Size::S32,
6454            loc_a,
6455            loc_b,
6456            ret,
6457            ImmType::None,
6458            true,
6459        )
6460    }
6461
6462    fn f32_add(
6463        &mut self,
6464        loc_a: Location,
6465        loc_b: Location,
6466        ret: Location,
6467    ) -> Result<(), CompileError> {
6468        self.emit_relaxed_binop3_fp(
6469            Assembler::emit_add,
6470            Size::S32,
6471            loc_a,
6472            loc_b,
6473            ret,
6474            ImmType::None,
6475            false,
6476        )
6477    }
6478
6479    fn f32_sub(
6480        &mut self,
6481        loc_a: Location,
6482        loc_b: Location,
6483        ret: Location,
6484    ) -> Result<(), CompileError> {
6485        self.emit_relaxed_binop3_fp(
6486            Assembler::emit_sub,
6487            Size::S32,
6488            loc_a,
6489            loc_b,
6490            ret,
6491            ImmType::None,
6492            false,
6493        )
6494    }
6495
6496    fn f32_mul(
6497        &mut self,
6498        loc_a: Location,
6499        loc_b: Location,
6500        ret: Location,
6501    ) -> Result<(), CompileError> {
6502        self.emit_relaxed_binop3_fp(
6503            Assembler::emit_mul,
6504            Size::S32,
6505            loc_a,
6506            loc_b,
6507            ret,
6508            ImmType::None,
6509            false,
6510        )
6511    }
6512
6513    fn f32_div(
6514        &mut self,
6515        loc_a: Location,
6516        loc_b: Location,
6517        ret: Location,
6518    ) -> Result<(), CompileError> {
6519        self.emit_relaxed_binop3_fp(
6520            Assembler::emit_fdiv,
6521            Size::S32,
6522            loc_a,
6523            loc_b,
6524            ret,
6525            ImmType::None,
6526            false,
6527        )
6528    }
6529
6530    fn gen_std_trampoline(
6531        &self,
6532        sig: &FunctionType,
6533        calling_convention: CallingConvention,
6534        progress_callback: Option<&CompilationProgressCallback>,
6535    ) -> Result<FunctionBody, CompileError> {
6536        gen_std_trampoline_riscv(sig, calling_convention, progress_callback)
6537    }
6538
6539    fn gen_std_dynamic_import_trampoline(
6540        &self,
6541        vmoffsets: &VMOffsets,
6542        sig: &FunctionType,
6543        _calling_convention: CallingConvention,
6544        progress_callback: Option<&CompilationProgressCallback>,
6545    ) -> Result<FunctionBody, CompileError> {
6546        gen_std_dynamic_import_trampoline_riscv(vmoffsets, sig, progress_callback)
6547    }
6548    // Singlepass calls import functions through a trampoline.
6549
6550    fn gen_import_call_trampoline(
6551        &self,
6552        vmoffsets: &VMOffsets,
6553        index: FunctionIndex,
6554        sig: &FunctionType,
6555        calling_convention: CallingConvention,
6556        progress_callback: Option<&CompilationProgressCallback>,
6557    ) -> Result<CustomSection, CompileError> {
6558        gen_import_call_trampoline_riscv(
6559            vmoffsets,
6560            index,
6561            sig,
6562            calling_convention,
6563            progress_callback,
6564        )
6565    }
6566
6567    #[cfg(feature = "unwind")]
6568    fn gen_dwarf_unwind_info(&mut self, code_len: usize) -> Option<UnwindInstructions> {
6569        let mut instructions = vec![];
6570        for &(instruction_offset, ref inst) in &self.unwind_ops {
6571            let instruction_offset = instruction_offset as u32;
6572            match *inst {
6573                UnwindOps::PushFP { up_to_sp } => {
6574                    instructions.push((
6575                        instruction_offset,
6576                        CallFrameInstruction::CfaOffset(up_to_sp as i32),
6577                    ));
6578                    instructions.push((
6579                        instruction_offset,
6580                        CallFrameInstruction::Offset(RiscV::FP, -(up_to_sp as i32)),
6581                    ));
6582                }
6583                UnwindOps::DefineNewFrame => {
6584                    instructions.push((
6585                        instruction_offset,
6586                        CallFrameInstruction::CfaRegister(RiscV::X8),
6587                    ));
6588                }
6589                UnwindOps::SaveRegister { reg, bp_neg_offset } => instructions.push((
6590                    instruction_offset,
6591                    CallFrameInstruction::Offset(reg.dwarf_index(), -bp_neg_offset),
6592                )),
6593                UnwindOps::SubtractFP { up_to_sp } => {
6594                    instructions.push((
6595                        instruction_offset,
6596                        CallFrameInstruction::CfaOffset(up_to_sp as i32),
6597                    ));
6598                }
6599                UnwindOps::Push2Regs { .. } => unimplemented!(),
6600            }
6601        }
6602        Some(UnwindInstructions {
6603            instructions,
6604            len: code_len as u32,
6605        })
6606    }
6607    #[cfg(not(feature = "unwind"))]
6608
6609    fn gen_dwarf_unwind_info(&mut self, _code_len: usize) -> Option<UnwindInstructions> {
6610        None
6611    }
6612
6613    fn gen_windows_unwind_info(&mut self, _code_len: usize) -> Option<Vec<u8>> {
6614        None
6615    }
6616}
wasmer_compiler_singlepass/machine_riscv.rs

wasmer_compiler_singlepass/
machine_riscv.rs
