wasmer_compiler_singlepass/
machine_riscv.rs

1//! RISC-V machine scaffolding.
2
3use dynasmrt::{DynasmError, VecAssembler, riscv::RiscvRelocation};
4use fixedbitset::FixedBitSet;
5#[cfg(feature = "unwind")]
6use gimli::{RiscV, write::CallFrameInstruction};
7
8use wasmer_compiler::{
9    CANONICAL_NAN_F32, CANONICAL_NAN_F64,
10    types::{
11        address_map::InstructionAddressMap,
12        function::FunctionBody,
13        relocation::{Relocation, RelocationKind, RelocationTarget},
14        section::CustomSection,
15    },
16    wasmparser::MemArg,
17};
18use wasmer_types::{
19    CompilationProgressCallback, CompileError, FunctionIndex, FunctionType, SourceLoc, TrapCode,
20    TrapInformation, VMOffsets,
21    target::{CallingConvention, Target},
22};
23
24use crate::{
25    codegen_error,
26    common_decl::*,
27    emitter_riscv::*,
28    location::{Location as AbstractLocation, Reg},
29    machine::*,
30    riscv_decl::{FPR, GPR},
31    unwind::{UnwindInstructions, UnwindOps, UnwindRegister},
32};
33
34type Assembler = VecAssembler<RiscvRelocation>;
35type Location = AbstractLocation<GPR, FPR>;
36
37use std::{
38    collections::HashMap,
39    ops::{Deref, DerefMut},
40};
41/// The RISC-V assembler wrapper, providing FPU feature tracking and a dynasmrt assembler.
42pub struct AssemblerRiscv {
43    /// Inner dynasm assembler.
44    pub inner: Assembler,
45}
46
47impl AssemblerRiscv {
48    /// Create a new RISC-V assembler.
49    pub fn new(base_addr: usize, _target: Option<Target>) -> Result<Self, CompileError> {
50        // TODO: detect RISC-V FPU extensions (e.g., F/D)
51        Ok(Self {
52            inner: Assembler::new(base_addr),
53        })
54    }
55
56    /// Finalize to machine code bytes.
57    pub fn finalize(self) -> Result<Vec<u8>, DynasmError> {
58        self.inner.finalize()
59    }
60}
61
62impl Deref for AssemblerRiscv {
63    type Target = Assembler;
64    fn deref(&self) -> &Self::Target {
65        &self.inner
66    }
67}
68
69impl DerefMut for AssemblerRiscv {
70    fn deref_mut(&mut self) -> &mut Self::Target {
71        &mut self.inner
72    }
73}
74
75/// The RISC-V machine state and code emitter.
76pub struct MachineRiscv {
77    assembler: AssemblerRiscv,
78    allow_unaligned_memory_accesses: bool,
79    used_gprs: FixedBitSet,
80    used_fprs: FixedBitSet,
81    trap_table: TrapTable,
82    /// Map from byte offset into wasm function to range of native instructions.
83    /// Ordered by increasing InstructionAddressMap::srcloc.
84    instructions_address_map: Vec<InstructionAddressMap>,
85    /// The source location for the current operator.
86    src_loc: u32,
87    /// Vector of unwind operations with offset.
88    unwind_ops: Vec<(usize, UnwindOps<GPR, FPR>)>,
89}
90
91const SCRATCH_REG: GPR = GPR::X28;
92
93impl MachineRiscv {
94    /// Creates a new RISC-V machine for code generation.
95    pub fn new(
96        target: Option<Target>,
97        allow_unaligned_memory_accesses: bool,
98    ) -> Result<Self, CompileError> {
99        // TODO: for now always require FPU
100        Ok(MachineRiscv {
101            assembler: AssemblerRiscv::new(0, target)?,
102            allow_unaligned_memory_accesses,
103            used_gprs: FixedBitSet::with_capacity(32),
104            used_fprs: FixedBitSet::with_capacity(32),
105            trap_table: TrapTable::default(),
106            instructions_address_map: vec![],
107            src_loc: 0,
108            unwind_ops: vec![],
109        })
110    }
111
112    fn used_gprs_contains(&self, r: &GPR) -> bool {
113        self.used_gprs.contains(r.into_index())
114    }
115    fn used_gprs_insert(&mut self, r: GPR) {
116        self.used_gprs.insert(r.into_index());
117    }
118    fn used_gprs_remove(&mut self, r: &GPR) -> bool {
119        let ret = self.used_gprs_contains(r);
120        self.used_gprs.set(r.into_index(), false);
121        ret
122    }
123
124    fn used_fp_contains(&self, r: &FPR) -> bool {
125        self.used_fprs.contains(r.into_index())
126    }
127    fn used_fprs_insert(&mut self, r: FPR) {
128        self.used_fprs.insert(r.into_index());
129    }
130    fn used_fprs_remove(&mut self, r: &FPR) -> bool {
131        let ret = self.used_fp_contains(r);
132        self.used_fprs.set(r.into_index(), false);
133        ret
134    }
135
136    fn location_to_reg(
137        &mut self,
138        sz: Size,
139        src: Location,
140        temps: &mut Vec<GPR>,
141        allow_imm: ImmType,
142        read_val: bool,
143        wanted: Option<GPR>,
144    ) -> Result<Location, CompileError> {
145        match src {
146            Location::GPR(_) | Location::SIMD(_) => Ok(src),
147            Location::Memory(reg, val) => {
148                let tmp = if let Some(wanted) = wanted {
149                    wanted
150                } else {
151                    let tmp = self.acquire_temp_gpr().ok_or_else(|| {
152                        CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
153                    })?;
154                    temps.push(tmp);
155                    tmp
156                };
157                if read_val {
158                    if ImmType::Bits12.compatible_imm(val as _) {
159                        self.assembler.emit_ld(sz, false, Location::GPR(tmp), src)?;
160                    } else {
161                        if reg == tmp {
162                            codegen_error!("singlepass reg == tmp unreachable");
163                        }
164                        self.assembler.emit_mov_imm(Location::GPR(tmp), val as _)?;
165                        self.assembler.emit_add(
166                            Size::S64,
167                            Location::GPR(reg),
168                            Location::GPR(tmp),
169                            Location::GPR(tmp),
170                        )?;
171                        self.assembler.emit_ld(
172                            sz,
173                            false,
174                            Location::GPR(tmp),
175                            Location::Memory(tmp, 0),
176                        )?;
177                    }
178                }
179                Ok(Location::GPR(tmp))
180            }
181            _ if src.is_imm() => {
182                let imm = src.imm_value_scalar().unwrap();
183                if imm == 0 {
184                    Ok(Location::GPR(GPR::XZero))
185                } else if allow_imm.compatible_imm(imm) {
186                    Ok(src)
187                } else {
188                    let tmp = if let Some(wanted) = wanted {
189                        wanted
190                    } else {
191                        let tmp = self.acquire_temp_gpr().ok_or_else(|| {
192                            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
193                        })?;
194                        temps.push(tmp);
195                        tmp
196                    };
197                    self.assembler.emit_mov_imm(Location::GPR(tmp), imm as _)?;
198                    Ok(Location::GPR(tmp))
199                }
200            }
201            _ => todo!("unsupported location"),
202        }
203    }
204
205    fn location_to_fpr(
206        &mut self,
207        sz: Size,
208        src: Location,
209        temps: &mut Vec<FPR>,
210        allow_imm: ImmType,
211        read_val: bool,
212    ) -> Result<Location, CompileError> {
213        match src {
214            Location::SIMD(_) => Ok(src),
215            Location::GPR(_) => {
216                let tmp = self.acquire_temp_simd().ok_or_else(|| {
217                    CompileError::Codegen("singlepass cannot acquire temp fpr".to_owned())
218                })?;
219                temps.push(tmp);
220                if read_val {
221                    self.assembler.emit_mov(sz, src, Location::SIMD(tmp))?;
222                }
223                Ok(Location::SIMD(tmp))
224            }
225            Location::Memory(_, _) => {
226                let tmp = self.acquire_temp_simd().ok_or_else(|| {
227                    CompileError::Codegen("singlepass cannot acquire temp fpr".to_owned())
228                })?;
229                temps.push(tmp);
230                if read_val {
231                    self.emit_relaxed_load(sz, false, Location::SIMD(tmp), src)?;
232                }
233                Ok(Location::SIMD(tmp))
234            }
235            _ if src.is_imm() => {
236                let tmp = self.acquire_temp_simd().ok_or_else(|| {
237                    CompileError::Codegen("singlepass cannot acquire temp fpr".to_owned())
238                })?;
239                temps.push(tmp);
240
241                let mut gpr_temps = vec![];
242                let dst =
243                    self.location_to_reg(sz, src, &mut gpr_temps, allow_imm, read_val, None)?;
244                self.assembler.emit_mov(sz, dst, Location::SIMD(tmp))?;
245                for r in gpr_temps {
246                    self.release_gpr(r);
247                }
248
249                Ok(Location::SIMD(tmp))
250            }
251            _ => todo!("unsupported location"),
252        }
253    }
254
255    fn emit_relaxed_binop(
256        &mut self,
257        op: fn(&mut Assembler, Size, Location, Location) -> Result<(), CompileError>,
258        sz: Size,
259        src: Location,
260        dst: Location,
261    ) -> Result<(), CompileError> {
262        let mut temps = vec![];
263        let src = self.location_to_reg(sz, src, &mut temps, ImmType::None, true, None)?;
264        let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
265        op(&mut self.assembler, sz, src, dest)?;
266        if dst != dest {
267            self.move_location(sz, dest, dst)?;
268        }
269        for r in temps {
270            self.release_gpr(r);
271        }
272        Ok(())
273    }
274
275    fn emit_relaxed_binop_fp(
276        &mut self,
277        op: fn(&mut Assembler, Size, Location, Location) -> Result<(), CompileError>,
278        sz: Size,
279        src: Location,
280        dst: Location,
281        putback: bool,
282    ) -> Result<(), CompileError> {
283        let mut temps = vec![];
284        let src = self.location_to_fpr(sz, src, &mut temps, ImmType::None, true)?;
285        let dest = self.location_to_fpr(sz, dst, &mut temps, ImmType::None, !putback)?;
286        op(&mut self.assembler, sz, src, dest)?;
287        if dst != dest && putback {
288            self.move_location(sz, dest, dst)?;
289        }
290        for r in temps {
291            self.release_simd(r);
292        }
293        Ok(())
294    }
295
296    fn emit_relaxed_binop3(
297        &mut self,
298        op: fn(&mut Assembler, Size, Location, Location, Location) -> Result<(), CompileError>,
299        sz: Size,
300        src1: Location,
301        src2: Location,
302        dst: Location,
303        allow_imm: ImmType,
304    ) -> Result<(), CompileError> {
305        let mut temps = vec![];
306        let src1 = self.location_to_reg(sz, src1, &mut temps, ImmType::None, true, None)?;
307        let src2 = self.location_to_reg(sz, src2, &mut temps, allow_imm, true, None)?;
308        let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
309        op(&mut self.assembler, sz, src1, src2, dest)?;
310        if dst != dest {
311            self.move_location(sz, dest, dst)?;
312        }
313        for r in temps {
314            self.release_gpr(r);
315        }
316        Ok(())
317    }
318
319    fn emit_relaxed_atomic_binop3(
320        &mut self,
321        op: AtomicBinaryOp,
322        sz: Size,
323        dst: Location,
324        addr: GPR,
325        src: Location,
326    ) -> Result<(), CompileError> {
327        let mut temps = vec![];
328        let source = self.location_to_reg(sz, src, &mut temps, ImmType::None, false, None)?;
329        let dest = self.location_to_reg(Size::S64, dst, &mut temps, ImmType::None, false, None)?;
330        let (Location::GPR(source), Location::GPR(dest)) = (source, dest) else {
331            panic!("emit_relaxed_atomic_binop3 expects locations in registers");
332        };
333
334        // RISC-V does not provide atomic operations for binary operations for S8 and S16 types.
335        // And so we must rely on 32-bit atomic operations with a proper masking.
336        match sz {
337            Size::S32 | Size::S64 => {
338                if op == AtomicBinaryOp::Sub {
339                    self.assembler.emit_neg(
340                        Size::S64,
341                        Location::GPR(source),
342                        Location::GPR(source),
343                    )?;
344                    self.assembler.emit_atomic_binop(
345                        AtomicBinaryOp::Add,
346                        sz,
347                        dest,
348                        addr,
349                        source,
350                    )?;
351                } else {
352                    self.assembler
353                        .emit_atomic_binop(op, sz, dest, addr, source)?;
354                }
355                self.assembler.emit_rwfence()?;
356            }
357            Size::S8 | Size::S16 => {
358                let aligned_addr = self.acquire_temp_gpr().ok_or_else(|| {
359                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
360                })?;
361                temps.push(aligned_addr);
362                let bit_offset = self.acquire_temp_gpr().ok_or_else(|| {
363                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
364                })?;
365                temps.push(bit_offset);
366                let bit_mask = self.acquire_temp_gpr().ok_or_else(|| {
367                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
368                })?;
369                temps.push(bit_mask);
370
371                self.assembler.emit_and(
372                    Size::S64,
373                    Location::GPR(addr),
374                    Location::Imm64(-4i64 as _),
375                    Location::GPR(aligned_addr),
376                )?;
377                self.assembler.emit_and(
378                    Size::S64,
379                    Location::GPR(addr),
380                    Location::Imm64(3),
381                    Location::GPR(bit_offset),
382                )?;
383                self.assembler.emit_sll(
384                    Size::S64,
385                    Location::GPR(bit_offset),
386                    Location::Imm64(3),
387                    Location::GPR(bit_offset),
388                )?;
389                self.assembler.emit_mov_imm(
390                    Location::GPR(bit_mask),
391                    if sz == Size::S8 {
392                        u8::MAX as _
393                    } else {
394                        u16::MAX as _
395                    },
396                )?;
397                self.assembler.emit_and(
398                    Size::S32,
399                    Location::GPR(source),
400                    Location::GPR(bit_mask),
401                    Location::GPR(source),
402                )?;
403                self.assembler.emit_sll(
404                    Size::S64,
405                    Location::GPR(bit_mask),
406                    Location::GPR(bit_offset),
407                    Location::GPR(bit_mask),
408                )?;
409                self.assembler.emit_sll(
410                    Size::S64,
411                    Location::GPR(source),
412                    Location::GPR(bit_offset),
413                    Location::GPR(source),
414                )?;
415
416                match op {
417                    AtomicBinaryOp::Add | AtomicBinaryOp::Sub | AtomicBinaryOp::Exchange => {
418                        let loaded_value = self.acquire_temp_gpr().ok_or_else(|| {
419                            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
420                        })?;
421                        temps.push(loaded_value);
422                        let tmp = self.acquire_temp_gpr().ok_or_else(|| {
423                            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
424                        })?;
425                        temps.push(tmp);
426
427                        // Loop
428                        let label_retry = self.get_label();
429                        self.emit_label(label_retry)?;
430
431                        self.assembler
432                            .emit_reserved_ld(Size::S32, loaded_value, aligned_addr)?;
433
434                        match op {
435                            AtomicBinaryOp::Add => self.assembler.emit_add(
436                                Size::S64,
437                                Location::GPR(loaded_value),
438                                Location::GPR(source),
439                                Location::GPR(tmp),
440                            )?,
441                            AtomicBinaryOp::Sub => self.assembler.emit_sub(
442                                Size::S64,
443                                Location::GPR(loaded_value),
444                                Location::GPR(source),
445                                Location::GPR(tmp),
446                            )?,
447                            AtomicBinaryOp::Exchange => self.assembler.emit_mov(
448                                Size::S64,
449                                Location::GPR(source),
450                                Location::GPR(tmp),
451                            )?,
452                            _ => unreachable!(),
453                        }
454
455                        self.assembler.emit_xor(
456                            Size::S64,
457                            Location::GPR(tmp),
458                            Location::GPR(loaded_value),
459                            Location::GPR(tmp),
460                        )?;
461                        self.assembler.emit_and(
462                            Size::S64,
463                            Location::GPR(tmp),
464                            Location::GPR(bit_mask),
465                            Location::GPR(tmp),
466                        )?;
467                        self.assembler.emit_xor(
468                            Size::S64,
469                            Location::GPR(tmp),
470                            Location::GPR(loaded_value),
471                            Location::GPR(tmp),
472                        )?;
473                        self.assembler
474                            .emit_reserved_sd(Size::S32, tmp, aligned_addr, tmp)?;
475                        // The following 2 register don't have a liveness intersection, reuse it.
476                        let tmp2 = addr;
477
478                        self.assembler
479                            .emit_on_true_label(Location::GPR(tmp), label_retry, tmp2)?;
480
481                        self.assembler.emit_rwfence()?;
482
483                        // Return the previous value
484                        self.assembler.emit_and(
485                            Size::S32,
486                            Location::GPR(loaded_value),
487                            Location::GPR(bit_mask),
488                            Location::GPR(dest),
489                        )?;
490                        self.assembler.emit_srl(
491                            Size::S32,
492                            Location::GPR(dest),
493                            Location::GPR(bit_offset),
494                            Location::GPR(dest),
495                        )?;
496                    }
497                    AtomicBinaryOp::Or | AtomicBinaryOp::Xor => {
498                        self.assembler.emit_atomic_binop(
499                            op,
500                            Size::S32,
501                            dest,
502                            aligned_addr,
503                            source,
504                        )?;
505                        self.assembler.emit_rwfence()?;
506                        self.assembler.emit_and(
507                            Size::S32,
508                            Location::GPR(dest),
509                            Location::GPR(bit_mask),
510                            Location::GPR(dest),
511                        )?;
512                        self.assembler.emit_srl(
513                            Size::S32,
514                            Location::GPR(dest),
515                            Location::GPR(bit_offset),
516                            Location::GPR(dest),
517                        )?;
518                    }
519                    AtomicBinaryOp::And => {
520                        self.assembler.emit_not(
521                            Size::S64,
522                            Location::GPR(bit_mask),
523                            Location::GPR(bit_mask),
524                        )?;
525                        self.assembler.emit_or(
526                            Size::S64,
527                            Location::GPR(bit_mask),
528                            Location::GPR(source),
529                            Location::GPR(source),
530                        )?;
531                        self.assembler.emit_not(
532                            Size::S64,
533                            Location::GPR(bit_mask),
534                            Location::GPR(bit_mask),
535                        )?;
536                        self.assembler.emit_atomic_binop(
537                            op,
538                            Size::S32,
539                            dest,
540                            aligned_addr,
541                            source,
542                        )?;
543                        self.assembler.emit_rwfence()?;
544                        self.assembler.emit_and(
545                            Size::S32,
546                            Location::GPR(dest),
547                            Location::GPR(bit_mask),
548                            Location::GPR(dest),
549                        )?;
550                        self.assembler.emit_srl(
551                            Size::S32,
552                            Location::GPR(dest),
553                            Location::GPR(bit_offset),
554                            Location::GPR(dest),
555                        )?;
556                    }
557                }
558            }
559        }
560
561        if dst != Location::GPR(dest) {
562            self.move_location(sz, Location::GPR(dest), dst)?;
563        }
564
565        for r in temps {
566            self.release_gpr(r);
567        }
568        Ok(())
569    }
570
571    fn emit_relaxed_atomic_cmpxchg(
572        &mut self,
573        size: Size,
574        dst: Location,
575        addr: GPR,
576        new: Location,
577        cmp: Location,
578    ) -> Result<(), CompileError> {
579        let mut temps = vec![];
580        let cmp = self.location_to_reg(size, cmp, &mut temps, ImmType::None, true, None)?;
581        let new = self.location_to_reg(size, new, &mut temps, ImmType::None, true, None)?;
582        let (Location::GPR(cmp), Location::GPR(new)) = (cmp, new) else {
583            panic!("emit_relaxed_atomic_cmpxchg expects locations in registers");
584        };
585
586        let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
587            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
588        })?;
589        temps.push(jmp_tmp);
590
591        match size {
592            Size::S32 | Size::S64 => {
593                let value = self.acquire_temp_gpr().ok_or_else(|| {
594                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
595                })?;
596                temps.push(value);
597                let cond = self.acquire_temp_gpr().ok_or_else(|| {
598                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
599                })?;
600                temps.push(cond);
601
602                // main re-try loop
603                let label_retry = self.get_label();
604                let label_after_retry = self.get_label();
605                self.emit_label(label_retry)?;
606
607                self.assembler.emit_reserved_ld(size, value, addr)?;
608                self.assembler.emit_cmp(
609                    Condition::Eq,
610                    Location::GPR(value),
611                    Location::GPR(cmp),
612                    Location::GPR(cond),
613                )?;
614                self.assembler.emit_on_false_label(
615                    Location::GPR(cond),
616                    label_after_retry,
617                    jmp_tmp,
618                )?;
619                self.assembler.emit_reserved_sd(size, cond, addr, new)?;
620                self.assembler
621                    .emit_on_true_label(Location::GPR(cond), label_retry, jmp_tmp)?;
622
623                // after re-try get the previous value
624                self.assembler.emit_rwfence()?;
625                self.emit_label(label_after_retry)?;
626
627                self.assembler.emit_mov(size, Location::GPR(value), dst)?;
628            }
629            Size::S8 | Size::S16 => {
630                // We're tight with the number of available registers.
631                let value = SCRATCH_REG;
632                let tmp = self.acquire_temp_gpr().ok_or_else(|| {
633                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
634                })?;
635                temps.push(tmp);
636                let bit_offset = self.acquire_temp_gpr().ok_or_else(|| {
637                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
638                })?;
639                temps.push(bit_offset);
640                let bit_mask = self.acquire_temp_gpr().ok_or_else(|| {
641                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
642                })?;
643                temps.push(bit_mask);
644
645                // before the loop
646                self.assembler.emit_and(
647                    Size::S64,
648                    Location::GPR(addr),
649                    Location::Imm64(3),
650                    Location::GPR(bit_offset),
651                )?;
652                self.assembler.emit_and(
653                    Size::S64,
654                    Location::GPR(addr),
655                    Location::Imm64(-4i64 as _),
656                    Location::GPR(addr),
657                )?;
658                self.assembler.emit_sll(
659                    Size::S64,
660                    Location::GPR(bit_offset),
661                    Location::Imm64(3),
662                    Location::GPR(bit_offset),
663                )?;
664                self.assembler.emit_mov_imm(
665                    Location::GPR(bit_mask),
666                    if size == Size::S8 {
667                        u8::MAX as _
668                    } else {
669                        u16::MAX as _
670                    },
671                )?;
672                self.assembler.emit_and(
673                    Size::S32,
674                    Location::GPR(new),
675                    Location::GPR(bit_mask),
676                    Location::GPR(new),
677                )?;
678                self.assembler.emit_sll(
679                    Size::S64,
680                    Location::GPR(bit_mask),
681                    Location::GPR(bit_offset),
682                    Location::GPR(bit_mask),
683                )?;
684                self.assembler.emit_sll(
685                    Size::S64,
686                    Location::GPR(new),
687                    Location::GPR(bit_offset),
688                    Location::GPR(new),
689                )?;
690                self.assembler.emit_sll(
691                    Size::S64,
692                    Location::GPR(cmp),
693                    Location::GPR(bit_offset),
694                    Location::GPR(cmp),
695                )?;
696
697                // main re-try loop
698                let label_retry = self.get_label();
699                let label_after_retry = self.get_label();
700                self.emit_label(label_retry)?;
701
702                self.assembler.emit_reserved_ld(Size::S32, value, addr)?;
703                self.assembler.emit_and(
704                    Size::S32,
705                    Location::GPR(value),
706                    Location::GPR(bit_mask),
707                    Location::GPR(tmp),
708                )?;
709
710                // The following 2 register don't have a liveness intersection, reuse it.
711                let cond = tmp;
712
713                self.assembler.emit_cmp(
714                    Condition::Eq,
715                    Location::GPR(tmp),
716                    Location::GPR(cmp),
717                    Location::GPR(cond),
718                )?;
719                self.assembler.emit_on_false_label(
720                    Location::GPR(cond),
721                    label_after_retry,
722                    jmp_tmp,
723                )?;
724
725                // mask new to the 4B word
726                self.assembler.emit_xor(
727                    Size::S32,
728                    Location::GPR(value),
729                    Location::GPR(new),
730                    Location::GPR(tmp),
731                )?;
732                self.assembler.emit_and(
733                    Size::S32,
734                    Location::GPR(tmp),
735                    Location::GPR(bit_mask),
736                    Location::GPR(tmp),
737                )?;
738                self.assembler.emit_xor(
739                    Size::S32,
740                    Location::GPR(tmp),
741                    Location::GPR(value),
742                    Location::GPR(tmp),
743                )?;
744                self.assembler
745                    .emit_reserved_sd(Size::S32, cond, addr, tmp)?;
746                self.assembler
747                    .emit_on_true_label(Location::GPR(cond), label_retry, jmp_tmp)?;
748
749                // After re-try get the previous value
750                self.assembler.emit_rwfence()?;
751                self.emit_label(label_after_retry)?;
752
753                self.assembler.emit_and(
754                    Size::S32,
755                    Location::GPR(value),
756                    Location::GPR(bit_mask),
757                    Location::GPR(tmp),
758                )?;
759                self.assembler.emit_srl(
760                    Size::S32,
761                    Location::GPR(tmp),
762                    Location::GPR(bit_offset),
763                    Location::GPR(tmp),
764                )?;
765                self.assembler
766                    .emit_mov(Size::S32, Location::GPR(tmp), dst)?;
767            }
768        }
769
770        for r in temps {
771            self.release_gpr(r);
772        }
773        Ok(())
774    }
775
776    #[allow(clippy::too_many_arguments)]
777    fn emit_relaxed_binop3_fp(
778        &mut self,
779        op: fn(&mut Assembler, Size, Location, Location, Location) -> Result<(), CompileError>,
780        sz: Size,
781        src1: Location,
782        src2: Location,
783        dst: Location,
784        allow_imm: ImmType,
785        return_nan_if_present: bool,
786    ) -> Result<(), CompileError> {
787        let mut temps = vec![];
788        let mut gprs = vec![];
789        let src1 = self.location_to_fpr(sz, src1, &mut temps, ImmType::None, true)?;
790        let src2 = self.location_to_fpr(sz, src2, &mut temps, allow_imm, true)?;
791        let dest = self.location_to_fpr(sz, dst, &mut temps, ImmType::None, false)?;
792
793        let label_after = self.get_label();
794        if return_nan_if_present {
795            let tmp = self.acquire_temp_gpr().ok_or_else(|| {
796                CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
797            })?;
798            gprs.push(tmp);
799            let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
800                CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
801            })?;
802            gprs.push(jmp_tmp);
803
804            // Return an ArithmeticNan if either src1 or (and) src2 have a NaN value.
805            let canonical_nan = match sz {
806                Size::S32 => CANONICAL_NAN_F32 as u64,
807                Size::S64 => CANONICAL_NAN_F64,
808                _ => unreachable!(),
809            };
810
811            self.assembler
812                .emit_mov_imm(Location::GPR(tmp), canonical_nan as _)?;
813            self.assembler.emit_mov(sz, Location::GPR(tmp), dest)?;
814
815            self.assembler
816                .emit_fcmp(Condition::Eq, sz, src1, src1, Location::GPR(tmp))?;
817            self.assembler
818                .emit_on_false_label(Location::GPR(tmp), label_after, jmp_tmp)?;
819
820            self.assembler
821                .emit_fcmp(Condition::Eq, sz, src2, src2, Location::GPR(tmp))?;
822            self.assembler
823                .emit_on_false_label(Location::GPR(tmp), label_after, jmp_tmp)?;
824        }
825
826        op(&mut self.assembler, sz, src1, src2, dest)?;
827        self.emit_label(label_after)?;
828
829        if dst != dest {
830            self.move_location(sz, dest, dst)?;
831        }
832        for r in temps {
833            self.release_simd(r);
834        }
835        for r in gprs {
836            self.release_gpr(r);
837        }
838        Ok(())
839    }
840
841    fn emit_relaxed_cmp(
842        &mut self,
843        c: Condition,
844        loc_a: Location,
845        loc_b: Location,
846        ret: Location,
847        sz: Size,
848        signed: bool,
849    ) -> Result<(), CompileError> {
850        // TODO: add support for immediate operations where some instructions (like `slti`) can be used
851        let mut temps = vec![];
852        let loc_a = self.location_to_reg(sz, loc_a, &mut temps, ImmType::None, true, None)?;
853        let loc_b = self.location_to_reg(sz, loc_b, &mut temps, ImmType::None, true, None)?;
854
855        if sz != Size::S64 {
856            self.assembler.emit_extend(sz, signed, loc_a, loc_a)?;
857            self.assembler.emit_extend(sz, signed, loc_b, loc_b)?;
858        }
859
860        let dest = self.location_to_reg(sz, ret, &mut temps, ImmType::None, false, None)?;
861        self.assembler.emit_cmp(c, loc_a, loc_b, dest)?;
862        if ret != dest {
863            self.move_location(sz, dest, ret)?;
864        }
865        for r in temps {
866            self.release_gpr(r);
867        }
868        Ok(())
869    }
870
871    /// I32 comparison with.
872    fn emit_cmpop_i32_dynamic_b(
873        &mut self,
874        c: Condition,
875        loc_a: Location,
876        loc_b: Location,
877        ret: Location,
878        signed: bool,
879    ) -> Result<(), CompileError> {
880        self.emit_relaxed_cmp(c, loc_a, loc_b, ret, Size::S32, signed)
881    }
882
883    /// I64 comparison with.
884    fn emit_cmpop_i64_dynamic_b(
885        &mut self,
886        c: Condition,
887        loc_a: Location,
888        loc_b: Location,
889        ret: Location,
890    ) -> Result<(), CompileError> {
891        self.emit_relaxed_cmp(c, loc_a, loc_b, ret, Size::S64, false)
892    }
893
894    /// NOTE: As observed on the VisionFive 2 board, when an unaligned memory write happens to write out of bounds (and thus triggers SIGSEGV),
895    /// the memory is partially modified and observable for a subsequent memory read operations.
896    /// Thus, we always check the boundaries.
897    #[allow(clippy::too_many_arguments)]
898    fn memory_op<F: FnOnce(&mut Self, GPR) -> Result<(), CompileError>>(
899        &mut self,
900        addr: Location,
901        memarg: &MemArg,
902        check_alignment: bool,
903        value_size: usize,
904        imported_memories: bool,
905        offset: i32,
906        heap_access_oob: Label,
907        unaligned_atomic: Label,
908        cb: F,
909    ) -> Result<(), CompileError> {
910        let value_size = value_size as i64;
911        let tmp_addr = self.acquire_temp_gpr().ok_or_else(|| {
912            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
913        })?;
914
915        // Reusing `tmp_addr` for temporary indirection here, since it's not used before the last reference to `{base,bound}_loc`.
916        let (base_loc, bound_loc) = if imported_memories {
917            // Imported memories require one level of indirection.
918            self.emit_relaxed_binop(
919                Assembler::emit_mov,
920                Size::S64,
921                Location::Memory(self.get_vmctx_reg(), offset),
922                Location::GPR(tmp_addr),
923            )?;
924            (Location::Memory(tmp_addr, 0), Location::Memory(tmp_addr, 8))
925        } else {
926            (
927                Location::Memory(self.get_vmctx_reg(), offset),
928                Location::Memory(self.get_vmctx_reg(), offset + 8),
929            )
930        };
931
932        let tmp_base = self.acquire_temp_gpr().ok_or_else(|| {
933            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
934        })?;
935        let tmp_bound = self.acquire_temp_gpr().ok_or_else(|| {
936            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
937        })?;
938
939        // Load base into temporary register.
940        self.emit_relaxed_load(Size::S64, false, Location::GPR(tmp_base), base_loc)?;
941
942        // Load bound into temporary register.
943        self.emit_relaxed_load(Size::S64, false, Location::GPR(tmp_bound), bound_loc)?;
944
945        // Wasm -> Effective.
946        // Assuming we never underflow - should always be true on Linux/macOS and Windows >=8,
947        // since the first page from 0x0 to 0x1000 is not accepted by mmap.
948        self.assembler.emit_add(
949            Size::S64,
950            Location::GPR(tmp_bound),
951            Location::GPR(tmp_base),
952            Location::GPR(tmp_bound),
953        )?;
954        self.assembler.emit_sub(
955            Size::S64,
956            Location::GPR(tmp_bound),
957            Location::Imm64(value_size as _),
958            Location::GPR(tmp_bound),
959        )?;
960
961        // Load effective address.
962        // `base_loc` and `bound_loc` becomes INVALID after this line, because `tmp_addr`
963        // might be reused.
964        self.move_location(Size::S32, addr, Location::GPR(tmp_addr))?;
965
966        let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
967            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
968        })?;
969
970        // Add offset to memory address.
971        if memarg.offset != 0 {
972            if ImmType::Bits12.compatible_imm(memarg.offset as _) {
973                self.assembler.emit_add(
974                    Size::S64,
975                    Location::Imm64(memarg.offset),
976                    Location::GPR(tmp_addr),
977                    Location::GPR(tmp_addr),
978                )?;
979            } else {
980                let tmp = self.acquire_temp_gpr().ok_or_else(|| {
981                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
982                })?;
983                self.assembler
984                    .emit_mov_imm(Location::GPR(tmp), memarg.offset as _)?;
985                self.assembler.emit_add(
986                    Size::S64,
987                    Location::GPR(tmp_addr),
988                    Location::GPR(tmp),
989                    Location::GPR(tmp_addr),
990                )?;
991                self.release_gpr(tmp);
992            }
993
994            // Trap if offset calculation overflowed in 32-bits by checking
995            // the upper half of the 64-bit register.
996            let tmp = self.acquire_temp_gpr().ok_or_else(|| {
997                CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
998            })?;
999            self.assembler.emit_srl(
1000                Size::S64,
1001                Location::GPR(tmp_addr),
1002                Location::Imm64(32),
1003                Location::GPR(tmp),
1004            )?;
1005            self.assembler
1006                .emit_on_true_label_far(Location::GPR(tmp), heap_access_oob, jmp_tmp)?;
1007            self.release_gpr(tmp);
1008        }
1009
1010        // Wasm linear memory -> real memory
1011        self.assembler.emit_add(
1012            Size::S64,
1013            Location::GPR(tmp_base),
1014            Location::GPR(tmp_addr),
1015            Location::GPR(tmp_addr),
1016        )?;
1017
1018        // tmp_base is already unused
1019        let cond = tmp_base;
1020
1021        // Trap if the end address of the requested area is above that of the linear memory.
1022        self.assembler.emit_cmp(
1023            Condition::Le,
1024            Location::GPR(tmp_addr),
1025            Location::GPR(tmp_bound),
1026            Location::GPR(cond),
1027        )?;
1028
1029        // `tmp_bound` is inclusive. So trap only if `tmp_addr > tmp_bound`.
1030        self.assembler
1031            .emit_on_false_label_far(Location::GPR(cond), heap_access_oob, jmp_tmp)?;
1032
1033        self.release_gpr(tmp_bound);
1034        self.release_gpr(cond);
1035
1036        let align = value_size as u32;
1037        if check_alignment && align != 1 {
1038            self.assembler.emit_and(
1039                Size::S64,
1040                Location::GPR(tmp_addr),
1041                Location::Imm64((align - 1) as u64),
1042                Location::GPR(cond),
1043            )?;
1044            self.assembler.emit_on_true_label_far(
1045                Location::GPR(cond),
1046                unaligned_atomic,
1047                jmp_tmp,
1048            )?;
1049        }
1050        let begin = self.assembler.get_offset().0;
1051        cb(self, tmp_addr)?;
1052        let end = self.assembler.get_offset().0;
1053        self.mark_address_range_with_trap_code(TrapCode::HeapAccessOutOfBounds, begin, end);
1054
1055        self.release_gpr(jmp_tmp);
1056        self.release_gpr(tmp_addr);
1057        Ok(())
1058    }
1059
1060    fn emit_relaxed_load(
1061        &mut self,
1062        sz: Size,
1063        signed: bool,
1064        dst: Location,
1065        src: Location,
1066    ) -> Result<(), CompileError> {
1067        let mut temps = vec![];
1068        let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
1069        match src {
1070            Location::Memory(addr, offset) => {
1071                if ImmType::Bits12.compatible_imm(offset as i64) {
1072                    self.assembler.emit_ld(sz, signed, dest, src)?;
1073                } else {
1074                    let tmp = self.acquire_temp_gpr().ok_or_else(|| {
1075                        CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1076                    })?;
1077                    self.assembler
1078                        .emit_mov_imm(Location::GPR(tmp), offset as i64)?;
1079                    self.assembler.emit_add(
1080                        Size::S64,
1081                        Location::GPR(addr),
1082                        Location::GPR(tmp),
1083                        Location::GPR(tmp),
1084                    )?;
1085                    self.assembler
1086                        .emit_ld(sz, signed, dest, Location::Memory(tmp, 0))?;
1087                    temps.push(tmp);
1088                }
1089            }
1090            _ => codegen_error!("singlepass emit_relaxed_load unreachable"),
1091        }
1092        if dst != dest {
1093            // Memory location is used for a local, save the entire register!
1094            self.move_location(Size::S64, dest, dst)?;
1095        }
1096        for r in temps {
1097            self.release_gpr(r);
1098        }
1099        Ok(())
1100    }
1101
1102    fn emit_maybe_unaligned_load(
1103        &mut self,
1104        sz: Size,
1105        signed: bool,
1106        dst: Location,
1107        src: GPR,
1108    ) -> Result<(), CompileError> {
1109        if !self.allow_unaligned_memory_accesses {
1110            return self.emit_relaxed_load(sz, signed, dst, Location::Memory(src, 0));
1111        }
1112
1113        if let Size::S8 = sz {
1114            return self.emit_relaxed_load(sz, signed, dst, Location::Memory(src, 0));
1115        }
1116
1117        let label_unaligned = self.get_label();
1118        let label_completed = self.get_label();
1119        let mut temps = vec![];
1120        let tmp_cond = self.acquire_temp_gpr().ok_or_else(|| {
1121            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1122        })?;
1123        temps.push(tmp_cond);
1124        let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
1125        self.assembler.emit_and(
1126            Size::S64,
1127            Location::GPR(src),
1128            Location::Imm64((sz.bytes() - 1) as u64),
1129            Location::GPR(tmp_cond),
1130        )?;
1131        let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
1132            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1133        })?;
1134        temps.push(jmp_tmp);
1135        self.assembler
1136            .emit_on_true_label(Location::GPR(tmp_cond), label_unaligned, jmp_tmp)?;
1137        // Aligned load
1138        self.assembler
1139            .emit_ld(sz, signed, dest, Location::Memory(src, 0))?;
1140        self.jmp_unconditional(label_completed)?;
1141        self.emit_label(label_unaligned)?;
1142
1143        // Unaligned load
1144        let tmp_value = tmp_cond;
1145
1146        // We assume little-endian for now
1147        self.assembler
1148            .emit_ld(Size::S8, false, dest, Location::Memory(src, 0))?;
1149        for i in 1..sz.bytes() {
1150            self.assembler.emit_ld(
1151                Size::S8,
1152                if i == sz.bytes() - 1 { signed } else { false },
1153                Location::GPR(tmp_value),
1154                Location::Memory(src, i as _),
1155            )?;
1156            self.assembler.emit_sll(
1157                Size::S64,
1158                Location::GPR(tmp_value),
1159                Location::Imm64(8 * i as u64),
1160                Location::GPR(tmp_value),
1161            )?;
1162            self.assembler
1163                .emit_or(Size::S64, dest, Location::GPR(tmp_value), dest)?;
1164        }
1165
1166        // Load completed
1167        self.emit_label(label_completed)?;
1168        if dst != dest {
1169            // Memory location is used for a local, save the entire register!
1170            self.move_location(Size::S64, dest, dst)?;
1171        }
1172
1173        for tmp in temps {
1174            self.release_gpr(tmp);
1175        }
1176        Ok(())
1177    }
1178
1179    fn emit_relaxed_store(
1180        &mut self,
1181        sz: Size,
1182        dst: Location,
1183        src: Location,
1184    ) -> Result<(), CompileError> {
1185        let mut temps = vec![];
1186        let dest = self.location_to_reg(Size::S64, dst, &mut temps, ImmType::None, true, None)?;
1187        match src {
1188            Location::Memory(addr, offset) => {
1189                if ImmType::Bits12.compatible_imm(offset as i64) {
1190                    self.assembler.emit_sd(sz, dest, src)?;
1191                } else {
1192                    let tmp = self.acquire_temp_gpr().ok_or_else(|| {
1193                        CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1194                    })?;
1195                    self.assembler
1196                        .emit_mov_imm(Location::GPR(tmp), offset as i64)?;
1197                    self.assembler.emit_add(
1198                        Size::S64,
1199                        Location::GPR(addr),
1200                        Location::GPR(tmp),
1201                        Location::GPR(tmp),
1202                    )?;
1203                    self.assembler.emit_sd(sz, dest, Location::Memory(tmp, 0))?;
1204                    temps.push(tmp);
1205                }
1206            }
1207            _ => codegen_error!("singlepass emit_relaxed_store unreachable"),
1208        }
1209        for r in temps {
1210            self.release_gpr(r);
1211        }
1212        Ok(())
1213    }
1214
1215    fn emit_maybe_unaligned_store(
1216        &mut self,
1217        sz: Size,
1218        src: Location,
1219        dst: GPR,
1220    ) -> Result<(), CompileError> {
1221        if !self.allow_unaligned_memory_accesses {
1222            return self.emit_relaxed_store(sz, src, Location::Memory(dst, 0));
1223        }
1224
1225        if let Size::S8 = sz {
1226            // `emit_relaxed_store` uses wrong order of src and dst.
1227            // The `src` parameter of `emit_relaxed_store` actually stores
1228            // the destination address, while the `dst` parameter of
1229            // `emit_relaxed_store` actually contains the source register.
1230            return self.emit_relaxed_store(sz, src, Location::Memory(dst, 0));
1231        }
1232
1233        let label_unaligned = self.get_label();
1234        let label_completed = self.get_label();
1235        let mut temps = vec![];
1236        let tmp_cond = self.acquire_temp_gpr().ok_or_else(|| {
1237            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1238        })?;
1239        temps.push(tmp_cond);
1240        let src_value =
1241            self.location_to_reg(Size::S64, src, &mut temps, ImmType::None, true, None)?;
1242        self.assembler.emit_and(
1243            Size::S64,
1244            Location::GPR(dst),
1245            Location::Imm64((sz.bytes() - 1) as u64),
1246            Location::GPR(tmp_cond),
1247        )?;
1248        let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
1249            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1250        })?;
1251        temps.push(jmp_tmp);
1252        self.assembler
1253            .emit_on_true_label(Location::GPR(tmp_cond), label_unaligned, jmp_tmp)?;
1254        // Aligned store
1255        self.assembler
1256            .emit_sd(sz, src_value, Location::Memory(dst, 0))?;
1257        self.jmp_unconditional(label_completed)?;
1258        self.emit_label(label_unaligned)?;
1259
1260        // Unaligned store
1261        let size_bytes = sz.bytes();
1262        // We assume little-endian for now
1263        for i in 0..size_bytes {
1264            self.assembler
1265                .emit_sd(Size::S8, src_value, Location::Memory(dst, i as _))?;
1266            if i != size_bytes - 1 {
1267                self.assembler
1268                    .emit_srl(Size::S64, src_value, Location::Imm64(8), src_value)?;
1269            }
1270        }
1271
1272        // Store completed
1273        self.emit_label(label_completed)?;
1274
1275        for tmp in temps {
1276            self.release_gpr(tmp);
1277        }
1278        Ok(())
1279    }
1280
1281    fn emit_rol(
1282        &mut self,
1283        sz: Size,
1284        loc_a: Location,
1285        loc_b: Location,
1286        ret: Location,
1287        allow_imm: ImmType,
1288    ) -> Result<(), CompileError> {
1289        let mut temps = vec![];
1290        let size_bits = sz.bits();
1291
1292        let src2 = if let Some(imm) = loc_b.imm_value_scalar() {
1293            Location::Imm32(size_bits - (imm as u32) % size_bits)
1294        } else {
1295            let tmp1 = self.location_to_reg(
1296                sz,
1297                Location::Imm32(size_bits),
1298                &mut temps,
1299                ImmType::None,
1300                true,
1301                None,
1302            )?;
1303            let tmp2 = self.location_to_reg(sz, loc_b, &mut temps, ImmType::None, true, None)?;
1304            self.assembler.emit_sub(sz, tmp1, tmp2, tmp1)?;
1305            tmp1
1306        };
1307
1308        self.emit_ror(sz, loc_a, src2, ret, allow_imm)?;
1309
1310        for r in temps {
1311            self.release_gpr(r);
1312        }
1313        Ok(())
1314    }
1315
1316    fn emit_ror(
1317        &mut self,
1318        sz: Size,
1319        loc_a: Location,
1320        loc_b: Location,
1321        ret: Location,
1322        allow_imm: ImmType,
1323    ) -> Result<(), CompileError> {
1324        let mut temps = vec![];
1325
1326        let imm = match sz {
1327            Size::S32 | Size::S64 => Location::Imm32(sz.bits() as u32),
1328            _ => codegen_error!("singlepass emit_ror unreachable"),
1329        };
1330        let imm = self.location_to_reg(sz, imm, &mut temps, ImmType::None, false, None)?;
1331
1332        let tmp1 = self.acquire_temp_gpr().ok_or_else(|| {
1333            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1334        })?;
1335        self.emit_relaxed_binop3(
1336            Assembler::emit_srl,
1337            sz,
1338            loc_a,
1339            loc_b,
1340            Location::GPR(tmp1),
1341            allow_imm,
1342        )?;
1343
1344        let tmp2 = self.acquire_temp_gpr().ok_or_else(|| {
1345            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1346        })?;
1347        self.emit_relaxed_binop3(
1348            Assembler::emit_sub,
1349            sz,
1350            imm,
1351            loc_b,
1352            Location::GPR(tmp2),
1353            allow_imm,
1354        )?;
1355        self.emit_relaxed_binop3(
1356            Assembler::emit_sll,
1357            sz,
1358            loc_a,
1359            Location::GPR(tmp2),
1360            Location::GPR(tmp2),
1361            ImmType::Bits12,
1362        )?;
1363        self.assembler.emit_or(
1364            sz,
1365            Location::GPR(tmp1),
1366            Location::GPR(tmp2),
1367            Location::GPR(tmp1),
1368        )?;
1369
1370        self.move_location(sz, Location::GPR(tmp1), ret)?;
1371        self.release_gpr(tmp1);
1372        self.release_gpr(tmp2);
1373        for r in temps {
1374            self.release_gpr(r);
1375        }
1376        Ok(())
1377    }
1378
1379    fn emit_popcnt(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1380        let arg = self.acquire_temp_gpr().ok_or_else(|| {
1381            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1382        })?;
1383        let cnt = self.acquire_temp_gpr().ok_or_else(|| {
1384            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1385        })?;
1386        let temp = self.acquire_temp_gpr().ok_or_else(|| {
1387            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1388        })?;
1389        self.move_location(sz, src, Location::GPR(arg))?;
1390
1391        self.move_location(sz, Location::Imm32(0), Location::GPR(cnt))?;
1392        let one_imm = match sz {
1393            Size::S32 => Location::Imm32(1),
1394            Size::S64 => Location::Imm64(1),
1395            _ => codegen_error!("singlepass emit_popcnt unreachable"),
1396        };
1397
1398        let label_loop = self.assembler.get_label();
1399        let label_exit = self.assembler.get_label();
1400
1401        self.assembler.emit_label(label_loop)?; // loop:
1402        self.assembler
1403            .emit_and(sz, Location::GPR(arg), one_imm, Location::GPR(temp))?;
1404        self.assembler.emit_add(
1405            sz,
1406            Location::GPR(cnt),
1407            Location::GPR(temp),
1408            Location::GPR(cnt),
1409        )?;
1410        self.assembler
1411            .emit_srl(sz, Location::GPR(arg), one_imm, Location::GPR(arg))?;
1412        let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
1413            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1414        })?;
1415        self.assembler
1416            .emit_on_false_label(Location::GPR(arg), label_exit, jmp_tmp)?;
1417        self.release_gpr(jmp_tmp);
1418        self.jmp_unconditional(label_loop)?;
1419
1420        self.assembler.emit_label(label_exit)?; // exit:
1421
1422        self.move_location(sz, Location::GPR(cnt), dst)?;
1423
1424        self.release_gpr(arg);
1425        self.release_gpr(cnt);
1426        self.release_gpr(temp);
1427        Ok(())
1428    }
1429
1430    fn emit_ctz(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1431        let size_bits = sz.bits();
1432        let arg = self.acquire_temp_gpr().ok_or_else(|| {
1433            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1434        })?;
1435        let cnt = self.acquire_temp_gpr().ok_or_else(|| {
1436            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1437        })?;
1438        let temp = self.acquire_temp_gpr().ok_or_else(|| {
1439            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1440        })?;
1441        self.move_location(sz, src, Location::GPR(arg))?;
1442
1443        let one_imm = match sz {
1444            Size::S32 => Location::Imm32(1),
1445            Size::S64 => Location::Imm64(1),
1446            _ => codegen_error!("singlepass emit_ctz unreachable"),
1447        };
1448
1449        let label_loop = self.assembler.get_label();
1450        let label_exit = self.assembler.get_label();
1451
1452        let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
1453            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1454        })?;
1455
1456        // if the value is zero, return size_bits
1457        self.move_location(sz, Location::Imm32(size_bits), Location::GPR(cnt))?;
1458        self.assembler
1459            .emit_on_false_label(Location::GPR(arg), label_exit, jmp_tmp)?;
1460
1461        self.move_location(sz, Location::Imm32(0), Location::GPR(cnt))?;
1462
1463        self.assembler.emit_label(label_loop)?; // loop:
1464        self.assembler
1465            .emit_and(sz, Location::GPR(arg), one_imm, Location::GPR(temp))?;
1466        self.assembler
1467            .emit_on_true_label(Location::GPR(temp), label_exit, jmp_tmp)?;
1468
1469        self.release_gpr(jmp_tmp);
1470
1471        self.assembler
1472            .emit_add(sz, Location::GPR(cnt), one_imm, Location::GPR(cnt))?;
1473        self.assembler
1474            .emit_srl(sz, Location::GPR(arg), one_imm, Location::GPR(arg))?;
1475        self.jmp_unconditional(label_loop)?;
1476
1477        self.assembler.emit_label(label_exit)?; // exit:
1478
1479        self.move_location(sz, Location::GPR(cnt), dst)?;
1480
1481        self.release_gpr(arg);
1482        self.release_gpr(cnt);
1483        self.release_gpr(temp);
1484        Ok(())
1485    }
1486
1487    fn emit_clz(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1488        let size_bits = sz.bits();
1489        let arg = self.acquire_temp_gpr().ok_or_else(|| {
1490            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1491        })?;
1492        let cnt = self.acquire_temp_gpr().ok_or_else(|| {
1493            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1494        })?;
1495        let tmp = self.acquire_temp_gpr().ok_or_else(|| {
1496            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1497        })?;
1498        self.move_location(sz, src, Location::GPR(arg))?;
1499
1500        let one_imm = match sz {
1501            Size::S32 => Location::Imm32(1),
1502            Size::S64 => Location::Imm64(1),
1503            _ => codegen_error!("singlepass emit_ctz unreachable"),
1504        };
1505
1506        let label_loop = self.assembler.get_label();
1507        let label_exit = self.assembler.get_label();
1508
1509        let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
1510            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1511        })?;
1512
1513        // if the value is zero, return size_bits
1514        self.move_location(sz, Location::Imm32(size_bits), Location::GPR(cnt))?;
1515        self.assembler
1516            .emit_on_false_label(Location::GPR(arg), label_exit, jmp_tmp)?;
1517
1518        self.move_location(sz, Location::Imm32(0), Location::GPR(cnt))?;
1519
1520        // loop:
1521        self.assembler.emit_label(label_loop)?;
1522        // Shift the argument by (bit_size - cnt - 1) and test if it's one
1523        self.move_location(
1524            Size::S32,
1525            Location::Imm32(size_bits - 1),
1526            Location::GPR(tmp),
1527        )?;
1528        self.assembler.emit_sub(
1529            Size::S32,
1530            Location::GPR(tmp),
1531            Location::GPR(cnt),
1532            Location::GPR(tmp),
1533        )?;
1534        self.assembler.emit_srl(
1535            sz,
1536            Location::GPR(arg),
1537            Location::GPR(tmp),
1538            Location::GPR(tmp),
1539        )?;
1540        self.assembler
1541            .emit_on_true_label(Location::GPR(tmp), label_exit, jmp_tmp)?;
1542
1543        self.assembler
1544            .emit_add(sz, Location::GPR(cnt), one_imm, Location::GPR(cnt))?;
1545        self.release_gpr(jmp_tmp);
1546        self.jmp_unconditional(label_loop)?;
1547
1548        self.assembler.emit_label(label_exit)?; // exit:i
1549
1550        self.move_location(sz, Location::GPR(cnt), dst)?;
1551
1552        self.release_gpr(arg);
1553        self.release_gpr(cnt);
1554        self.release_gpr(tmp);
1555        Ok(())
1556    }
1557
1558    fn convert_float_to_int(
1559        &mut self,
1560        loc: Location,
1561        size_in: Size,
1562        ret: Location,
1563        size_out: Size,
1564        signed: bool,
1565        sat: bool,
1566    ) -> Result<(), CompileError> {
1567        let mut gprs = vec![];
1568        let mut fprs = vec![];
1569        let src = self.location_to_fpr(size_in, loc, &mut fprs, ImmType::None, true)?;
1570        let dest = self.location_to_reg(size_out, ret, &mut gprs, ImmType::None, false, None)?;
1571
1572        if sat {
1573            // On RISC-V, if the input value is any NaN, the output of the operation is i32::MAX and thus we must
1574            // convert it to zero on our own.
1575            let end = self.assembler.get_label();
1576            let cond = self.acquire_temp_gpr().ok_or_else(|| {
1577                CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1578            })?;
1579            self.zero_location(size_out, dest)?;
1580            // if NaN -> skip convert operation
1581            self.assembler
1582                .emit_fcmp(Condition::Eq, size_in, src, src, Location::GPR(cond))?;
1583            let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
1584                CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1585            })?;
1586            self.assembler
1587                .emit_on_false_label(Location::GPR(cond), end, jmp_tmp)?;
1588            self.release_gpr(jmp_tmp);
1589            self.release_gpr(cond);
1590
1591            self.assembler
1592                .emit_fcvt(signed, size_in, src, size_out, dest)?;
1593            self.emit_label(end)?;
1594        } else {
1595            let old_fcsr = self.acquire_temp_gpr().ok_or_else(|| {
1596                CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1597            })?;
1598            self.move_location(
1599                Size::S32,
1600                Location::GPR(GPR::XZero),
1601                Location::GPR(old_fcsr),
1602            )?;
1603            self.assembler.emit_swap_fscr(old_fcsr)?;
1604            self.assembler
1605                .emit_fcvt(signed, size_in, src, size_out, dest)?;
1606            self.trap_float_conversion_errors(size_in, src, old_fcsr, &mut gprs)?;
1607            self.release_gpr(old_fcsr);
1608        }
1609
1610        if ret != dest {
1611            self.move_location(size_out, dest, ret)?;
1612        }
1613        for r in gprs {
1614            self.release_gpr(r);
1615        }
1616        for r in fprs {
1617            self.release_simd(r);
1618        }
1619        Ok(())
1620    }
1621
1622    fn convert_int_to_float(
1623        &mut self,
1624        loc: Location,
1625        size_in: Size,
1626        ret: Location,
1627        size_out: Size,
1628        signed: bool,
1629    ) -> Result<(), CompileError> {
1630        let mut gprs = vec![];
1631        let mut fprs = vec![];
1632        let src = self.location_to_reg(size_in, loc, &mut gprs, ImmType::None, true, None)?;
1633        let dest = self.location_to_fpr(size_out, ret, &mut fprs, ImmType::None, false)?;
1634        self.assembler
1635            .emit_fcvt(signed, size_in, src, size_out, dest)?;
1636        if ret != dest {
1637            self.move_location(Size::S32, dest, ret)?;
1638        }
1639        for r in gprs {
1640            self.release_gpr(r);
1641        }
1642        for r in fprs {
1643            self.release_simd(r);
1644        }
1645        Ok(())
1646    }
1647
1648    fn convert_float_to_float(
1649        &mut self,
1650        loc: Location,
1651        size_in: Size,
1652        ret: Location,
1653        size_out: Size,
1654    ) -> Result<(), CompileError> {
1655        let mut temps = vec![];
1656        let src = self.location_to_fpr(size_in, loc, &mut temps, ImmType::None, true)?;
1657        let dest = self.location_to_fpr(size_out, ret, &mut temps, ImmType::None, false)?;
1658
1659        match (size_in, size_out) {
1660            (Size::S32, Size::S64) => self
1661                .assembler
1662                .emit_fcvt(false, size_in, src, size_out, dest)?,
1663            (Size::S64, Size::S32) => self
1664                .assembler
1665                .emit_fcvt(false, size_in, src, size_out, dest)?,
1666            _ => codegen_error!("singlepass convert_float_to_float unreachable"),
1667        }
1668
1669        if ret != dest {
1670            self.move_location(size_out, dest, ret)?;
1671        }
1672        for r in temps {
1673            self.release_simd(r);
1674        }
1675        Ok(())
1676    }
1677
1678    fn trap_float_conversion_errors(
1679        &mut self,
1680        sz: Size,
1681        f: Location,
1682        old_fcsr: GPR,
1683        temps: &mut Vec<GPR>,
1684    ) -> Result<(), CompileError> {
1685        let fscr = self.acquire_temp_gpr().ok_or_else(|| {
1686            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1687        })?;
1688        self.zero_location(Size::S64, Location::GPR(fscr))?;
1689        temps.push(fscr);
1690
1691        let trap_badconv = self.assembler.get_label();
1692        let end = self.assembler.get_label();
1693
1694        self.assembler.emit_swap_fscr(fscr)?;
1695
1696        // The documentation link connected to the behavior connected to FCSR register: https://five-embeddev.com/riscv-user-isa-manual/Priv-v1.12/f.html.
1697        // clear all fflags bits except NV (1 << 4)
1698        self.assembler.emit_srl(
1699            Size::S32,
1700            Location::GPR(fscr),
1701            Location::Imm32(4),
1702            Location::GPR(fscr),
1703        )?;
1704
1705        let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
1706            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1707        })?;
1708        temps.push(jmp_tmp);
1709
1710        self.assembler
1711            .emit_on_false_label(Location::GPR(fscr), end, jmp_tmp)?;
1712
1713        // now need to check if it's overflow or NaN
1714        let cond = self.acquire_temp_gpr().ok_or_else(|| {
1715            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1716        })?;
1717        temps.push(cond);
1718
1719        self.assembler
1720            .emit_fcmp(Condition::Eq, sz, f, f, Location::GPR(cond))?;
1721        // fallthru: trap_overflow
1722        self.assembler
1723            .emit_on_false_label(Location::GPR(cond), trap_badconv, jmp_tmp)?;
1724        self.emit_illegal_op_internal(TrapCode::IntegerOverflow)?;
1725        self.emit_label(trap_badconv)?;
1726        self.emit_illegal_op_internal(TrapCode::BadConversionToInteger)?;
1727
1728        self.emit_label(end)?;
1729        self.assembler.emit_swap_fscr(old_fcsr)?;
1730
1731        Ok(())
1732    }
1733
1734    fn emit_illegal_op_internal(&mut self, trap: TrapCode) -> Result<(), CompileError> {
1735        self.assembler.emit_udf(trap as u8)
1736    }
1737
1738    fn emit_relaxed_fcmp(
1739        &mut self,
1740        c: Condition,
1741        size: Size,
1742        loc_a: Location,
1743        loc_b: Location,
1744        ret: Location,
1745    ) -> Result<(), CompileError> {
1746        // TODO: add support for immediate operations
1747        let mut fprs = vec![];
1748        let mut gprs = vec![];
1749
1750        let loc_a = self.location_to_fpr(size, loc_a, &mut fprs, ImmType::None, true)?;
1751        let loc_b = self.location_to_fpr(size, loc_b, &mut fprs, ImmType::None, true)?;
1752        let dest = self.location_to_reg(size, ret, &mut gprs, ImmType::None, false, None)?;
1753
1754        self.assembler.emit_fcmp(c, size, loc_a, loc_b, dest)?;
1755        if ret != dest {
1756            self.move_location(size, dest, ret)?;
1757        }
1758        for r in fprs {
1759            self.release_simd(r);
1760        }
1761        for r in gprs {
1762            self.release_gpr(r);
1763        }
1764        Ok(())
1765    }
1766
1767    fn emit_relaxed_fcvt_with_rounding(
1768        &mut self,
1769        rounding: RoundingMode,
1770        size: Size,
1771        loc: Location,
1772        ret: Location,
1773    ) -> Result<(), CompileError> {
1774        // For f64, values ≥ 2^52, the least significant bit of the significand represents 2,
1775        // so you can't represent odd integers or any fractional part.
1776        // Similarly, for f32, values ≥ 2^24 fulfil the same precondition.
1777
1778        let mut fprs = vec![];
1779        let tmp = self.acquire_temp_gpr().ok_or_else(|| {
1780            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1781        })?;
1782
1783        let loc = self.location_to_fpr(size, loc, &mut fprs, ImmType::None, true)?;
1784        let dest = self.location_to_fpr(size, ret, &mut fprs, ImmType::None, false)?;
1785
1786        let cond = self.acquire_temp_gpr().ok_or_else(|| {
1787            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1788        })?;
1789        let tmp1 = self.acquire_temp_simd().ok_or_else(|| {
1790            CompileError::Codegen("singlepass cannot acquire temp fpr".to_owned())
1791        })?;
1792        let tmp2 = self.acquire_temp_simd().ok_or_else(|| {
1793            CompileError::Codegen("singlepass cannot acquire temp fpr".to_owned())
1794        })?;
1795
1796        let label_after = self.get_label();
1797
1798        // Return an ArithmeticNan if either src1 or (and) src2 have a NaN value.
1799        let canonical_nan = match size {
1800            Size::S32 => CANONICAL_NAN_F32 as u64,
1801            Size::S64 => CANONICAL_NAN_F64,
1802            _ => unreachable!(),
1803        };
1804
1805        let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
1806            CompileError::Codegen("singlepass cannot acquire temp fpr".to_owned())
1807        })?;
1808
1809        self.assembler
1810            .emit_mov_imm(Location::GPR(tmp), canonical_nan as _)?;
1811        self.assembler.emit_mov(size, Location::GPR(tmp), dest)?;
1812        self.assembler
1813            .emit_fcmp(Condition::Eq, size, loc, loc, Location::GPR(cond))?;
1814        self.assembler
1815            .emit_on_false_label(Location::GPR(cond), label_after, jmp_tmp)?;
1816
1817        // TODO: refactor the constants
1818        if size == Size::S64 {
1819            self.assembler
1820                .emit_mov_imm(Location::GPR(cond), 0x4330000000000000)?;
1821            self.assembler
1822                .emit_mov(Size::S64, Location::GPR(cond), Location::SIMD(tmp1))?;
1823            self.f64_abs(loc, Location::SIMD(tmp2))?;
1824        } else {
1825            assert!(size == Size::S32);
1826            self.assembler
1827                .emit_mov_imm(Location::GPR(cond), 0x4b000000)?;
1828            self.assembler
1829                .emit_mov(Size::S32, Location::GPR(cond), Location::SIMD(tmp1))?;
1830            self.f32_abs(loc, Location::SIMD(tmp2))?;
1831        }
1832        self.emit_relaxed_fcmp(
1833            Condition::Lt,
1834            size,
1835            Location::SIMD(tmp2),
1836            Location::SIMD(tmp1),
1837            Location::GPR(cond),
1838        )?;
1839
1840        self.assembler.emit_mov(size, loc, dest)?;
1841        self.assembler
1842            .emit_on_false_label(Location::GPR(cond), label_after, jmp_tmp)?;
1843
1844        // Emit the actual conversion operation.
1845        self.assembler
1846            .emit_fcvt_with_rounding(rounding, size, loc, dest, cond)?;
1847        self.emit_label(label_after)?;
1848
1849        if ret != dest {
1850            self.move_location(size, dest, ret)?;
1851        }
1852
1853        for r in fprs {
1854            self.release_simd(r);
1855        }
1856        self.release_gpr(jmp_tmp);
1857        self.release_gpr(tmp);
1858        self.release_gpr(cond);
1859        self.release_simd(tmp1);
1860        self.release_simd(tmp2);
1861        Ok(())
1862    }
1863
1864    fn emit_unwind_op(&mut self, op: UnwindOps<GPR, FPR>) {
1865        self.unwind_ops.push((self.get_offset().0, op));
1866    }
1867}
1868
1869/// Get registers for first N function return values.
1870/// NOTE: The register set must be disjoint from pick_gpr registers!
1871/// Intentionally omit GPR::X17: it is reserved as a scratch/temporary register
1872/// and is frequently used while materializing/storing return values.
1873pub(crate) const RISCV_RETURN_VALUE_REGISTERS: [GPR; 7] = [
1874    GPR::X10,
1875    GPR::X11,
1876    GPR::X12,
1877    GPR::X13,
1878    GPR::X14,
1879    GPR::X15,
1880    GPR::X16,
1881];
1882
1883#[allow(dead_code)]
1884#[derive(PartialEq, Copy, Clone)]
1885pub(crate) enum ImmType {
1886    None,
1887    Bits12,
1888    // `add(w) X(rd) -imm` is used for subtraction with an immediate, so we need to check
1889    // the range of negated value.
1890    Bits12Subtraction,
1891    Shift32,
1892    Shift64,
1893}
1894
1895impl ImmType {
1896    pub(crate) fn compatible_imm(&self, imm: i64) -> bool {
1897        match self {
1898            ImmType::None => false,
1899            ImmType::Bits12 => (-0x800..0x800).contains(&imm),
1900            ImmType::Bits12Subtraction => (-0x801..0x801).contains(&imm),
1901            ImmType::Shift32 => (0..32).contains(&imm),
1902            ImmType::Shift64 => (0..64).contains(&imm),
1903        }
1904    }
1905}
1906
1907impl Machine for MachineRiscv {
1908    type GPR = GPR;
1909    type SIMD = FPR;
1910
1911    const STACK_ALIGNMENT: usize = 16;
1912
1913    fn assembler_get_offset(&self) -> Offset {
1914        self.assembler.get_offset()
1915    }
1916
1917    fn get_vmctx_reg(&self) -> Self::GPR {
1918        // Must be a callee-save register.
1919        GPR::X27
1920    }
1921
1922    fn pick_gpr(&self) -> Option<Self::GPR> {
1923        use GPR::*;
1924        // Ignore X28 as we use it as a scratch register
1925        static REGS: &[GPR] = &[X5, X6, X7, X29, X30, X31];
1926        for r in REGS {
1927            if !self.used_gprs_contains(r) {
1928                return Some(*r);
1929            }
1930        }
1931        None
1932    }
1933
1934    fn pick_temp_gpr(&self) -> Option<GPR> {
1935        use GPR::*;
1936        // Reserve a few registers for the first locals of a function!
1937        // These registers below are also used for argument passing, so they must
1938        // be saved/moved in the function prologue before being used as temps.
1939        static REGS: &[GPR] = &[X17, X16, X15, X14, X13, X12, X11, X10];
1940        for r in REGS {
1941            if !self.used_gprs_contains(r) {
1942                return Some(*r);
1943            }
1944        }
1945        None
1946    }
1947
1948    fn get_used_gprs(&self) -> Vec<Self::GPR> {
1949        GPR::iterator()
1950            .filter(|x| self.used_gprs.contains(x.into_index()))
1951            .cloned()
1952            .collect()
1953    }
1954
1955    fn get_used_simd(&self) -> Vec<Self::SIMD> {
1956        FPR::iterator()
1957            .filter(|x| self.used_fprs.contains(x.into_index()))
1958            .cloned()
1959            .collect()
1960    }
1961
1962    fn acquire_temp_gpr(&mut self) -> Option<Self::GPR> {
1963        let gpr = self.pick_temp_gpr();
1964        if let Some(x) = gpr {
1965            self.used_gprs_insert(x);
1966        }
1967        gpr
1968    }
1969
1970    fn release_gpr(&mut self, gpr: Self::GPR) {
1971        assert!(self.used_gprs_remove(&gpr));
1972    }
1973
1974    fn reserve_unused_temp_gpr(&mut self, gpr: Self::GPR) -> Self::GPR {
1975        assert!(!self.used_gprs_contains(&gpr));
1976        self.used_gprs_insert(gpr);
1977        gpr
1978    }
1979
1980    fn reserve_gpr(&mut self, gpr: Self::GPR) {
1981        self.used_gprs_insert(gpr);
1982    }
1983
1984    fn push_used_gpr(&mut self, used_gprs: &[Self::GPR]) -> Result<usize, CompileError> {
1985        for r in used_gprs.iter() {
1986            self.assembler.emit_push(Size::S64, Location::GPR(*r))?;
1987        }
1988        Ok(used_gprs.len() * 16)
1989    }
1990
1991    fn pop_used_gpr(&mut self, used_gprs: &[Self::GPR]) -> Result<(), CompileError> {
1992        for r in used_gprs.iter().rev() {
1993            self.emit_pop(Size::S64, Location::GPR(*r))?;
1994        }
1995        Ok(())
1996    }
1997
1998    fn pick_simd(&self) -> Option<Self::SIMD> {
1999        use FPR::*;
2000        static REGS: &[FPR] = &[F0, F1, F2, F3, F4, F5, F6, F7];
2001        for r in REGS {
2002            if !self.used_fp_contains(r) {
2003                return Some(*r);
2004            }
2005        }
2006        None
2007    }
2008
2009    fn pick_temp_simd(&self) -> Option<FPR> {
2010        use FPR::*;
2011        static REGS: &[FPR] = &[F28, F29, F31];
2012        for r in REGS {
2013            if !self.used_fp_contains(r) {
2014                return Some(*r);
2015            }
2016        }
2017        None
2018    }
2019
2020    fn acquire_temp_simd(&mut self) -> Option<Self::SIMD> {
2021        let fpr = self.pick_temp_simd();
2022        if let Some(x) = fpr {
2023            self.used_fprs_insert(x);
2024        }
2025        fpr
2026    }
2027
2028    fn reserve_simd(&mut self, fpr: Self::SIMD) {
2029        self.used_fprs_insert(fpr);
2030    }
2031
2032    fn release_simd(&mut self, fpr: Self::SIMD) {
2033        assert!(self.used_fprs_remove(&fpr));
2034    }
2035
2036    fn push_used_simd(&mut self, used_neons: &[Self::SIMD]) -> Result<usize, CompileError> {
2037        let stack_adjust = (used_neons.len() * 8) as u32;
2038        self.extend_stack(stack_adjust)?;
2039
2040        for (i, r) in used_neons.iter().enumerate() {
2041            self.assembler.emit_sd(
2042                Size::S64,
2043                Location::SIMD(*r),
2044                Location::Memory(GPR::Sp, (i * 8) as i32),
2045            )?;
2046        }
2047        Ok(stack_adjust as usize)
2048    }
2049
2050    fn pop_used_simd(&mut self, used_neons: &[Self::SIMD]) -> Result<(), CompileError> {
2051        for (i, r) in used_neons.iter().enumerate() {
2052            self.assembler.emit_ld(
2053                Size::S64,
2054                false,
2055                Location::SIMD(*r),
2056                Location::Memory(GPR::Sp, (i * 8) as i32),
2057            )?;
2058        }
2059        let stack_adjust = (used_neons.len() * 8) as u32;
2060        self.assembler.emit_add(
2061            Size::S64,
2062            Location::GPR(GPR::Sp),
2063            Location::Imm64(stack_adjust as _),
2064            Location::GPR(GPR::Sp),
2065        )
2066    }
2067
2068    fn set_srcloc(&mut self, offset: u32) {
2069        self.src_loc = offset;
2070    }
2071
2072    fn mark_address_range_with_trap_code(&mut self, code: TrapCode, begin: usize, end: usize) {
2073        for i in begin..end {
2074            self.trap_table.offset_to_code.insert(i, code);
2075        }
2076        self.mark_instruction_address_end(begin);
2077    }
2078
2079    fn mark_address_with_trap_code(&mut self, code: TrapCode) {
2080        let offset = self.assembler.get_offset().0;
2081        self.trap_table.offset_to_code.insert(offset, code);
2082        self.mark_instruction_address_end(offset);
2083    }
2084
2085    fn mark_instruction_with_trap_code(&mut self, code: TrapCode) -> usize {
2086        let offset = self.assembler.get_offset().0;
2087        self.trap_table.offset_to_code.insert(offset, code);
2088        offset
2089    }
2090
2091    fn mark_instruction_address_end(&mut self, begin: usize) {
2092        self.instructions_address_map.push(InstructionAddressMap {
2093            srcloc: SourceLoc::new(self.src_loc),
2094            code_offset: begin,
2095            code_len: self.assembler.get_offset().0 - begin,
2096        });
2097    }
2098
2099    fn insert_stackoverflow(&mut self) {
2100        let offset = 0;
2101        self.trap_table
2102            .offset_to_code
2103            .insert(offset, TrapCode::StackOverflow);
2104        self.mark_instruction_address_end(offset);
2105    }
2106
2107    fn collect_trap_information(&self) -> Vec<TrapInformation> {
2108        self.trap_table
2109            .offset_to_code
2110            .clone()
2111            .into_iter()
2112            .map(|(offset, code)| TrapInformation {
2113                code_offset: offset as u32,
2114                trap_code: code,
2115            })
2116            .collect()
2117    }
2118
2119    fn instructions_address_map(&self) -> Vec<InstructionAddressMap> {
2120        self.instructions_address_map.clone()
2121    }
2122
2123    fn local_on_stack(&mut self, stack_offset: i32) -> Location {
2124        Location::Memory(GPR::Fp, -stack_offset)
2125    }
2126
2127    fn extend_stack(&mut self, delta_stack_offset: u32) -> Result<(), CompileError> {
2128        let delta = if ImmType::Bits12Subtraction.compatible_imm(delta_stack_offset as _) {
2129            Location::Imm64(delta_stack_offset as _)
2130        } else {
2131            self.assembler
2132                .emit_mov_imm(Location::GPR(SCRATCH_REG), delta_stack_offset as _)?;
2133            Location::GPR(SCRATCH_REG)
2134        };
2135        self.assembler.emit_sub(
2136            Size::S64,
2137            Location::GPR(GPR::Sp),
2138            delta,
2139            Location::GPR(GPR::Sp),
2140        )
2141    }
2142
2143    fn truncate_stack(&mut self, delta_stack_offset: u32) -> Result<(), CompileError> {
2144        let delta = if ImmType::Bits12.compatible_imm(delta_stack_offset as _) {
2145            Location::Imm64(delta_stack_offset as _)
2146        } else {
2147            self.assembler
2148                .emit_mov_imm(Location::GPR(SCRATCH_REG), delta_stack_offset as _)?;
2149            Location::GPR(SCRATCH_REG)
2150        };
2151        self.assembler.emit_add(
2152            Size::S64,
2153            Location::GPR(GPR::Sp),
2154            delta,
2155            Location::GPR(GPR::Sp),
2156        )
2157    }
2158
2159    fn zero_location(&mut self, size: Size, location: Location) -> Result<(), CompileError> {
2160        self.move_location(size, Location::GPR(GPR::XZero), location)
2161    }
2162
2163    fn local_pointer(&self) -> Self::GPR {
2164        GPR::Fp
2165    }
2166
2167    fn move_location_for_native(
2168        &mut self,
2169        _size: Size,
2170        loc: Location,
2171        dest: Location,
2172    ) -> Result<(), CompileError> {
2173        match loc {
2174            Location::Imm64(_)
2175            | Location::Imm32(_)
2176            | Location::Imm8(_)
2177            | Location::Memory(_, _) => {
2178                self.move_location(Size::S64, loc, Location::GPR(SCRATCH_REG))?;
2179                self.move_location(Size::S64, Location::GPR(SCRATCH_REG), dest)
2180            }
2181            _ => self.move_location(Size::S64, loc, dest),
2182        }
2183    }
2184
2185    fn is_local_on_stack(&self, idx: usize) -> bool {
2186        idx > 9
2187    }
2188
2189    fn get_local_location(&self, idx: usize, callee_saved_regs_size: usize) -> Location {
2190        // Use callee-saved registers for the first locals.
2191        match idx {
2192            0 => Location::GPR(GPR::X9),
2193            1 => Location::GPR(GPR::X18),
2194            2 => Location::GPR(GPR::X19),
2195            3 => Location::GPR(GPR::X20),
2196            4 => Location::GPR(GPR::X21),
2197            5 => Location::GPR(GPR::X22),
2198            6 => Location::GPR(GPR::X23),
2199            7 => Location::GPR(GPR::X24),
2200            8 => Location::GPR(GPR::X25),
2201            9 => Location::GPR(GPR::X26),
2202            _ => Location::Memory(GPR::Fp, -(((idx - 9) * 8 + callee_saved_regs_size) as i32)),
2203        }
2204    }
2205
2206    fn move_local(&mut self, stack_offset: i32, location: Location) -> Result<(), CompileError> {
2207        self.move_location(
2208            Size::S64,
2209            location,
2210            Location::Memory(GPR::Fp, -stack_offset),
2211        )?;
2212
2213        match location {
2214            Location::GPR(x) => self.emit_unwind_op(UnwindOps::SaveRegister {
2215                reg: UnwindRegister::GPR(x),
2216                bp_neg_offset: stack_offset,
2217            }),
2218            Location::SIMD(x) => self.emit_unwind_op(UnwindOps::SaveRegister {
2219                reg: UnwindRegister::FPR(x),
2220                bp_neg_offset: stack_offset,
2221            }),
2222            _ => (),
2223        }
2224        Ok(())
2225    }
2226
2227    fn get_param_registers(&self) -> &'static [Self::GPR] {
2228        &[
2229            GPR::X10,
2230            GPR::X11,
2231            GPR::X12,
2232            GPR::X13,
2233            GPR::X14,
2234            GPR::X15,
2235            GPR::X16,
2236            GPR::X17,
2237        ]
2238    }
2239
2240    fn get_param_location(
2241        &self,
2242        idx: usize,
2243        _sz: Size,
2244        stack_args: &mut usize,
2245        _calling_convention: CallingConvention,
2246    ) -> Location {
2247        let register_params: &[GPR] = self.get_param_registers();
2248        if let Some(reg) = register_params.get(idx) {
2249            Location::GPR(*reg)
2250        } else {
2251            let loc = Location::Memory(GPR::Sp, *stack_args as i32);
2252            *stack_args += 8;
2253            loc
2254        }
2255    }
2256
2257    fn get_call_param_location(
2258        &self,
2259        return_slots: usize,
2260        idx: usize,
2261        _sz: Size,
2262        stack_args: &mut usize,
2263        _calling_convention: CallingConvention,
2264    ) -> Location {
2265        let return_values_memory_size =
2266            8 * return_slots.saturating_sub(RISCV_RETURN_VALUE_REGISTERS.len()) as i32;
2267        self.get_param_registers().get(idx).map_or_else(
2268            || {
2269                let loc =
2270                    Location::Memory(GPR::Fp, 16 + return_values_memory_size + *stack_args as i32);
2271                *stack_args += 8;
2272                loc
2273            },
2274            |reg| Location::GPR(*reg),
2275        )
2276    }
2277
2278    fn get_simple_param_location(&self, idx: usize) -> Self::GPR {
2279        self.get_param_registers()[idx]
2280    }
2281
2282    fn adjust_gpr_param_location(
2283        &mut self,
2284        register: Self::GPR,
2285        size: Size,
2286    ) -> Result<(), CompileError> {
2287        // https://five-embeddev.com/riscv-user-isa-manual/Priv-v1.12/rv64.html
2288        // > The compiler and calling convention maintain an invariant that all 32-bit values are held in a sign-extended format in 64-bit registers.
2289        // > Even 32-bit unsigned integers extend bit 31 into bits 63 through 32. Consequently, conversion between unsigned and signed 32-bit integers
2290        // > is a no-op, as is conversion from a signed 32-bit integer to a signed 64-bit integer.
2291        match size {
2292            Size::S64 => Ok(()),
2293            Size::S32 => self.assembler.emit_extend(
2294                Size::S32,
2295                true,
2296                Location::GPR(register),
2297                Location::GPR(register),
2298            ),
2299            Size::S8 | Size::S16 => {
2300                codegen_error!("singlepass adjust_gpr_param_location unreachable")
2301            }
2302        }
2303    }
2304
2305    fn get_return_value_location(
2306        &self,
2307        idx: usize,
2308        stack_location: &mut usize,
2309    ) -> AbstractLocation<Self::GPR, Self::SIMD> {
2310        RISCV_RETURN_VALUE_REGISTERS.get(idx).map_or_else(
2311            || {
2312                let loc = Location::Memory(GPR::Sp, *stack_location as i32);
2313                *stack_location += 8;
2314                loc
2315            },
2316            |reg| Location::GPR(*reg),
2317        )
2318    }
2319
2320    fn get_call_return_value_location(
2321        &self,
2322        idx: usize,
2323    ) -> AbstractLocation<Self::GPR, Self::SIMD> {
2324        RISCV_RETURN_VALUE_REGISTERS.get(idx).map_or_else(
2325            || {
2326                Location::Memory(
2327                    GPR::Fp,
2328                    (16 + (idx - RISCV_RETURN_VALUE_REGISTERS.len()) * 8) as i32,
2329                )
2330            },
2331            |reg| Location::GPR(*reg),
2332        )
2333    }
2334
2335    fn move_location(
2336        &mut self,
2337        size: Size,
2338        source: Location,
2339        dest: Location,
2340    ) -> Result<(), CompileError> {
2341        match (source, dest) {
2342            (Location::GPR(_), Location::GPR(_)) => self.assembler.emit_mov(size, source, dest),
2343            (Location::Imm32(_) | Location::Imm64(_), Location::GPR(_)) => self
2344                .assembler
2345                .emit_mov_imm(dest, source.imm_value_scalar().unwrap()),
2346            (Location::GPR(_), Location::Memory(addr, offset)) => {
2347                if ImmType::Bits12.compatible_imm(offset as _) {
2348                    self.assembler.emit_sd(size, source, dest)
2349                } else {
2350                    let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2351                        CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2352                    })?;
2353                    self.assembler
2354                        .emit_mov_imm(Location::GPR(tmp), offset as _)?;
2355                    self.assembler.emit_add(
2356                        Size::S64,
2357                        Location::GPR(addr),
2358                        Location::GPR(tmp),
2359                        Location::GPR(tmp),
2360                    )?;
2361                    self.assembler
2362                        .emit_sd(size, source, Location::Memory(tmp, 0))?;
2363                    self.release_gpr(tmp);
2364                    Ok(())
2365                }
2366            }
2367            (Location::SIMD(_), Location::Memory(..)) => {
2368                let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2369                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2370                })?;
2371                self.move_location(size, source, Location::GPR(tmp))?;
2372                self.move_location(size, Location::GPR(tmp), dest)?;
2373                self.release_gpr(tmp);
2374                Ok(())
2375            }
2376            (Location::Memory(addr, offset), Location::GPR(_)) => {
2377                let addr = if ImmType::Bits12.compatible_imm(offset as _) {
2378                    source
2379                } else {
2380                    self.assembler
2381                        .emit_mov_imm(Location::GPR(SCRATCH_REG), offset as _)?;
2382                    self.assembler.emit_add(
2383                        Size::S64,
2384                        Location::GPR(addr),
2385                        Location::GPR(SCRATCH_REG),
2386                        Location::GPR(SCRATCH_REG),
2387                    )?;
2388                    Location::Memory(SCRATCH_REG, 0)
2389                };
2390                self.assembler.emit_ld(size, false, dest, addr)
2391            }
2392            (Location::GPR(_), Location::SIMD(_)) => self.assembler.emit_mov(size, source, dest),
2393            (Location::SIMD(_), Location::GPR(_)) => self.assembler.emit_mov(size, source, dest),
2394            (Location::SIMD(_), Location::SIMD(_)) => self.assembler.emit_mov(size, source, dest),
2395            _ => todo!("unsupported move: {size:?} {source:?} {dest:?}"),
2396        }
2397    }
2398
2399    fn move_location_extend(
2400        &mut self,
2401        size_val: Size,
2402        signed: bool,
2403        source: Location,
2404        size_op: Size,
2405        dest: Location,
2406    ) -> Result<(), CompileError> {
2407        if size_op != Size::S64 {
2408            codegen_error!("singlepass move_location_extend unreachable");
2409        }
2410        let mut temps = vec![];
2411        let dst = self.location_to_reg(size_op, dest, &mut temps, ImmType::None, false, None)?;
2412        let src = match (size_val, signed, source) {
2413            (Size::S64, _, _) => source,
2414            (_, _, Location::GPR(_)) => {
2415                self.assembler.emit_extend(size_val, signed, source, dst)?;
2416                dst
2417            }
2418            (_, _, Location::Memory(_, _)) => {
2419                self.emit_relaxed_load(size_val, signed, dst, source)?;
2420                dst
2421            }
2422            _ => codegen_error!(
2423                "singlepass can't emit move_location_extend {:?} {:?} {:?} => {:?} {:?}",
2424                size_val,
2425                signed,
2426                source,
2427                size_op,
2428                dest
2429            ),
2430        };
2431        if src != dst {
2432            self.move_location(size_op, src, dst)?;
2433        }
2434        if dst != dest {
2435            self.move_location(size_op, dst, dest)?;
2436        }
2437        for r in temps {
2438            self.release_gpr(r);
2439        }
2440        Ok(())
2441    }
2442
2443    fn init_stack_loc(
2444        &mut self,
2445        init_stack_loc_cnt: u64,
2446        last_stack_loc: Location,
2447    ) -> Result<(), CompileError> {
2448        let label = self.assembler.get_label();
2449        let mut temps = vec![];
2450        let dest = self.acquire_temp_gpr().ok_or_else(|| {
2451            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2452        })?;
2453        temps.push(dest);
2454        let cnt = self.location_to_reg(
2455            Size::S64,
2456            Location::Imm64(init_stack_loc_cnt),
2457            &mut temps,
2458            ImmType::None,
2459            true,
2460            None,
2461        )?;
2462        match last_stack_loc {
2463            Location::GPR(_) => codegen_error!("singlepass init_stack_loc unreachable"),
2464            Location::SIMD(_) => codegen_error!("singlepass init_stack_loc unreachable"),
2465            Location::Memory(reg, offset) => {
2466                if ImmType::Bits12.compatible_imm(offset as _) {
2467                    self.assembler.emit_add(
2468                        Size::S64,
2469                        Location::GPR(reg),
2470                        Location::Imm64(offset as _),
2471                        Location::GPR(dest),
2472                    )?;
2473                } else {
2474                    let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2475                        CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2476                    })?;
2477                    self.assembler
2478                        .emit_mov_imm(Location::GPR(tmp), offset as _)?;
2479                    self.assembler.emit_add(
2480                        Size::S64,
2481                        Location::GPR(reg),
2482                        Location::GPR(tmp),
2483                        Location::GPR(dest),
2484                    )?;
2485                    temps.push(tmp);
2486                }
2487            }
2488            _ => codegen_error!("singlepass can't emit init_stack_loc {:?}", last_stack_loc),
2489        };
2490        self.assembler.emit_label(label)?;
2491        self.assembler.emit_sd(
2492            Size::S64,
2493            Location::GPR(GPR::XZero),
2494            Location::Memory(dest, 0),
2495        )?;
2496        self.assembler
2497            .emit_sub(Size::S64, cnt, Location::Imm64(1), cnt)?;
2498        self.assembler.emit_add(
2499            Size::S64,
2500            Location::GPR(dest),
2501            Location::Imm64(8),
2502            Location::GPR(dest),
2503        )?;
2504        let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
2505            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2506        })?;
2507        temps.push(jmp_tmp);
2508        self.assembler.emit_on_true_label(cnt, label, jmp_tmp)?;
2509        for r in temps {
2510            self.release_gpr(r);
2511        }
2512        Ok(())
2513    }
2514
2515    fn restore_saved_area(&mut self, saved_area_offset: i32) -> Result<(), CompileError> {
2516        self.assembler.emit_sub(
2517            Size::S64,
2518            Location::GPR(GPR::Fp),
2519            Location::Imm64(saved_area_offset as _),
2520            Location::GPR(GPR::Sp),
2521        )
2522    }
2523
2524    fn pop_location(&mut self, location: Location) -> Result<(), CompileError> {
2525        self.emit_pop(Size::S64, location)
2526    }
2527
2528    fn assembler_finalize(
2529        self,
2530        assembly_comments: HashMap<usize, AssemblyComment>,
2531    ) -> Result<FinalizedAssembly, CompileError> {
2532        Ok(FinalizedAssembly {
2533            body: self.assembler.finalize().map_err(|e| {
2534                CompileError::Codegen(format!("Assembler failed finalization with: {e:?}"))
2535            })?,
2536            assembly_comments,
2537        })
2538    }
2539
2540    fn get_offset(&self) -> Offset {
2541        self.assembler.get_offset()
2542    }
2543
2544    fn finalize_function(&mut self) -> Result<(), CompileError> {
2545        self.assembler.finalize_function()?;
2546        Ok(())
2547    }
2548
2549    fn emit_function_prolog(&mut self) -> Result<(), CompileError> {
2550        self.assembler.emit_sub(
2551            Size::S64,
2552            Location::GPR(GPR::Sp),
2553            Location::Imm64(16),
2554            Location::GPR(GPR::Sp),
2555        )?;
2556        self.emit_unwind_op(UnwindOps::SubtractFP { up_to_sp: 16 });
2557
2558        self.assembler.emit_sd(
2559            Size::S64,
2560            Location::GPR(GPR::X1), // return address register
2561            Location::Memory(GPR::Sp, 8),
2562        )?;
2563        self.assembler.emit_sd(
2564            Size::S64,
2565            Location::GPR(GPR::Fp),
2566            Location::Memory(GPR::Sp, 0),
2567        )?;
2568        self.emit_unwind_op(UnwindOps::SaveRegister {
2569            reg: UnwindRegister::GPR(GPR::X1),
2570            bp_neg_offset: 8,
2571        });
2572        self.emit_unwind_op(UnwindOps::SaveRegister {
2573            reg: UnwindRegister::GPR(GPR::Fp),
2574            bp_neg_offset: 16,
2575        });
2576
2577        self.assembler
2578            .emit_mov(Size::S64, Location::GPR(GPR::Sp), Location::GPR(GPR::Fp))?;
2579        self.emit_unwind_op(UnwindOps::DefineNewFrame);
2580        Ok(())
2581    }
2582
2583    fn emit_function_epilog(&mut self) -> Result<(), CompileError> {
2584        self.assembler
2585            .emit_mov(Size::S64, Location::GPR(GPR::Fp), Location::GPR(GPR::Sp))?;
2586        self.assembler.emit_ld(
2587            Size::S64,
2588            false,
2589            Location::GPR(GPR::X1), // return address register
2590            Location::Memory(GPR::Sp, 8),
2591        )?;
2592        self.assembler.emit_ld(
2593            Size::S64,
2594            false,
2595            Location::GPR(GPR::Fp),
2596            Location::Memory(GPR::Sp, 0),
2597        )?;
2598        self.assembler.emit_add(
2599            Size::S64,
2600            Location::GPR(GPR::Sp),
2601            Location::Imm64(16),
2602            Location::GPR(GPR::Sp),
2603        )?;
2604
2605        Ok(())
2606    }
2607
2608    fn emit_function_return_float(&mut self) -> Result<(), CompileError> {
2609        self.assembler
2610            .emit_mov(Size::S64, Location::GPR(GPR::X10), Location::SIMD(FPR::F10))
2611    }
2612
2613    fn canonicalize_nan(
2614        &mut self,
2615        sz: Size,
2616        input: Location,
2617        output: Location,
2618    ) -> Result<(), CompileError> {
2619        let mut temps = vec![];
2620        // use FMAX (input, input) => output to automatically normalize the NaN
2621        match (sz, input, output) {
2622            (Size::S32, Location::SIMD(_), Location::SIMD(_)) => {
2623                self.assembler.emit_fmax(sz, input, input, output)?;
2624            }
2625            (Size::S64, Location::SIMD(_), Location::SIMD(_)) => {
2626                self.assembler.emit_fmax(sz, input, input, output)?;
2627            }
2628            (Size::S32, Location::SIMD(_), _) | (Size::S64, Location::SIMD(_), _) => {
2629                let tmp = self.location_to_fpr(sz, output, &mut temps, ImmType::None, false)?;
2630                self.assembler.emit_fmax(sz, input, input, tmp)?;
2631                self.move_location(sz, tmp, output)?;
2632            }
2633            (Size::S32, Location::Memory(_, _), _) | (Size::S64, Location::Memory(_, _), _) => {
2634                let src = self.location_to_fpr(sz, input, &mut temps, ImmType::None, true)?;
2635                let tmp = self.location_to_fpr(sz, output, &mut temps, ImmType::None, false)?;
2636                self.assembler.emit_fmax(sz, src, src, tmp)?;
2637                if tmp != output {
2638                    self.move_location(sz, tmp, output)?;
2639                }
2640            }
2641            _ => codegen_error!(
2642                "singlepass can't emit canonicalize_nan {:?} {:?} {:?}",
2643                sz,
2644                input,
2645                output
2646            ),
2647        }
2648
2649        for r in temps {
2650            self.release_simd(r);
2651        }
2652        Ok(())
2653    }
2654
2655    fn emit_illegal_op(&mut self, trap: TrapCode) -> Result<(), CompileError> {
2656        let offset = self.assembler.get_offset().0;
2657        self.assembler.emit_udf(trap as u8)?;
2658        self.mark_instruction_address_end(offset);
2659        Ok(())
2660    }
2661
2662    fn get_label(&mut self) -> Label {
2663        self.assembler.new_dynamic_label()
2664    }
2665
2666    fn emit_label(&mut self, label: Label) -> Result<(), CompileError> {
2667        self.assembler.emit_label(label)
2668    }
2669
2670    fn get_gpr_for_call(&self) -> Self::GPR {
2671        GPR::X1
2672    }
2673
2674    fn emit_call_register(&mut self, register: Self::GPR) -> Result<(), CompileError> {
2675        self.assembler.emit_call_register(register)
2676    }
2677
2678    fn emit_call_label(&mut self, label: Label) -> Result<(), CompileError> {
2679        self.assembler.emit_call_label(label)
2680    }
2681
2682    fn arch_emit_indirect_call_with_trampoline(
2683        &mut self,
2684        _location: Location,
2685    ) -> Result<(), CompileError> {
2686        codegen_error!("singlepass arch_emit_indirect_call_with_trampoline unimplemented")
2687    }
2688
2689    fn emit_call_location(&mut self, location: Location) -> Result<(), CompileError> {
2690        let mut temps = vec![];
2691        let loc = self.location_to_reg(
2692            Size::S64,
2693            location,
2694            &mut temps,
2695            ImmType::None,
2696            true,
2697            Some(self.get_gpr_for_call()),
2698        )?;
2699        match loc {
2700            Location::GPR(reg) => self.assembler.emit_call_register(reg),
2701            _ => codegen_error!("singlepass can't emit CALL Location"),
2702        }?;
2703        for r in temps {
2704            self.release_gpr(r);
2705        }
2706        Ok(())
2707    }
2708
2709    fn emit_debug_breakpoint(&mut self) -> Result<(), CompileError> {
2710        self.assembler.emit_brk()
2711    }
2712
2713    fn location_add(
2714        &mut self,
2715        size: Size,
2716        source: Location,
2717        dest: Location,
2718        _flags: bool,
2719    ) -> Result<(), CompileError> {
2720        let mut temps = vec![];
2721        let src = self.location_to_reg(size, source, &mut temps, ImmType::Bits12, true, None)?;
2722        let dst = self.location_to_reg(size, dest, &mut temps, ImmType::None, true, None)?;
2723        self.assembler.emit_add(size, dst, src, dst)?;
2724        if dst != dest {
2725            self.move_location(size, dst, dest)?;
2726        }
2727        for r in temps {
2728            self.release_gpr(r);
2729        }
2730        Ok(())
2731    }
2732
2733    fn location_cmp(
2734        &mut self,
2735        _size: Size,
2736        _source: Location,
2737        _dest: Location,
2738    ) -> Result<(), CompileError> {
2739        codegen_error!("singlepass location_cmp not implemented")
2740    }
2741
2742    fn jmp_unconditional(&mut self, label: Label) -> Result<(), CompileError> {
2743        let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2744            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2745        })?;
2746        self.assembler.emit_j_label(label, Some(tmp))?;
2747        self.release_gpr(tmp);
2748        Ok(())
2749    }
2750
2751    fn jmp_on_condition(
2752        &mut self,
2753        cond: UnsignedCondition,
2754        size: Size,
2755        loc_a: AbstractLocation<Self::GPR, Self::SIMD>,
2756        loc_b: AbstractLocation<Self::GPR, Self::SIMD>,
2757        label: Label,
2758    ) -> Result<(), CompileError> {
2759        let c = match cond {
2760            UnsignedCondition::Equal => Condition::Eq,
2761            UnsignedCondition::NotEqual => Condition::Ne,
2762            UnsignedCondition::Above => Condition::Gtu,
2763            UnsignedCondition::AboveEqual => Condition::Geu,
2764            UnsignedCondition::Below => Condition::Ltu,
2765            UnsignedCondition::BelowEqual => Condition::Leu,
2766        };
2767
2768        let mut temps = vec![];
2769        let loc_a = self.location_to_reg(size, loc_a, &mut temps, ImmType::None, true, None)?;
2770        let loc_b = self.location_to_reg(size, loc_b, &mut temps, ImmType::None, true, None)?;
2771
2772        if size != Size::S64 {
2773            self.assembler.emit_extend(size, false, loc_a, loc_a)?;
2774            self.assembler.emit_extend(size, false, loc_b, loc_b)?;
2775        }
2776
2777        let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2778            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2779        })?;
2780        temps.push(tmp);
2781        self.assembler
2782            .emit_jmp_on_condition(c, loc_a, loc_b, label, tmp)?;
2783
2784        for r in temps {
2785            self.release_gpr(r);
2786        }
2787        Ok(())
2788    }
2789
2790    fn emit_jmp_to_jumptable(&mut self, label: Label, cond: Location) -> Result<(), CompileError> {
2791        let tmp1 = self.acquire_temp_gpr().ok_or_else(|| {
2792            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2793        })?;
2794        let tmp2 = self.acquire_temp_gpr().ok_or_else(|| {
2795            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2796        })?;
2797
2798        self.assembler.emit_load_label(tmp1, label)?;
2799        self.move_location(Size::S32, cond, Location::GPR(tmp2))?;
2800
2801        // Multiply by 8, for RISC-V, a jump is made via 2 instructions:
2802        // auipc and j. This way we can navigate greater code regions.
2803        self.assembler.emit_sll(
2804            Size::S32,
2805            Location::GPR(tmp2),
2806            Location::Imm32(3),
2807            Location::GPR(tmp2),
2808        )?;
2809        self.assembler.emit_add(
2810            Size::S64,
2811            Location::GPR(tmp1),
2812            Location::GPR(tmp2),
2813            Location::GPR(tmp2),
2814        )?;
2815        self.assembler.emit_j_register(tmp2)?;
2816        self.release_gpr(tmp2);
2817        self.release_gpr(tmp1);
2818        Ok(())
2819    }
2820
2821    fn align_for_loop(&mut self) -> Result<(), CompileError> {
2822        // nothing to do on RISC-V
2823        Ok(())
2824    }
2825
2826    fn emit_ret(&mut self) -> Result<(), CompileError> {
2827        self.assembler.emit_ret()
2828    }
2829
2830    fn emit_push(&mut self, size: Size, loc: Location) -> Result<(), CompileError> {
2831        self.assembler.emit_push(size, loc)
2832    }
2833
2834    fn emit_pop(&mut self, size: Size, loc: Location) -> Result<(), CompileError> {
2835        self.assembler.emit_pop(size, loc)
2836    }
2837
2838    fn emit_relaxed_mov(
2839        &mut self,
2840        sz: Size,
2841        src: Location,
2842        dst: Location,
2843    ) -> Result<(), CompileError> {
2844        self.emit_relaxed_binop(Assembler::emit_mov, sz, src, dst)
2845    }
2846
2847    fn emit_relaxed_cmp(
2848        &mut self,
2849        _sz: Size,
2850        _src: Location,
2851        _dst: Location,
2852    ) -> Result<(), CompileError> {
2853        todo!();
2854    }
2855
2856    fn emit_memory_fence(&mut self) -> Result<(), CompileError> {
2857        self.assembler.emit_rwfence()
2858    }
2859
2860    fn emit_relaxed_sign_extension(
2861        &mut self,
2862        sz_src: Size,
2863        src: Location,
2864        sz_dst: Size,
2865        dst: Location,
2866    ) -> Result<(), CompileError> {
2867        let mut temps = vec![];
2868
2869        match (src, dst) {
2870            (Location::Memory(reg, offset), Location::GPR(_)) => {
2871                let src = if ImmType::Bits12.compatible_imm(offset as _) {
2872                    src
2873                } else {
2874                    let tmp =
2875                        self.location_to_reg(sz_src, src, &mut temps, ImmType::None, true, None)?;
2876                    self.assembler.emit_mov_imm(tmp, offset as _)?;
2877                    self.assembler
2878                        .emit_add(Size::S64, Location::GPR(reg), tmp, tmp)?;
2879                    let Location::GPR(tmp) = tmp else {
2880                        unreachable!()
2881                    };
2882                    Location::Memory(tmp, 0)
2883                };
2884                self.assembler.emit_ld(sz_src, true, dst, src)?;
2885            }
2886            _ => {
2887                let src =
2888                    self.location_to_reg(sz_src, src, &mut temps, ImmType::None, true, None)?;
2889                let dest =
2890                    self.location_to_reg(sz_dst, dst, &mut temps, ImmType::None, false, None)?;
2891                self.assembler.emit_extend(sz_src, true, src, dest)?;
2892                if dst != dest {
2893                    self.move_location(sz_dst, dest, dst)?;
2894                }
2895            }
2896        }
2897
2898        for r in temps {
2899            self.release_gpr(r);
2900        }
2901
2902        Ok(())
2903    }
2904
2905    fn emit_imul_imm32(
2906        &mut self,
2907        size: Size,
2908        imm32: u32,
2909        gpr: Self::GPR,
2910    ) -> Result<(), CompileError> {
2911        let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2912            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2913        })?;
2914        self.assembler
2915            .emit_mov_imm(Location::GPR(tmp), imm32 as _)?;
2916        self.assembler.emit_mul(
2917            size,
2918            Location::GPR(gpr),
2919            Location::GPR(tmp),
2920            Location::GPR(gpr),
2921        )?;
2922        self.release_gpr(tmp);
2923        Ok(())
2924    }
2925
2926    fn emit_binop_add32(
2927        &mut self,
2928        loc_a: Location,
2929        loc_b: Location,
2930        ret: Location,
2931    ) -> Result<(), CompileError> {
2932        self.emit_relaxed_binop3(
2933            Assembler::emit_add,
2934            Size::S32,
2935            loc_a,
2936            loc_b,
2937            ret,
2938            ImmType::Bits12,
2939        )
2940    }
2941
2942    fn emit_binop_sub32(
2943        &mut self,
2944        loc_a: Location,
2945        loc_b: Location,
2946        ret: Location,
2947    ) -> Result<(), CompileError> {
2948        self.emit_relaxed_binop3(
2949            Assembler::emit_sub,
2950            Size::S32,
2951            loc_a,
2952            loc_b,
2953            ret,
2954            ImmType::Bits12Subtraction,
2955        )
2956    }
2957
2958    fn emit_binop_mul32(
2959        &mut self,
2960        loc_a: Location,
2961        loc_b: Location,
2962        ret: Location,
2963    ) -> Result<(), CompileError> {
2964        self.emit_relaxed_binop3(
2965            Assembler::emit_mul,
2966            Size::S32,
2967            loc_a,
2968            loc_b,
2969            ret,
2970            ImmType::None,
2971        )
2972    }
2973
2974    fn emit_binop_udiv32(
2975        &mut self,
2976        loc_a: Location,
2977        loc_b: Location,
2978        ret: Location,
2979        integer_division_by_zero: Label,
2980    ) -> Result<usize, CompileError> {
2981        let mut temps = vec![];
2982        let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
2983        let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
2984        let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
2985
2986        let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
2987            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2988        })?;
2989        temps.push(jmp_tmp);
2990
2991        self.assembler
2992            .emit_on_false_label_far(src2, integer_division_by_zero, jmp_tmp)?;
2993        let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
2994        self.assembler.emit_udiv(Size::S32, src1, src2, dest)?;
2995        if ret != dest {
2996            self.move_location(Size::S32, dest, ret)?;
2997        }
2998        for r in temps {
2999            self.release_gpr(r);
3000        }
3001        Ok(offset)
3002    }
3003
3004    fn emit_binop_sdiv32(
3005        &mut self,
3006        loc_a: Location,
3007        loc_b: Location,
3008        ret: Location,
3009        integer_division_by_zero: Label,
3010        integer_overflow: Label,
3011    ) -> Result<usize, CompileError> {
3012        let mut temps = vec![];
3013        let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
3014        let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
3015        let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3016
3017        let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
3018            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3019        })?;
3020        temps.push(jmp_tmp);
3021
3022        self.assembler
3023            .emit_on_false_label_far(src2, integer_division_by_zero, jmp_tmp)?;
3024        let label_nooverflow = self.assembler.get_label();
3025        let tmp = self.location_to_reg(
3026            Size::S32,
3027            Location::Imm32(i32::MIN as u32),
3028            &mut temps,
3029            ImmType::None,
3030            true,
3031            None,
3032        )?;
3033        self.assembler.emit_cmp(Condition::Ne, tmp, src1, tmp)?;
3034        self.assembler
3035            .emit_on_true_label(tmp, label_nooverflow, jmp_tmp)?;
3036        self.move_location(Size::S32, Location::Imm32(-1i32 as _), tmp)?;
3037        self.assembler.emit_cmp(Condition::Eq, tmp, src2, tmp)?;
3038        self.assembler
3039            .emit_on_true_label_far(tmp, integer_overflow, jmp_tmp)?;
3040        let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
3041        self.assembler.emit_label(label_nooverflow)?;
3042        self.assembler.emit_sdiv(Size::S32, src1, src2, dest)?;
3043        if ret != dest {
3044            self.move_location(Size::S32, dest, ret)?;
3045        }
3046        for r in temps {
3047            self.release_gpr(r);
3048        }
3049        Ok(offset)
3050    }
3051
3052    fn emit_binop_urem32(
3053        &mut self,
3054        loc_a: Location,
3055        loc_b: Location,
3056        ret: Location,
3057        integer_division_by_zero: Label,
3058    ) -> Result<usize, CompileError> {
3059        let mut temps = vec![];
3060        let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
3061        let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
3062        let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3063
3064        let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
3065            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3066        })?;
3067        temps.push(jmp_tmp);
3068
3069        self.assembler
3070            .emit_on_false_label_far(src2, integer_division_by_zero, jmp_tmp)?;
3071        let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
3072        self.assembler.emit_urem(Size::S32, src1, src2, dest)?;
3073        if ret != dest {
3074            self.move_location(Size::S32, dest, ret)?;
3075        }
3076        for r in temps {
3077            self.release_gpr(r);
3078        }
3079        Ok(offset)
3080    }
3081
3082    fn emit_binop_srem32(
3083        &mut self,
3084        loc_a: Location,
3085        loc_b: Location,
3086        ret: Location,
3087        integer_division_by_zero: Label,
3088    ) -> Result<usize, CompileError> {
3089        let mut temps = vec![];
3090        let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
3091        let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
3092        let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3093
3094        let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
3095            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3096        })?;
3097        temps.push(jmp_tmp);
3098
3099        self.assembler
3100            .emit_on_false_label_far(src2, integer_division_by_zero, jmp_tmp)?;
3101        let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
3102        self.assembler.emit_srem(Size::S32, src1, src2, dest)?;
3103        if ret != dest {
3104            self.move_location(Size::S32, dest, ret)?;
3105        }
3106        for r in temps {
3107            self.release_gpr(r);
3108        }
3109        Ok(offset)
3110    }
3111
3112    fn emit_binop_and32(
3113        &mut self,
3114        loc_a: Location,
3115        loc_b: Location,
3116        ret: Location,
3117    ) -> Result<(), CompileError> {
3118        self.emit_relaxed_binop3(
3119            Assembler::emit_and,
3120            Size::S32,
3121            loc_a,
3122            loc_b,
3123            ret,
3124            ImmType::Bits12,
3125        )
3126    }
3127
3128    fn emit_binop_or32(
3129        &mut self,
3130        loc_a: Location,
3131        loc_b: Location,
3132        ret: Location,
3133    ) -> Result<(), CompileError> {
3134        self.emit_relaxed_binop3(
3135            Assembler::emit_or,
3136            Size::S32,
3137            loc_a,
3138            loc_b,
3139            ret,
3140            ImmType::Bits12,
3141        )
3142    }
3143
3144    fn emit_binop_xor32(
3145        &mut self,
3146        loc_a: Location,
3147        loc_b: Location,
3148        ret: Location,
3149    ) -> Result<(), CompileError> {
3150        self.emit_relaxed_binop3(
3151            Assembler::emit_xor,
3152            Size::S32,
3153            loc_a,
3154            loc_b,
3155            ret,
3156            ImmType::Bits12,
3157        )
3158    }
3159
3160    fn i32_cmp_ge_s(
3161        &mut self,
3162        loc_a: Location,
3163        loc_b: Location,
3164        ret: Location,
3165    ) -> Result<(), CompileError> {
3166        self.emit_cmpop_i32_dynamic_b(Condition::Ge, loc_a, loc_b, ret, true)
3167    }
3168
3169    fn i32_cmp_gt_s(
3170        &mut self,
3171        loc_a: Location,
3172        loc_b: Location,
3173        ret: Location,
3174    ) -> Result<(), CompileError> {
3175        self.emit_cmpop_i32_dynamic_b(Condition::Gt, loc_a, loc_b, ret, true)
3176    }
3177
3178    fn i32_cmp_le_s(
3179        &mut self,
3180        loc_a: Location,
3181        loc_b: Location,
3182        ret: Location,
3183    ) -> Result<(), CompileError> {
3184        self.emit_cmpop_i32_dynamic_b(Condition::Le, loc_a, loc_b, ret, true)
3185    }
3186
3187    fn i32_cmp_lt_s(
3188        &mut self,
3189        loc_a: Location,
3190        loc_b: Location,
3191        ret: Location,
3192    ) -> Result<(), CompileError> {
3193        self.emit_cmpop_i32_dynamic_b(Condition::Lt, loc_a, loc_b, ret, true)
3194    }
3195
3196    fn i32_cmp_ge_u(
3197        &mut self,
3198        loc_a: Location,
3199        loc_b: Location,
3200        ret: Location,
3201    ) -> Result<(), CompileError> {
3202        self.emit_cmpop_i32_dynamic_b(Condition::Geu, loc_a, loc_b, ret, false)
3203    }
3204
3205    fn i32_cmp_gt_u(
3206        &mut self,
3207        loc_a: Location,
3208        loc_b: Location,
3209        ret: Location,
3210    ) -> Result<(), CompileError> {
3211        self.emit_cmpop_i32_dynamic_b(Condition::Gtu, loc_a, loc_b, ret, false)
3212    }
3213
3214    fn i32_cmp_le_u(
3215        &mut self,
3216        loc_a: Location,
3217        loc_b: Location,
3218        ret: Location,
3219    ) -> Result<(), CompileError> {
3220        self.emit_cmpop_i32_dynamic_b(Condition::Leu, loc_a, loc_b, ret, false)
3221    }
3222
3223    fn i32_cmp_lt_u(
3224        &mut self,
3225        loc_a: Location,
3226        loc_b: Location,
3227        ret: Location,
3228    ) -> Result<(), CompileError> {
3229        self.emit_cmpop_i32_dynamic_b(Condition::Ltu, loc_a, loc_b, ret, false)
3230    }
3231
3232    fn i32_cmp_ne(
3233        &mut self,
3234        loc_a: Location,
3235        loc_b: Location,
3236        ret: Location,
3237    ) -> Result<(), CompileError> {
3238        self.emit_cmpop_i32_dynamic_b(Condition::Ne, loc_a, loc_b, ret, true)
3239    }
3240
3241    fn i32_cmp_eq(
3242        &mut self,
3243        loc_a: Location,
3244        loc_b: Location,
3245        ret: Location,
3246    ) -> Result<(), CompileError> {
3247        self.emit_cmpop_i32_dynamic_b(Condition::Eq, loc_a, loc_b, ret, true)
3248    }
3249
3250    fn i32_clz(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
3251        self.emit_clz(Size::S32, loc, ret)
3252    }
3253
3254    fn i32_ctz(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
3255        self.emit_ctz(Size::S32, loc, ret)
3256    }
3257
3258    fn i32_popcnt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
3259        self.emit_popcnt(Size::S32, loc, ret)
3260    }
3261
3262    fn i32_shl(
3263        &mut self,
3264        loc_a: Location,
3265        loc_b: Location,
3266        ret: Location,
3267    ) -> Result<(), CompileError> {
3268        self.emit_relaxed_binop3(
3269            Assembler::emit_sll,
3270            Size::S32,
3271            loc_a,
3272            loc_b,
3273            ret,
3274            ImmType::Shift32,
3275        )
3276    }
3277
3278    fn i32_shr(
3279        &mut self,
3280        loc_a: Location,
3281        loc_b: Location,
3282        ret: Location,
3283    ) -> Result<(), CompileError> {
3284        self.emit_relaxed_binop3(
3285            Assembler::emit_srl,
3286            Size::S32,
3287            loc_a,
3288            loc_b,
3289            ret,
3290            ImmType::Shift32,
3291        )
3292    }
3293
3294    fn i32_sar(
3295        &mut self,
3296        loc_a: Location,
3297        loc_b: Location,
3298        ret: Location,
3299    ) -> Result<(), CompileError> {
3300        self.emit_relaxed_binop3(
3301            Assembler::emit_sra,
3302            Size::S32,
3303            loc_a,
3304            loc_b,
3305            ret,
3306            ImmType::Shift32,
3307        )
3308    }
3309
3310    fn i32_rol(
3311        &mut self,
3312        loc_a: Location,
3313        loc_b: Location,
3314        ret: Location,
3315    ) -> Result<(), CompileError> {
3316        self.emit_rol(Size::S32, loc_a, loc_b, ret, ImmType::Shift32)
3317    }
3318
3319    fn i32_ror(
3320        &mut self,
3321        loc_a: Location,
3322        loc_b: Location,
3323        ret: Location,
3324    ) -> Result<(), CompileError> {
3325        self.emit_ror(Size::S32, loc_a, loc_b, ret, ImmType::Shift32)
3326    }
3327
3328    fn i32_load(
3329        &mut self,
3330        addr: Location,
3331        memarg: &MemArg,
3332        ret: Location,
3333        _need_check: bool,
3334        imported_memories: bool,
3335        offset: i32,
3336        heap_access_oob: Label,
3337        unaligned_atomic: Label,
3338    ) -> Result<(), CompileError> {
3339        self.memory_op(
3340            addr,
3341            memarg,
3342            false,
3343            4,
3344            imported_memories,
3345            offset,
3346            heap_access_oob,
3347            unaligned_atomic,
3348            |this, addr| this.emit_maybe_unaligned_load(Size::S32, true, ret, addr),
3349        )
3350    }
3351
3352    fn i32_load_8u(
3353        &mut self,
3354        addr: Location,
3355        memarg: &MemArg,
3356        ret: Location,
3357        _need_check: bool,
3358        imported_memories: bool,
3359        offset: i32,
3360        heap_access_oob: Label,
3361        unaligned_atomic: Label,
3362    ) -> Result<(), CompileError> {
3363        self.memory_op(
3364            addr,
3365            memarg,
3366            false,
3367            1,
3368            imported_memories,
3369            offset,
3370            heap_access_oob,
3371            unaligned_atomic,
3372            |this, addr| this.emit_maybe_unaligned_load(Size::S8, false, ret, addr),
3373        )
3374    }
3375
3376    fn i32_load_8s(
3377        &mut self,
3378        addr: Location,
3379        memarg: &MemArg,
3380        ret: Location,
3381        _need_check: bool,
3382        imported_memories: bool,
3383        offset: i32,
3384        heap_access_oob: Label,
3385        unaligned_atomic: Label,
3386    ) -> Result<(), CompileError> {
3387        self.memory_op(
3388            addr,
3389            memarg,
3390            false,
3391            1,
3392            imported_memories,
3393            offset,
3394            heap_access_oob,
3395            unaligned_atomic,
3396            |this, addr| this.emit_maybe_unaligned_load(Size::S8, true, ret, addr),
3397        )
3398    }
3399
3400    fn i32_load_16u(
3401        &mut self,
3402        addr: Location,
3403        memarg: &MemArg,
3404        ret: Location,
3405        _need_check: bool,
3406        imported_memories: bool,
3407        offset: i32,
3408        heap_access_oob: Label,
3409        unaligned_atomic: Label,
3410    ) -> Result<(), CompileError> {
3411        self.memory_op(
3412            addr,
3413            memarg,
3414            false,
3415            2,
3416            imported_memories,
3417            offset,
3418            heap_access_oob,
3419            unaligned_atomic,
3420            |this, addr| this.emit_maybe_unaligned_load(Size::S16, false, ret, addr),
3421        )
3422    }
3423
3424    fn i32_load_16s(
3425        &mut self,
3426        addr: Location,
3427        memarg: &MemArg,
3428        ret: Location,
3429        _need_check: bool,
3430        imported_memories: bool,
3431        offset: i32,
3432        heap_access_oob: Label,
3433        unaligned_atomic: Label,
3434    ) -> Result<(), CompileError> {
3435        self.memory_op(
3436            addr,
3437            memarg,
3438            false,
3439            2,
3440            imported_memories,
3441            offset,
3442            heap_access_oob,
3443            unaligned_atomic,
3444            |this, addr| this.emit_maybe_unaligned_load(Size::S16, true, ret, addr),
3445        )
3446    }
3447
3448    fn i32_atomic_load(
3449        &mut self,
3450        addr: Location,
3451        memarg: &MemArg,
3452        ret: Location,
3453        _need_check: bool,
3454        imported_memories: bool,
3455        offset: i32,
3456        heap_access_oob: Label,
3457        unaligned_atomic: Label,
3458    ) -> Result<(), CompileError> {
3459        self.memory_op(
3460            addr,
3461            memarg,
3462            true,
3463            4,
3464            imported_memories,
3465            offset,
3466            heap_access_oob,
3467            unaligned_atomic,
3468            |this, addr| this.emit_relaxed_load(Size::S32, true, ret, Location::Memory(addr, 0)),
3469        )
3470    }
3471
3472    fn i32_atomic_load_8u(
3473        &mut self,
3474        addr: Location,
3475        memarg: &MemArg,
3476        ret: Location,
3477        _need_check: bool,
3478        imported_memories: bool,
3479        offset: i32,
3480        heap_access_oob: Label,
3481        unaligned_atomic: Label,
3482    ) -> Result<(), CompileError> {
3483        self.memory_op(
3484            addr,
3485            memarg,
3486            true,
3487            1,
3488            imported_memories,
3489            offset,
3490            heap_access_oob,
3491            unaligned_atomic,
3492            |this, addr| this.emit_relaxed_load(Size::S8, false, ret, Location::Memory(addr, 0)),
3493        )
3494    }
3495
3496    fn i32_atomic_load_16u(
3497        &mut self,
3498        addr: Location,
3499        memarg: &MemArg,
3500        ret: Location,
3501        _need_check: bool,
3502        imported_memories: bool,
3503        offset: i32,
3504        heap_access_oob: Label,
3505        unaligned_atomic: Label,
3506    ) -> Result<(), CompileError> {
3507        self.memory_op(
3508            addr,
3509            memarg,
3510            true,
3511            2,
3512            imported_memories,
3513            offset,
3514            heap_access_oob,
3515            unaligned_atomic,
3516            |this, addr| this.emit_relaxed_load(Size::S16, false, ret, Location::Memory(addr, 0)),
3517        )
3518    }
3519
3520    fn i32_save(
3521        &mut self,
3522        value: Location,
3523        memarg: &MemArg,
3524        addr: Location,
3525        _need_check: bool,
3526        imported_memories: bool,
3527        offset: i32,
3528        heap_access_oob: Label,
3529        unaligned_atomic: Label,
3530    ) -> Result<(), CompileError> {
3531        self.memory_op(
3532            addr,
3533            memarg,
3534            false,
3535            4,
3536            imported_memories,
3537            offset,
3538            heap_access_oob,
3539            unaligned_atomic,
3540            |this, addr| this.emit_maybe_unaligned_store(Size::S32, value, addr),
3541        )
3542    }
3543
3544    fn i32_save_8(
3545        &mut self,
3546        value: Location,
3547        memarg: &MemArg,
3548        addr: Location,
3549        _need_check: bool,
3550        imported_memories: bool,
3551        offset: i32,
3552        heap_access_oob: Label,
3553        unaligned_atomic: Label,
3554    ) -> Result<(), CompileError> {
3555        self.memory_op(
3556            addr,
3557            memarg,
3558            false,
3559            1,
3560            imported_memories,
3561            offset,
3562            heap_access_oob,
3563            unaligned_atomic,
3564            |this, addr| this.emit_maybe_unaligned_store(Size::S8, value, addr),
3565        )
3566    }
3567
3568    fn i32_save_16(
3569        &mut self,
3570        value: Location,
3571        memarg: &MemArg,
3572        addr: Location,
3573        _need_check: bool,
3574        imported_memories: bool,
3575        offset: i32,
3576        heap_access_oob: Label,
3577        unaligned_atomic: Label,
3578    ) -> Result<(), CompileError> {
3579        self.memory_op(
3580            addr,
3581            memarg,
3582            false,
3583            2,
3584            imported_memories,
3585            offset,
3586            heap_access_oob,
3587            unaligned_atomic,
3588            |this, addr| this.emit_maybe_unaligned_store(Size::S16, value, addr),
3589        )
3590    }
3591
3592    fn i32_atomic_save(
3593        &mut self,
3594        value: Location,
3595        memarg: &MemArg,
3596        addr: Location,
3597        _need_check: bool,
3598        imported_memories: bool,
3599        offset: i32,
3600        heap_access_oob: Label,
3601        unaligned_atomic: Label,
3602    ) -> Result<(), CompileError> {
3603        self.memory_op(
3604            addr,
3605            memarg,
3606            true,
3607            4,
3608            imported_memories,
3609            offset,
3610            heap_access_oob,
3611            unaligned_atomic,
3612            |this, addr| this.emit_relaxed_store(Size::S32, value, Location::Memory(addr, 0)),
3613        )?;
3614        self.assembler.emit_rwfence()
3615    }
3616
3617    fn i32_atomic_save_8(
3618        &mut self,
3619        value: Location,
3620        memarg: &MemArg,
3621        addr: Location,
3622        _need_check: bool,
3623        imported_memories: bool,
3624        offset: i32,
3625        heap_access_oob: Label,
3626        unaligned_atomic: Label,
3627    ) -> Result<(), CompileError> {
3628        self.memory_op(
3629            addr,
3630            memarg,
3631            true,
3632            1,
3633            imported_memories,
3634            offset,
3635            heap_access_oob,
3636            unaligned_atomic,
3637            |this, addr| this.emit_relaxed_store(Size::S8, value, Location::Memory(addr, 0)),
3638        )?;
3639        self.assembler.emit_rwfence()
3640    }
3641
3642    fn i32_atomic_save_16(
3643        &mut self,
3644        value: Location,
3645        memarg: &MemArg,
3646        addr: Location,
3647        _need_check: bool,
3648        imported_memories: bool,
3649        offset: i32,
3650        heap_access_oob: Label,
3651        unaligned_atomic: Label,
3652    ) -> Result<(), CompileError> {
3653        self.memory_op(
3654            addr,
3655            memarg,
3656            true,
3657            2,
3658            imported_memories,
3659            offset,
3660            heap_access_oob,
3661            unaligned_atomic,
3662            |this, addr| this.emit_relaxed_store(Size::S16, value, Location::Memory(addr, 0)),
3663        )?;
3664        self.assembler.emit_rwfence()
3665    }
3666
3667    fn i32_atomic_add(
3668        &mut self,
3669        loc: Location,
3670        target: Location,
3671        memarg: &MemArg,
3672        ret: Location,
3673        _need_check: bool,
3674        imported_memories: bool,
3675        offset: i32,
3676        heap_access_oob: Label,
3677        unaligned_atomic: Label,
3678    ) -> Result<(), CompileError> {
3679        self.memory_op(
3680            target,
3681            memarg,
3682            true,
3683            4,
3684            imported_memories,
3685            offset,
3686            heap_access_oob,
3687            unaligned_atomic,
3688            |this, addr| {
3689                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Add, Size::S32, ret, addr, loc)
3690            },
3691        )
3692    }
3693
3694    fn i32_atomic_add_8u(
3695        &mut self,
3696        loc: Location,
3697        target: Location,
3698        memarg: &MemArg,
3699        ret: Location,
3700        _need_check: bool,
3701        imported_memories: bool,
3702        offset: i32,
3703        heap_access_oob: Label,
3704        unaligned_atomic: Label,
3705    ) -> Result<(), CompileError> {
3706        self.memory_op(
3707            target,
3708            memarg,
3709            true,
3710            1,
3711            imported_memories,
3712            offset,
3713            heap_access_oob,
3714            unaligned_atomic,
3715            |this, addr| {
3716                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Add, Size::S8, ret, addr, loc)
3717            },
3718        )
3719    }
3720
3721    fn i32_atomic_add_16u(
3722        &mut self,
3723        loc: Location,
3724        target: Location,
3725        memarg: &MemArg,
3726        ret: Location,
3727        _need_check: bool,
3728        imported_memories: bool,
3729        offset: i32,
3730        heap_access_oob: Label,
3731        unaligned_atomic: Label,
3732    ) -> Result<(), CompileError> {
3733        self.memory_op(
3734            target,
3735            memarg,
3736            true,
3737            2,
3738            imported_memories,
3739            offset,
3740            heap_access_oob,
3741            unaligned_atomic,
3742            |this, addr| {
3743                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Add, Size::S16, ret, addr, loc)
3744            },
3745        )
3746    }
3747
3748    fn i32_atomic_sub(
3749        &mut self,
3750        loc: Location,
3751        target: Location,
3752        memarg: &MemArg,
3753        ret: Location,
3754        _need_check: bool,
3755        imported_memories: bool,
3756        offset: i32,
3757        heap_access_oob: Label,
3758        unaligned_atomic: Label,
3759    ) -> Result<(), CompileError> {
3760        self.memory_op(
3761            target,
3762            memarg,
3763            true,
3764            4,
3765            imported_memories,
3766            offset,
3767            heap_access_oob,
3768            unaligned_atomic,
3769            |this, addr| {
3770                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Sub, Size::S32, ret, addr, loc)
3771            },
3772        )
3773    }
3774
3775    fn i32_atomic_sub_8u(
3776        &mut self,
3777        loc: Location,
3778        target: Location,
3779        memarg: &MemArg,
3780        ret: Location,
3781        _need_check: bool,
3782        imported_memories: bool,
3783        offset: i32,
3784        heap_access_oob: Label,
3785        unaligned_atomic: Label,
3786    ) -> Result<(), CompileError> {
3787        self.memory_op(
3788            target,
3789            memarg,
3790            true,
3791            1,
3792            imported_memories,
3793            offset,
3794            heap_access_oob,
3795            unaligned_atomic,
3796            |this, addr| {
3797                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Sub, Size::S8, ret, addr, loc)
3798            },
3799        )
3800    }
3801
3802    fn i32_atomic_sub_16u(
3803        &mut self,
3804        loc: Location,
3805        target: Location,
3806        memarg: &MemArg,
3807        ret: Location,
3808        _need_check: bool,
3809        imported_memories: bool,
3810        offset: i32,
3811        heap_access_oob: Label,
3812        unaligned_atomic: Label,
3813    ) -> Result<(), CompileError> {
3814        self.memory_op(
3815            target,
3816            memarg,
3817            true,
3818            2,
3819            imported_memories,
3820            offset,
3821            heap_access_oob,
3822            unaligned_atomic,
3823            |this, addr| {
3824                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Sub, Size::S16, ret, addr, loc)
3825            },
3826        )
3827    }
3828
3829    fn i32_atomic_and(
3830        &mut self,
3831        loc: Location,
3832        target: Location,
3833        memarg: &MemArg,
3834        ret: Location,
3835        _need_check: bool,
3836        imported_memories: bool,
3837        offset: i32,
3838        heap_access_oob: Label,
3839        unaligned_atomic: Label,
3840    ) -> Result<(), CompileError> {
3841        self.memory_op(
3842            target,
3843            memarg,
3844            true,
3845            4,
3846            imported_memories,
3847            offset,
3848            heap_access_oob,
3849            unaligned_atomic,
3850            |this, addr| {
3851                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::And, Size::S32, ret, addr, loc)
3852            },
3853        )
3854    }
3855
3856    fn i32_atomic_and_8u(
3857        &mut self,
3858        loc: Location,
3859        target: Location,
3860        memarg: &MemArg,
3861        ret: Location,
3862        _need_check: bool,
3863        imported_memories: bool,
3864        offset: i32,
3865        heap_access_oob: Label,
3866        unaligned_atomic: Label,
3867    ) -> Result<(), CompileError> {
3868        self.memory_op(
3869            target,
3870            memarg,
3871            true,
3872            1,
3873            imported_memories,
3874            offset,
3875            heap_access_oob,
3876            unaligned_atomic,
3877            |this, addr| {
3878                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::And, Size::S8, ret, addr, loc)
3879            },
3880        )
3881    }
3882
3883    fn i32_atomic_and_16u(
3884        &mut self,
3885        loc: Location,
3886        target: Location,
3887        memarg: &MemArg,
3888        ret: Location,
3889        _need_check: bool,
3890        imported_memories: bool,
3891        offset: i32,
3892        heap_access_oob: Label,
3893        unaligned_atomic: Label,
3894    ) -> Result<(), CompileError> {
3895        self.memory_op(
3896            target,
3897            memarg,
3898            true,
3899            2,
3900            imported_memories,
3901            offset,
3902            heap_access_oob,
3903            unaligned_atomic,
3904            |this, addr| {
3905                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::And, Size::S16, ret, addr, loc)
3906            },
3907        )
3908    }
3909
3910    fn i32_atomic_or(
3911        &mut self,
3912        loc: Location,
3913        target: Location,
3914        memarg: &MemArg,
3915        ret: Location,
3916        _need_check: bool,
3917        imported_memories: bool,
3918        offset: i32,
3919        heap_access_oob: Label,
3920        unaligned_atomic: Label,
3921    ) -> Result<(), CompileError> {
3922        self.memory_op(
3923            target,
3924            memarg,
3925            true,
3926            4,
3927            imported_memories,
3928            offset,
3929            heap_access_oob,
3930            unaligned_atomic,
3931            |this, addr| {
3932                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Or, Size::S32, ret, addr, loc)
3933            },
3934        )
3935    }
3936
3937    fn i32_atomic_or_8u(
3938        &mut self,
3939        loc: Location,
3940        target: Location,
3941        memarg: &MemArg,
3942        ret: Location,
3943        _need_check: bool,
3944        imported_memories: bool,
3945        offset: i32,
3946        heap_access_oob: Label,
3947        unaligned_atomic: Label,
3948    ) -> Result<(), CompileError> {
3949        self.memory_op(
3950            target,
3951            memarg,
3952            true,
3953            1,
3954            imported_memories,
3955            offset,
3956            heap_access_oob,
3957            unaligned_atomic,
3958            |this, addr| {
3959                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Or, Size::S8, ret, addr, loc)
3960            },
3961        )
3962    }
3963
3964    fn i32_atomic_or_16u(
3965        &mut self,
3966        loc: Location,
3967        target: Location,
3968        memarg: &MemArg,
3969        ret: Location,
3970        _need_check: bool,
3971        imported_memories: bool,
3972        offset: i32,
3973        heap_access_oob: Label,
3974        unaligned_atomic: Label,
3975    ) -> Result<(), CompileError> {
3976        self.memory_op(
3977            target,
3978            memarg,
3979            true,
3980            2,
3981            imported_memories,
3982            offset,
3983            heap_access_oob,
3984            unaligned_atomic,
3985            |this, addr| {
3986                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Or, Size::S16, ret, addr, loc)
3987            },
3988        )
3989    }
3990
3991    fn i32_atomic_xor(
3992        &mut self,
3993        loc: Location,
3994        target: Location,
3995        memarg: &MemArg,
3996        ret: Location,
3997        _need_check: bool,
3998        imported_memories: bool,
3999        offset: i32,
4000        heap_access_oob: Label,
4001        unaligned_atomic: Label,
4002    ) -> Result<(), CompileError> {
4003        self.memory_op(
4004            target,
4005            memarg,
4006            true,
4007            4,
4008            imported_memories,
4009            offset,
4010            heap_access_oob,
4011            unaligned_atomic,
4012            |this, addr| {
4013                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Xor, Size::S32, ret, addr, loc)
4014            },
4015        )
4016    }
4017
4018    fn i32_atomic_xor_8u(
4019        &mut self,
4020        loc: Location,
4021        target: Location,
4022        memarg: &MemArg,
4023        ret: Location,
4024        _need_check: bool,
4025        imported_memories: bool,
4026        offset: i32,
4027        heap_access_oob: Label,
4028        unaligned_atomic: Label,
4029    ) -> Result<(), CompileError> {
4030        self.memory_op(
4031            target,
4032            memarg,
4033            true,
4034            1,
4035            imported_memories,
4036            offset,
4037            heap_access_oob,
4038            unaligned_atomic,
4039            |this, addr| {
4040                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Xor, Size::S8, ret, addr, loc)
4041            },
4042        )
4043    }
4044
4045    fn i32_atomic_xor_16u(
4046        &mut self,
4047        loc: Location,
4048        target: Location,
4049        memarg: &MemArg,
4050        ret: Location,
4051        _need_check: bool,
4052        imported_memories: bool,
4053        offset: i32,
4054        heap_access_oob: Label,
4055        unaligned_atomic: Label,
4056    ) -> Result<(), CompileError> {
4057        self.memory_op(
4058            target,
4059            memarg,
4060            true,
4061            2,
4062            imported_memories,
4063            offset,
4064            heap_access_oob,
4065            unaligned_atomic,
4066            |this, addr| {
4067                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Xor, Size::S16, ret, addr, loc)
4068            },
4069        )
4070    }
4071
4072    fn i32_atomic_xchg(
4073        &mut self,
4074        loc: Location,
4075        target: Location,
4076        memarg: &MemArg,
4077        ret: Location,
4078        _need_check: bool,
4079        imported_memories: bool,
4080        offset: i32,
4081        heap_access_oob: Label,
4082        unaligned_atomic: Label,
4083    ) -> Result<(), CompileError> {
4084        self.memory_op(
4085            target,
4086            memarg,
4087            true,
4088            4,
4089            imported_memories,
4090            offset,
4091            heap_access_oob,
4092            unaligned_atomic,
4093            |this, addr| {
4094                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Exchange, Size::S32, ret, addr, loc)
4095            },
4096        )
4097    }
4098
4099    fn i32_atomic_xchg_8u(
4100        &mut self,
4101        loc: Location,
4102        target: Location,
4103        memarg: &MemArg,
4104        ret: Location,
4105        _need_check: bool,
4106        imported_memories: bool,
4107        offset: i32,
4108        heap_access_oob: Label,
4109        unaligned_atomic: Label,
4110    ) -> Result<(), CompileError> {
4111        self.memory_op(
4112            target,
4113            memarg,
4114            true,
4115            1,
4116            imported_memories,
4117            offset,
4118            heap_access_oob,
4119            unaligned_atomic,
4120            |this, addr| {
4121                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Exchange, Size::S8, ret, addr, loc)
4122            },
4123        )
4124    }
4125
4126    fn i32_atomic_xchg_16u(
4127        &mut self,
4128        loc: Location,
4129        target: Location,
4130        memarg: &MemArg,
4131        ret: Location,
4132        _need_check: bool,
4133        imported_memories: bool,
4134        offset: i32,
4135        heap_access_oob: Label,
4136        unaligned_atomic: Label,
4137    ) -> Result<(), CompileError> {
4138        self.memory_op(
4139            target,
4140            memarg,
4141            true,
4142            2,
4143            imported_memories,
4144            offset,
4145            heap_access_oob,
4146            unaligned_atomic,
4147            |this, addr| {
4148                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Exchange, Size::S16, ret, addr, loc)
4149            },
4150        )
4151    }
4152
4153    fn i32_atomic_cmpxchg(
4154        &mut self,
4155        new: Location,
4156        cmp: Location,
4157        target: Location,
4158        memarg: &MemArg,
4159        ret: Location,
4160        _need_check: bool,
4161        imported_memories: bool,
4162        offset: i32,
4163        heap_access_oob: Label,
4164        unaligned_atomic: Label,
4165    ) -> Result<(), CompileError> {
4166        self.memory_op(
4167            target,
4168            memarg,
4169            true,
4170            4,
4171            imported_memories,
4172            offset,
4173            heap_access_oob,
4174            unaligned_atomic,
4175            |this, addr| this.emit_relaxed_atomic_cmpxchg(Size::S32, ret, addr, new, cmp),
4176        )
4177    }
4178
4179    fn i32_atomic_cmpxchg_8u(
4180        &mut self,
4181        new: Location,
4182        cmp: Location,
4183        target: Location,
4184        memarg: &MemArg,
4185        ret: Location,
4186        _need_check: bool,
4187        imported_memories: bool,
4188        offset: i32,
4189        heap_access_oob: Label,
4190        unaligned_atomic: Label,
4191    ) -> Result<(), CompileError> {
4192        self.memory_op(
4193            target,
4194            memarg,
4195            true,
4196            1,
4197            imported_memories,
4198            offset,
4199            heap_access_oob,
4200            unaligned_atomic,
4201            |this, addr| this.emit_relaxed_atomic_cmpxchg(Size::S8, ret, addr, new, cmp),
4202        )
4203    }
4204
4205    fn i32_atomic_cmpxchg_16u(
4206        &mut self,
4207        new: Location,
4208        cmp: Location,
4209        target: Location,
4210        memarg: &MemArg,
4211        ret: Location,
4212        _need_check: bool,
4213        imported_memories: bool,
4214        offset: i32,
4215        heap_access_oob: Label,
4216        unaligned_atomic: Label,
4217    ) -> Result<(), CompileError> {
4218        self.memory_op(
4219            target,
4220            memarg,
4221            true,
4222            2,
4223            imported_memories,
4224            offset,
4225            heap_access_oob,
4226            unaligned_atomic,
4227            |this, addr| this.emit_relaxed_atomic_cmpxchg(Size::S16, ret, addr, new, cmp),
4228        )
4229    }
4230
4231    fn emit_call_with_reloc(
4232        &mut self,
4233        reloc_target: RelocationTarget,
4234    ) -> Result<Vec<Relocation>, CompileError> {
4235        let mut relocations = vec![];
4236        let next = self.get_label();
4237        let reloc_at = self.assembler.get_offset().0;
4238        self.emit_label(next)?; // this is to be sure the current imm26 value is 0
4239        self.assembler.emit_call_label(next)?;
4240        relocations.push(Relocation {
4241            kind: RelocationKind::RiscvCall,
4242            reloc_target,
4243            offset: reloc_at as u32,
4244            addend: 0,
4245        });
4246        Ok(relocations)
4247    }
4248
4249    fn emit_binop_add64(
4250        &mut self,
4251        loc_a: Location,
4252        loc_b: Location,
4253        ret: Location,
4254    ) -> Result<(), CompileError> {
4255        self.emit_relaxed_binop3(
4256            Assembler::emit_add,
4257            Size::S64,
4258            loc_a,
4259            loc_b,
4260            ret,
4261            ImmType::Bits12,
4262        )
4263    }
4264
4265    fn emit_binop_sub64(
4266        &mut self,
4267        loc_a: Location,
4268        loc_b: Location,
4269        ret: Location,
4270    ) -> Result<(), CompileError> {
4271        self.emit_relaxed_binop3(
4272            Assembler::emit_sub,
4273            Size::S64,
4274            loc_a,
4275            loc_b,
4276            ret,
4277            ImmType::Bits12Subtraction,
4278        )
4279    }
4280
4281    fn emit_binop_mul64(
4282        &mut self,
4283        loc_a: Location,
4284        loc_b: Location,
4285        ret: Location,
4286    ) -> Result<(), CompileError> {
4287        self.emit_relaxed_binop3(
4288            Assembler::emit_mul,
4289            Size::S64,
4290            loc_a,
4291            loc_b,
4292            ret,
4293            ImmType::None,
4294        )
4295    }
4296
4297    fn emit_binop_udiv64(
4298        &mut self,
4299        loc_a: Location,
4300        loc_b: Location,
4301        ret: Location,
4302        integer_division_by_zero: Label,
4303    ) -> Result<usize, CompileError> {
4304        let mut temps = vec![];
4305        let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4306        let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4307        let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4308
4309        let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
4310            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4311        })?;
4312        temps.push(jmp_tmp);
4313
4314        self.assembler
4315            .emit_on_false_label_far(src2, integer_division_by_zero, jmp_tmp)?;
4316        let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
4317        self.assembler.emit_udiv(Size::S64, src1, src2, dest)?;
4318        if ret != dest {
4319            self.move_location(Size::S64, dest, ret)?;
4320        }
4321        for r in temps {
4322            self.release_gpr(r);
4323        }
4324        Ok(offset)
4325    }
4326
4327    fn emit_binop_sdiv64(
4328        &mut self,
4329        loc_a: Location,
4330        loc_b: Location,
4331        ret: Location,
4332        integer_division_by_zero: Label,
4333        integer_overflow: Label,
4334    ) -> Result<usize, CompileError> {
4335        let mut temps = vec![];
4336        let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4337        let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4338        let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4339
4340        let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
4341            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4342        })?;
4343        temps.push(jmp_tmp);
4344
4345        self.assembler
4346            .emit_on_false_label_far(src2, integer_division_by_zero, jmp_tmp)?;
4347        let label_nooverflow = self.assembler.get_label();
4348        let tmp = self.location_to_reg(
4349            Size::S64,
4350            Location::Imm64(i64::MIN as u64),
4351            &mut temps,
4352            ImmType::None,
4353            true,
4354            None,
4355        )?;
4356
4357        self.assembler.emit_cmp(Condition::Ne, tmp, src1, tmp)?;
4358        self.assembler
4359            .emit_on_true_label(tmp, label_nooverflow, jmp_tmp)?;
4360        self.move_location(Size::S64, Location::Imm64(-1i64 as _), tmp)?;
4361        self.assembler.emit_cmp(Condition::Eq, tmp, src2, tmp)?;
4362        self.assembler
4363            .emit_on_true_label_far(tmp, integer_overflow, jmp_tmp)?;
4364        let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
4365        self.assembler.emit_label(label_nooverflow)?;
4366        self.assembler.emit_sdiv(Size::S64, src1, src2, dest)?;
4367        if ret != dest {
4368            self.move_location(Size::S64, dest, ret)?;
4369        }
4370        for r in temps {
4371            self.release_gpr(r);
4372        }
4373        Ok(offset)
4374    }
4375
4376    fn emit_binop_urem64(
4377        &mut self,
4378        loc_a: Location,
4379        loc_b: Location,
4380        ret: Location,
4381        integer_division_by_zero: Label,
4382    ) -> Result<usize, CompileError> {
4383        let mut temps = vec![];
4384        let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4385        let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4386        let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4387
4388        let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
4389            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4390        })?;
4391        temps.push(jmp_tmp);
4392
4393        self.assembler
4394            .emit_on_false_label_far(src2, integer_division_by_zero, jmp_tmp)?;
4395        let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
4396        self.assembler.emit_urem(Size::S64, src1, src2, dest)?;
4397        if ret != dest {
4398            self.move_location(Size::S64, dest, ret)?;
4399        }
4400        for r in temps {
4401            self.release_gpr(r);
4402        }
4403        Ok(offset)
4404    }
4405
4406    fn emit_binop_srem64(
4407        &mut self,
4408        loc_a: Location,
4409        loc_b: Location,
4410        ret: Location,
4411        integer_division_by_zero: Label,
4412    ) -> Result<usize, CompileError> {
4413        let mut temps = vec![];
4414        let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4415        let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4416        let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4417
4418        let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
4419            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4420        })?;
4421        temps.push(jmp_tmp);
4422
4423        self.assembler
4424            .emit_on_false_label_far(src2, integer_division_by_zero, jmp_tmp)?;
4425        let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
4426        self.assembler.emit_srem(Size::S64, src1, src2, dest)?;
4427        if ret != dest {
4428            self.move_location(Size::S64, dest, ret)?;
4429        }
4430        for r in temps {
4431            self.release_gpr(r);
4432        }
4433        Ok(offset)
4434    }
4435
4436    fn emit_binop_and64(
4437        &mut self,
4438        loc_a: Location,
4439        loc_b: Location,
4440        ret: Location,
4441    ) -> Result<(), CompileError> {
4442        self.emit_relaxed_binop3(
4443            Assembler::emit_and,
4444            Size::S64,
4445            loc_a,
4446            loc_b,
4447            ret,
4448            ImmType::Bits12,
4449        )
4450    }
4451
4452    fn emit_binop_or64(
4453        &mut self,
4454        loc_a: Location,
4455        loc_b: Location,
4456        ret: Location,
4457    ) -> Result<(), CompileError> {
4458        self.emit_relaxed_binop3(
4459            Assembler::emit_or,
4460            Size::S64,
4461            loc_a,
4462            loc_b,
4463            ret,
4464            ImmType::Bits12,
4465        )
4466    }
4467
4468    fn emit_binop_xor64(
4469        &mut self,
4470        loc_a: Location,
4471        loc_b: Location,
4472        ret: Location,
4473    ) -> Result<(), CompileError> {
4474        self.emit_relaxed_binop3(
4475            Assembler::emit_xor,
4476            Size::S64,
4477            loc_a,
4478            loc_b,
4479            ret,
4480            ImmType::Bits12,
4481        )
4482    }
4483
4484    fn i64_cmp_ge_s(
4485        &mut self,
4486        loc_a: Location,
4487        loc_b: Location,
4488        ret: Location,
4489    ) -> Result<(), CompileError> {
4490        self.emit_cmpop_i64_dynamic_b(Condition::Ge, loc_a, loc_b, ret)
4491    }
4492
4493    fn i64_cmp_gt_s(
4494        &mut self,
4495        loc_a: Location,
4496        loc_b: Location,
4497        ret: Location,
4498    ) -> Result<(), CompileError> {
4499        self.emit_cmpop_i64_dynamic_b(Condition::Gt, loc_a, loc_b, ret)
4500    }
4501
4502    fn i64_cmp_le_s(
4503        &mut self,
4504        loc_a: Location,
4505        loc_b: Location,
4506        ret: Location,
4507    ) -> Result<(), CompileError> {
4508        self.emit_cmpop_i64_dynamic_b(Condition::Le, loc_a, loc_b, ret)
4509    }
4510
4511    fn i64_cmp_lt_s(
4512        &mut self,
4513        loc_a: Location,
4514        loc_b: Location,
4515        ret: Location,
4516    ) -> Result<(), CompileError> {
4517        self.emit_cmpop_i64_dynamic_b(Condition::Lt, loc_a, loc_b, ret)
4518    }
4519
4520    fn i64_cmp_ge_u(
4521        &mut self,
4522        loc_a: Location,
4523        loc_b: Location,
4524        ret: Location,
4525    ) -> Result<(), CompileError> {
4526        self.emit_cmpop_i64_dynamic_b(Condition::Geu, loc_a, loc_b, ret)
4527    }
4528
4529    fn i64_cmp_gt_u(
4530        &mut self,
4531        loc_a: Location,
4532        loc_b: Location,
4533        ret: Location,
4534    ) -> Result<(), CompileError> {
4535        self.emit_cmpop_i64_dynamic_b(Condition::Gtu, loc_a, loc_b, ret)
4536    }
4537
4538    fn i64_cmp_le_u(
4539        &mut self,
4540        loc_a: Location,
4541        loc_b: Location,
4542        ret: Location,
4543    ) -> Result<(), CompileError> {
4544        self.emit_cmpop_i64_dynamic_b(Condition::Leu, loc_a, loc_b, ret)
4545    }
4546
4547    fn i64_cmp_lt_u(
4548        &mut self,
4549        loc_a: Location,
4550        loc_b: Location,
4551        ret: Location,
4552    ) -> Result<(), CompileError> {
4553        self.emit_cmpop_i64_dynamic_b(Condition::Ltu, loc_a, loc_b, ret)
4554    }
4555
4556    fn i64_cmp_ne(
4557        &mut self,
4558        loc_a: Location,
4559        loc_b: Location,
4560        ret: Location,
4561    ) -> Result<(), CompileError> {
4562        self.emit_cmpop_i64_dynamic_b(Condition::Ne, loc_a, loc_b, ret)
4563    }
4564
4565    fn i64_cmp_eq(
4566        &mut self,
4567        loc_a: Location,
4568        loc_b: Location,
4569        ret: Location,
4570    ) -> Result<(), CompileError> {
4571        self.emit_cmpop_i64_dynamic_b(Condition::Eq, loc_a, loc_b, ret)
4572    }
4573
4574    fn i64_clz(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
4575        self.emit_clz(Size::S64, loc, ret)
4576    }
4577
4578    fn i64_ctz(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
4579        self.emit_ctz(Size::S64, loc, ret)
4580    }
4581
4582    fn i64_popcnt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
4583        self.emit_popcnt(Size::S64, loc, ret)
4584    }
4585
4586    fn i64_shl(
4587        &mut self,
4588        loc_a: Location,
4589        loc_b: Location,
4590        ret: Location,
4591    ) -> Result<(), CompileError> {
4592        self.emit_relaxed_binop3(
4593            Assembler::emit_sll,
4594            Size::S64,
4595            loc_a,
4596            loc_b,
4597            ret,
4598            ImmType::Shift64,
4599        )
4600    }
4601
4602    fn i64_shr(
4603        &mut self,
4604        loc_a: Location,
4605        loc_b: Location,
4606        ret: Location,
4607    ) -> Result<(), CompileError> {
4608        self.emit_relaxed_binop3(
4609            Assembler::emit_srl,
4610            Size::S64,
4611            loc_a,
4612            loc_b,
4613            ret,
4614            ImmType::Shift64,
4615        )
4616    }
4617
4618    fn i64_sar(
4619        &mut self,
4620        loc_a: Location,
4621        loc_b: Location,
4622        ret: Location,
4623    ) -> Result<(), CompileError> {
4624        self.emit_relaxed_binop3(
4625            Assembler::emit_sra,
4626            Size::S64,
4627            loc_a,
4628            loc_b,
4629            ret,
4630            ImmType::Shift64,
4631        )
4632    }
4633
4634    fn i64_rol(
4635        &mut self,
4636        loc_a: Location,
4637        loc_b: Location,
4638        ret: Location,
4639    ) -> Result<(), CompileError> {
4640        self.emit_rol(Size::S64, loc_a, loc_b, ret, ImmType::Shift64)
4641    }
4642
4643    fn i64_ror(
4644        &mut self,
4645        loc_a: Location,
4646        loc_b: Location,
4647        ret: Location,
4648    ) -> Result<(), CompileError> {
4649        self.emit_ror(Size::S64, loc_a, loc_b, ret, ImmType::Shift64)
4650    }
4651
4652    fn i64_load(
4653        &mut self,
4654        addr: Location,
4655        memarg: &MemArg,
4656        ret: Location,
4657        _need_check: bool,
4658        imported_memories: bool,
4659        offset: i32,
4660        heap_access_oob: Label,
4661        unaligned_atomic: Label,
4662    ) -> Result<(), CompileError> {
4663        self.memory_op(
4664            addr,
4665            memarg,
4666            false,
4667            8,
4668            imported_memories,
4669            offset,
4670            heap_access_oob,
4671            unaligned_atomic,
4672            |this, addr| this.emit_maybe_unaligned_load(Size::S64, true, ret, addr),
4673        )
4674    }
4675
4676    fn i64_load_8u(
4677        &mut self,
4678        addr: Location,
4679        memarg: &MemArg,
4680        ret: Location,
4681        _need_check: bool,
4682        imported_memories: bool,
4683        offset: i32,
4684        heap_access_oob: Label,
4685        unaligned_atomic: Label,
4686    ) -> Result<(), CompileError> {
4687        self.memory_op(
4688            addr,
4689            memarg,
4690            false,
4691            1,
4692            imported_memories,
4693            offset,
4694            heap_access_oob,
4695            unaligned_atomic,
4696            |this, addr| this.emit_maybe_unaligned_load(Size::S8, false, ret, addr),
4697        )
4698    }
4699
4700    fn i64_load_8s(
4701        &mut self,
4702        addr: Location,
4703        memarg: &MemArg,
4704        ret: Location,
4705        _need_check: bool,
4706        imported_memories: bool,
4707        offset: i32,
4708        heap_access_oob: Label,
4709        unaligned_atomic: Label,
4710    ) -> Result<(), CompileError> {
4711        self.memory_op(
4712            addr,
4713            memarg,
4714            false,
4715            1,
4716            imported_memories,
4717            offset,
4718            heap_access_oob,
4719            unaligned_atomic,
4720            |this, addr| this.emit_maybe_unaligned_load(Size::S8, true, ret, addr),
4721        )
4722    }
4723
4724    fn i64_load_32u(
4725        &mut self,
4726        addr: Location,
4727        memarg: &MemArg,
4728        ret: Location,
4729        _need_check: bool,
4730        imported_memories: bool,
4731        offset: i32,
4732        heap_access_oob: Label,
4733        unaligned_atomic: Label,
4734    ) -> Result<(), CompileError> {
4735        self.memory_op(
4736            addr,
4737            memarg,
4738            false,
4739            4,
4740            imported_memories,
4741            offset,
4742            heap_access_oob,
4743            unaligned_atomic,
4744            |this, addr| this.emit_maybe_unaligned_load(Size::S32, false, ret, addr),
4745        )
4746    }
4747
4748    fn i64_load_32s(
4749        &mut self,
4750        addr: Location,
4751        memarg: &MemArg,
4752        ret: Location,
4753        _need_check: bool,
4754        imported_memories: bool,
4755        offset: i32,
4756        heap_access_oob: Label,
4757        unaligned_atomic: Label,
4758    ) -> Result<(), CompileError> {
4759        self.memory_op(
4760            addr,
4761            memarg,
4762            false,
4763            4,
4764            imported_memories,
4765            offset,
4766            heap_access_oob,
4767            unaligned_atomic,
4768            |this, addr| this.emit_maybe_unaligned_load(Size::S32, true, ret, addr),
4769        )
4770    }
4771
4772    fn i64_load_16u(
4773        &mut self,
4774        addr: Location,
4775        memarg: &MemArg,
4776        ret: Location,
4777        _need_check: bool,
4778        imported_memories: bool,
4779        offset: i32,
4780        heap_access_oob: Label,
4781        unaligned_atomic: Label,
4782    ) -> Result<(), CompileError> {
4783        self.memory_op(
4784            addr,
4785            memarg,
4786            false,
4787            2,
4788            imported_memories,
4789            offset,
4790            heap_access_oob,
4791            unaligned_atomic,
4792            |this, addr| this.emit_maybe_unaligned_load(Size::S16, false, ret, addr),
4793        )
4794    }
4795
4796    fn i64_load_16s(
4797        &mut self,
4798        addr: Location,
4799        memarg: &MemArg,
4800        ret: Location,
4801        _need_check: bool,
4802        imported_memories: bool,
4803        offset: i32,
4804        heap_access_oob: Label,
4805        unaligned_atomic: Label,
4806    ) -> Result<(), CompileError> {
4807        self.memory_op(
4808            addr,
4809            memarg,
4810            false,
4811            2,
4812            imported_memories,
4813            offset,
4814            heap_access_oob,
4815            unaligned_atomic,
4816            |this, addr| this.emit_maybe_unaligned_load(Size::S16, true, ret, addr),
4817        )
4818    }
4819
4820    fn i64_atomic_load(
4821        &mut self,
4822        addr: Location,
4823        memarg: &MemArg,
4824        ret: Location,
4825        _need_check: bool,
4826        imported_memories: bool,
4827        offset: i32,
4828        heap_access_oob: Label,
4829        unaligned_atomic: Label,
4830    ) -> Result<(), CompileError> {
4831        self.memory_op(
4832            addr,
4833            memarg,
4834            true,
4835            8,
4836            imported_memories,
4837            offset,
4838            heap_access_oob,
4839            unaligned_atomic,
4840            |this, addr| this.emit_relaxed_load(Size::S64, true, ret, Location::Memory(addr, 0)),
4841        )
4842    }
4843
4844    fn i64_atomic_load_8u(
4845        &mut self,
4846        addr: Location,
4847        memarg: &MemArg,
4848        ret: Location,
4849        _need_check: bool,
4850        imported_memories: bool,
4851        offset: i32,
4852        heap_access_oob: Label,
4853        unaligned_atomic: Label,
4854    ) -> Result<(), CompileError> {
4855        self.memory_op(
4856            addr,
4857            memarg,
4858            true,
4859            1,
4860            imported_memories,
4861            offset,
4862            heap_access_oob,
4863            unaligned_atomic,
4864            |this, addr| this.emit_relaxed_load(Size::S8, false, ret, Location::Memory(addr, 0)),
4865        )
4866    }
4867
4868    fn i64_atomic_load_16u(
4869        &mut self,
4870        addr: Location,
4871        memarg: &MemArg,
4872        ret: Location,
4873        _need_check: bool,
4874        imported_memories: bool,
4875        offset: i32,
4876        heap_access_oob: Label,
4877        unaligned_atomic: Label,
4878    ) -> Result<(), CompileError> {
4879        self.memory_op(
4880            addr,
4881            memarg,
4882            true,
4883            2,
4884            imported_memories,
4885            offset,
4886            heap_access_oob,
4887            unaligned_atomic,
4888            |this, addr| this.emit_relaxed_load(Size::S16, false, ret, Location::Memory(addr, 0)),
4889        )
4890    }
4891
4892    fn i64_atomic_load_32u(
4893        &mut self,
4894        addr: Location,
4895        memarg: &MemArg,
4896        ret: Location,
4897        _need_check: bool,
4898        imported_memories: bool,
4899        offset: i32,
4900        heap_access_oob: Label,
4901        unaligned_atomic: Label,
4902    ) -> Result<(), CompileError> {
4903        self.memory_op(
4904            addr,
4905            memarg,
4906            true,
4907            4,
4908            imported_memories,
4909            offset,
4910            heap_access_oob,
4911            unaligned_atomic,
4912            |this, addr| this.emit_relaxed_load(Size::S32, false, ret, Location::Memory(addr, 0)),
4913        )
4914    }
4915
4916    fn i64_save(
4917        &mut self,
4918        value: Location,
4919        memarg: &MemArg,
4920        addr: Location,
4921        _need_check: bool,
4922        imported_memories: bool,
4923        offset: i32,
4924        heap_access_oob: Label,
4925        unaligned_atomic: Label,
4926    ) -> Result<(), CompileError> {
4927        self.memory_op(
4928            addr,
4929            memarg,
4930            false,
4931            8,
4932            imported_memories,
4933            offset,
4934            heap_access_oob,
4935            unaligned_atomic,
4936            |this, addr| this.emit_maybe_unaligned_store(Size::S64, value, addr),
4937        )
4938    }
4939
4940    fn i64_save_8(
4941        &mut self,
4942        value: Location,
4943        memarg: &MemArg,
4944        addr: Location,
4945        _need_check: bool,
4946        imported_memories: bool,
4947        offset: i32,
4948        heap_access_oob: Label,
4949        unaligned_atomic: Label,
4950    ) -> Result<(), CompileError> {
4951        self.memory_op(
4952            addr,
4953            memarg,
4954            false,
4955            1,
4956            imported_memories,
4957            offset,
4958            heap_access_oob,
4959            unaligned_atomic,
4960            |this, addr| this.emit_maybe_unaligned_store(Size::S8, value, addr),
4961        )
4962    }
4963
4964    fn i64_save_16(
4965        &mut self,
4966        value: Location,
4967        memarg: &MemArg,
4968        addr: Location,
4969        _need_check: bool,
4970        imported_memories: bool,
4971        offset: i32,
4972        heap_access_oob: Label,
4973        unaligned_atomic: Label,
4974    ) -> Result<(), CompileError> {
4975        self.memory_op(
4976            addr,
4977            memarg,
4978            false,
4979            2,
4980            imported_memories,
4981            offset,
4982            heap_access_oob,
4983            unaligned_atomic,
4984            |this, addr| this.emit_maybe_unaligned_store(Size::S16, value, addr),
4985        )
4986    }
4987
4988    fn i64_save_32(
4989        &mut self,
4990        value: Location,
4991        memarg: &MemArg,
4992        addr: Location,
4993        _need_check: bool,
4994        imported_memories: bool,
4995        offset: i32,
4996        heap_access_oob: Label,
4997        unaligned_atomic: Label,
4998    ) -> Result<(), CompileError> {
4999        self.memory_op(
5000            addr,
5001            memarg,
5002            false,
5003            4,
5004            imported_memories,
5005            offset,
5006            heap_access_oob,
5007            unaligned_atomic,
5008            |this, addr| this.emit_maybe_unaligned_store(Size::S32, value, addr),
5009        )
5010    }
5011
5012    fn i64_atomic_save(
5013        &mut self,
5014        value: Location,
5015        memarg: &MemArg,
5016        addr: Location,
5017        _need_check: bool,
5018        imported_memories: bool,
5019        offset: i32,
5020        heap_access_oob: Label,
5021        unaligned_atomic: Label,
5022    ) -> Result<(), CompileError> {
5023        self.memory_op(
5024            addr,
5025            memarg,
5026            true,
5027            8,
5028            imported_memories,
5029            offset,
5030            heap_access_oob,
5031            unaligned_atomic,
5032            |this, addr| this.emit_relaxed_store(Size::S64, value, Location::Memory(addr, 0)),
5033        )?;
5034        self.assembler.emit_rwfence()
5035    }
5036
5037    fn i64_atomic_save_8(
5038        &mut self,
5039        value: Location,
5040        memarg: &MemArg,
5041        addr: Location,
5042        _need_check: bool,
5043        imported_memories: bool,
5044        offset: i32,
5045        heap_access_oob: Label,
5046        unaligned_atomic: Label,
5047    ) -> Result<(), CompileError> {
5048        self.memory_op(
5049            addr,
5050            memarg,
5051            true,
5052            1,
5053            imported_memories,
5054            offset,
5055            heap_access_oob,
5056            unaligned_atomic,
5057            |this, addr| this.emit_relaxed_store(Size::S8, value, Location::Memory(addr, 0)),
5058        )?;
5059        self.assembler.emit_rwfence()
5060    }
5061
5062    fn i64_atomic_save_16(
5063        &mut self,
5064        value: Location,
5065        memarg: &MemArg,
5066        addr: Location,
5067        _need_check: bool,
5068        imported_memories: bool,
5069        offset: i32,
5070        heap_access_oob: Label,
5071        unaligned_atomic: Label,
5072    ) -> Result<(), CompileError> {
5073        self.memory_op(
5074            addr,
5075            memarg,
5076            true,
5077            2,
5078            imported_memories,
5079            offset,
5080            heap_access_oob,
5081            unaligned_atomic,
5082            |this, addr| this.emit_relaxed_store(Size::S16, value, Location::Memory(addr, 0)),
5083        )?;
5084        self.assembler.emit_rwfence()
5085    }
5086
5087    fn i64_atomic_save_32(
5088        &mut self,
5089        value: Location,
5090        memarg: &MemArg,
5091        addr: Location,
5092        _need_check: bool,
5093        imported_memories: bool,
5094        offset: i32,
5095        heap_access_oob: Label,
5096        unaligned_atomic: Label,
5097    ) -> Result<(), CompileError> {
5098        self.memory_op(
5099            addr,
5100            memarg,
5101            true,
5102            4,
5103            imported_memories,
5104            offset,
5105            heap_access_oob,
5106            unaligned_atomic,
5107            |this, addr| this.emit_relaxed_store(Size::S32, value, Location::Memory(addr, 0)),
5108        )?;
5109        self.assembler.emit_rwfence()
5110    }
5111
5112    fn i64_atomic_add(
5113        &mut self,
5114        loc: Location,
5115        target: Location,
5116        memarg: &MemArg,
5117        ret: Location,
5118        _need_check: bool,
5119        imported_memories: bool,
5120        offset: i32,
5121        heap_access_oob: Label,
5122        unaligned_atomic: Label,
5123    ) -> Result<(), CompileError> {
5124        self.memory_op(
5125            target,
5126            memarg,
5127            true,
5128            8,
5129            imported_memories,
5130            offset,
5131            heap_access_oob,
5132            unaligned_atomic,
5133            |this, addr| {
5134                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Add, Size::S64, ret, addr, loc)
5135            },
5136        )
5137    }
5138
5139    fn i64_atomic_add_8u(
5140        &mut self,
5141        loc: Location,
5142        target: Location,
5143        memarg: &MemArg,
5144        ret: Location,
5145        _need_check: bool,
5146        imported_memories: bool,
5147        offset: i32,
5148        heap_access_oob: Label,
5149        unaligned_atomic: Label,
5150    ) -> Result<(), CompileError> {
5151        self.memory_op(
5152            target,
5153            memarg,
5154            true,
5155            1,
5156            imported_memories,
5157            offset,
5158            heap_access_oob,
5159            unaligned_atomic,
5160            |this, addr| {
5161                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Add, Size::S8, ret, addr, loc)
5162            },
5163        )
5164    }
5165
5166    fn i64_atomic_add_16u(
5167        &mut self,
5168        loc: Location,
5169        target: Location,
5170        memarg: &MemArg,
5171        ret: Location,
5172        _need_check: bool,
5173        imported_memories: bool,
5174        offset: i32,
5175        heap_access_oob: Label,
5176        unaligned_atomic: Label,
5177    ) -> Result<(), CompileError> {
5178        self.memory_op(
5179            target,
5180            memarg,
5181            true,
5182            2,
5183            imported_memories,
5184            offset,
5185            heap_access_oob,
5186            unaligned_atomic,
5187            |this, addr| {
5188                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Add, Size::S16, ret, addr, loc)
5189            },
5190        )
5191    }
5192
5193    fn i64_atomic_add_32u(
5194        &mut self,
5195        loc: Location,
5196        target: Location,
5197        memarg: &MemArg,
5198        ret: Location,
5199        _need_check: bool,
5200        imported_memories: bool,
5201        offset: i32,
5202        heap_access_oob: Label,
5203        unaligned_atomic: Label,
5204    ) -> Result<(), CompileError> {
5205        self.memory_op(
5206            target,
5207            memarg,
5208            true,
5209            4,
5210            imported_memories,
5211            offset,
5212            heap_access_oob,
5213            unaligned_atomic,
5214            |this, addr| {
5215                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Add, Size::S32, ret, addr, loc)
5216            },
5217        )
5218    }
5219
5220    fn i64_atomic_sub(
5221        &mut self,
5222        loc: Location,
5223        target: Location,
5224        memarg: &MemArg,
5225        ret: Location,
5226        _need_check: bool,
5227        imported_memories: bool,
5228        offset: i32,
5229        heap_access_oob: Label,
5230        unaligned_atomic: Label,
5231    ) -> Result<(), CompileError> {
5232        self.memory_op(
5233            target,
5234            memarg,
5235            true,
5236            8,
5237            imported_memories,
5238            offset,
5239            heap_access_oob,
5240            unaligned_atomic,
5241            |this, addr| {
5242                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Sub, Size::S64, ret, addr, loc)
5243            },
5244        )
5245    }
5246
5247    fn i64_atomic_sub_8u(
5248        &mut self,
5249        loc: Location,
5250        target: Location,
5251        memarg: &MemArg,
5252        ret: Location,
5253        _need_check: bool,
5254        imported_memories: bool,
5255        offset: i32,
5256        heap_access_oob: Label,
5257        unaligned_atomic: Label,
5258    ) -> Result<(), CompileError> {
5259        self.memory_op(
5260            target,
5261            memarg,
5262            true,
5263            1,
5264            imported_memories,
5265            offset,
5266            heap_access_oob,
5267            unaligned_atomic,
5268            |this, addr| {
5269                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Sub, Size::S8, ret, addr, loc)
5270            },
5271        )
5272    }
5273
5274    fn i64_atomic_sub_16u(
5275        &mut self,
5276        loc: Location,
5277        target: Location,
5278        memarg: &MemArg,
5279        ret: Location,
5280        _need_check: bool,
5281        imported_memories: bool,
5282        offset: i32,
5283        heap_access_oob: Label,
5284        unaligned_atomic: Label,
5285    ) -> Result<(), CompileError> {
5286        self.memory_op(
5287            target,
5288            memarg,
5289            true,
5290            2,
5291            imported_memories,
5292            offset,
5293            heap_access_oob,
5294            unaligned_atomic,
5295            |this, addr| {
5296                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Sub, Size::S16, ret, addr, loc)
5297            },
5298        )
5299    }
5300
5301    fn i64_atomic_sub_32u(
5302        &mut self,
5303        loc: Location,
5304        target: Location,
5305        memarg: &MemArg,
5306        ret: Location,
5307        _need_check: bool,
5308        imported_memories: bool,
5309        offset: i32,
5310        heap_access_oob: Label,
5311        unaligned_atomic: Label,
5312    ) -> Result<(), CompileError> {
5313        self.memory_op(
5314            target,
5315            memarg,
5316            true,
5317            4,
5318            imported_memories,
5319            offset,
5320            heap_access_oob,
5321            unaligned_atomic,
5322            |this, addr| {
5323                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Sub, Size::S32, ret, addr, loc)
5324            },
5325        )
5326    }
5327
5328    fn i64_atomic_and(
5329        &mut self,
5330        loc: Location,
5331        target: Location,
5332        memarg: &MemArg,
5333        ret: Location,
5334        _need_check: bool,
5335        imported_memories: bool,
5336        offset: i32,
5337        heap_access_oob: Label,
5338        unaligned_atomic: Label,
5339    ) -> Result<(), CompileError> {
5340        self.memory_op(
5341            target,
5342            memarg,
5343            true,
5344            8,
5345            imported_memories,
5346            offset,
5347            heap_access_oob,
5348            unaligned_atomic,
5349            |this, addr| {
5350                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::And, Size::S64, ret, addr, loc)
5351            },
5352        )
5353    }
5354
5355    fn i64_atomic_and_8u(
5356        &mut self,
5357        loc: Location,
5358        target: Location,
5359        memarg: &MemArg,
5360        ret: Location,
5361        _need_check: bool,
5362        imported_memories: bool,
5363        offset: i32,
5364        heap_access_oob: Label,
5365        unaligned_atomic: Label,
5366    ) -> Result<(), CompileError> {
5367        self.memory_op(
5368            target,
5369            memarg,
5370            true,
5371            1,
5372            imported_memories,
5373            offset,
5374            heap_access_oob,
5375            unaligned_atomic,
5376            |this, addr| {
5377                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::And, Size::S8, ret, addr, loc)
5378            },
5379        )
5380    }
5381
5382    fn i64_atomic_and_16u(
5383        &mut self,
5384        loc: Location,
5385        target: Location,
5386        memarg: &MemArg,
5387        ret: Location,
5388        _need_check: bool,
5389        imported_memories: bool,
5390        offset: i32,
5391        heap_access_oob: Label,
5392        unaligned_atomic: Label,
5393    ) -> Result<(), CompileError> {
5394        self.memory_op(
5395            target,
5396            memarg,
5397            true,
5398            2,
5399            imported_memories,
5400            offset,
5401            heap_access_oob,
5402            unaligned_atomic,
5403            |this, addr| {
5404                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::And, Size::S16, ret, addr, loc)
5405            },
5406        )
5407    }
5408
5409    fn i64_atomic_and_32u(
5410        &mut self,
5411        loc: Location,
5412        target: Location,
5413        memarg: &MemArg,
5414        ret: Location,
5415        _need_check: bool,
5416        imported_memories: bool,
5417        offset: i32,
5418        heap_access_oob: Label,
5419        unaligned_atomic: Label,
5420    ) -> Result<(), CompileError> {
5421        self.memory_op(
5422            target,
5423            memarg,
5424            true,
5425            4,
5426            imported_memories,
5427            offset,
5428            heap_access_oob,
5429            unaligned_atomic,
5430            |this, addr| {
5431                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::And, Size::S32, ret, addr, loc)
5432            },
5433        )
5434    }
5435
5436    fn i64_atomic_or(
5437        &mut self,
5438        loc: Location,
5439        target: Location,
5440        memarg: &MemArg,
5441        ret: Location,
5442        _need_check: bool,
5443        imported_memories: bool,
5444        offset: i32,
5445        heap_access_oob: Label,
5446        unaligned_atomic: Label,
5447    ) -> Result<(), CompileError> {
5448        self.memory_op(
5449            target,
5450            memarg,
5451            true,
5452            8,
5453            imported_memories,
5454            offset,
5455            heap_access_oob,
5456            unaligned_atomic,
5457            |this, addr| {
5458                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Or, Size::S64, ret, addr, loc)
5459            },
5460        )
5461    }
5462
5463    fn i64_atomic_or_8u(
5464        &mut self,
5465        loc: Location,
5466        target: Location,
5467        memarg: &MemArg,
5468        ret: Location,
5469        _need_check: bool,
5470        imported_memories: bool,
5471        offset: i32,
5472        heap_access_oob: Label,
5473        unaligned_atomic: Label,
5474    ) -> Result<(), CompileError> {
5475        self.memory_op(
5476            target,
5477            memarg,
5478            true,
5479            1,
5480            imported_memories,
5481            offset,
5482            heap_access_oob,
5483            unaligned_atomic,
5484            |this, addr| {
5485                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Or, Size::S8, ret, addr, loc)
5486            },
5487        )
5488    }
5489
5490    fn i64_atomic_or_16u(
5491        &mut self,
5492        loc: Location,
5493        target: Location,
5494        memarg: &MemArg,
5495        ret: Location,
5496        _need_check: bool,
5497        imported_memories: bool,
5498        offset: i32,
5499        heap_access_oob: Label,
5500        unaligned_atomic: Label,
5501    ) -> Result<(), CompileError> {
5502        self.memory_op(
5503            target,
5504            memarg,
5505            true,
5506            2,
5507            imported_memories,
5508            offset,
5509            heap_access_oob,
5510            unaligned_atomic,
5511            |this, addr| {
5512                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Or, Size::S16, ret, addr, loc)
5513            },
5514        )
5515    }
5516
5517    fn i64_atomic_or_32u(
5518        &mut self,
5519        loc: Location,
5520        target: Location,
5521        memarg: &MemArg,
5522        ret: Location,
5523        _need_check: bool,
5524        imported_memories: bool,
5525        offset: i32,
5526        heap_access_oob: Label,
5527        unaligned_atomic: Label,
5528    ) -> Result<(), CompileError> {
5529        self.memory_op(
5530            target,
5531            memarg,
5532            true,
5533            4,
5534            imported_memories,
5535            offset,
5536            heap_access_oob,
5537            unaligned_atomic,
5538            |this, addr| {
5539                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Or, Size::S32, ret, addr, loc)
5540            },
5541        )
5542    }
5543
5544    fn i64_atomic_xor(
5545        &mut self,
5546        loc: Location,
5547        target: Location,
5548        memarg: &MemArg,
5549        ret: Location,
5550        _need_check: bool,
5551        imported_memories: bool,
5552        offset: i32,
5553        heap_access_oob: Label,
5554        unaligned_atomic: Label,
5555    ) -> Result<(), CompileError> {
5556        self.memory_op(
5557            target,
5558            memarg,
5559            true,
5560            8,
5561            imported_memories,
5562            offset,
5563            heap_access_oob,
5564            unaligned_atomic,
5565            |this, addr| {
5566                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Xor, Size::S64, ret, addr, loc)
5567            },
5568        )
5569    }
5570
5571    fn i64_atomic_xor_8u(
5572        &mut self,
5573        loc: Location,
5574        target: Location,
5575        memarg: &MemArg,
5576        ret: Location,
5577        _need_check: bool,
5578        imported_memories: bool,
5579        offset: i32,
5580        heap_access_oob: Label,
5581        unaligned_atomic: Label,
5582    ) -> Result<(), CompileError> {
5583        self.memory_op(
5584            target,
5585            memarg,
5586            true,
5587            1,
5588            imported_memories,
5589            offset,
5590            heap_access_oob,
5591            unaligned_atomic,
5592            |this, addr| {
5593                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Xor, Size::S8, ret, addr, loc)
5594            },
5595        )
5596    }
5597
5598    fn i64_atomic_xor_16u(
5599        &mut self,
5600        loc: Location,
5601        target: Location,
5602        memarg: &MemArg,
5603        ret: Location,
5604        _need_check: bool,
5605        imported_memories: bool,
5606        offset: i32,
5607        heap_access_oob: Label,
5608        unaligned_atomic: Label,
5609    ) -> Result<(), CompileError> {
5610        self.memory_op(
5611            target,
5612            memarg,
5613            true,
5614            2,
5615            imported_memories,
5616            offset,
5617            heap_access_oob,
5618            unaligned_atomic,
5619            |this, addr| {
5620                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Xor, Size::S16, ret, addr, loc)
5621            },
5622        )
5623    }
5624
5625    fn i64_atomic_xor_32u(
5626        &mut self,
5627        loc: Location,
5628        target: Location,
5629        memarg: &MemArg,
5630        ret: Location,
5631        _need_check: bool,
5632        imported_memories: bool,
5633        offset: i32,
5634        heap_access_oob: Label,
5635        unaligned_atomic: Label,
5636    ) -> Result<(), CompileError> {
5637        self.memory_op(
5638            target,
5639            memarg,
5640            true,
5641            4,
5642            imported_memories,
5643            offset,
5644            heap_access_oob,
5645            unaligned_atomic,
5646            |this, addr| {
5647                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Xor, Size::S32, ret, addr, loc)
5648            },
5649        )
5650    }
5651
5652    fn i64_atomic_xchg(
5653        &mut self,
5654        loc: Location,
5655        target: Location,
5656        memarg: &MemArg,
5657        ret: Location,
5658        _need_check: bool,
5659        imported_memories: bool,
5660        offset: i32,
5661        heap_access_oob: Label,
5662        unaligned_atomic: Label,
5663    ) -> Result<(), CompileError> {
5664        self.memory_op(
5665            target,
5666            memarg,
5667            true,
5668            8,
5669            imported_memories,
5670            offset,
5671            heap_access_oob,
5672            unaligned_atomic,
5673            |this, addr| {
5674                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Exchange, Size::S64, ret, addr, loc)
5675            },
5676        )
5677    }
5678
5679    fn i64_atomic_xchg_8u(
5680        &mut self,
5681        loc: Location,
5682        target: Location,
5683        memarg: &MemArg,
5684        ret: Location,
5685        _need_check: bool,
5686        imported_memories: bool,
5687        offset: i32,
5688        heap_access_oob: Label,
5689        unaligned_atomic: Label,
5690    ) -> Result<(), CompileError> {
5691        self.memory_op(
5692            target,
5693            memarg,
5694            true,
5695            1,
5696            imported_memories,
5697            offset,
5698            heap_access_oob,
5699            unaligned_atomic,
5700            |this, addr| {
5701                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Exchange, Size::S8, ret, addr, loc)
5702            },
5703        )
5704    }
5705
5706    fn i64_atomic_xchg_16u(
5707        &mut self,
5708        loc: Location,
5709        target: Location,
5710        memarg: &MemArg,
5711        ret: Location,
5712        _need_check: bool,
5713        imported_memories: bool,
5714        offset: i32,
5715        heap_access_oob: Label,
5716        unaligned_atomic: Label,
5717    ) -> Result<(), CompileError> {
5718        self.memory_op(
5719            target,
5720            memarg,
5721            true,
5722            2,
5723            imported_memories,
5724            offset,
5725            heap_access_oob,
5726            unaligned_atomic,
5727            |this, addr| {
5728                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Exchange, Size::S16, ret, addr, loc)
5729            },
5730        )
5731    }
5732
5733    fn i64_atomic_xchg_32u(
5734        &mut self,
5735        loc: Location,
5736        target: Location,
5737        memarg: &MemArg,
5738        ret: Location,
5739        _need_check: bool,
5740        imported_memories: bool,
5741        offset: i32,
5742        heap_access_oob: Label,
5743        unaligned_atomic: Label,
5744    ) -> Result<(), CompileError> {
5745        self.memory_op(
5746            target,
5747            memarg,
5748            true,
5749            4,
5750            imported_memories,
5751            offset,
5752            heap_access_oob,
5753            unaligned_atomic,
5754            |this, addr| {
5755                this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Exchange, Size::S32, ret, addr, loc)
5756            },
5757        )
5758    }
5759
5760    fn i64_atomic_cmpxchg(
5761        &mut self,
5762        new: Location,
5763        cmp: Location,
5764        target: Location,
5765        memarg: &MemArg,
5766        ret: Location,
5767        _need_check: bool,
5768        imported_memories: bool,
5769        offset: i32,
5770        heap_access_oob: Label,
5771        unaligned_atomic: Label,
5772    ) -> Result<(), CompileError> {
5773        self.memory_op(
5774            target,
5775            memarg,
5776            true,
5777            8,
5778            imported_memories,
5779            offset,
5780            heap_access_oob,
5781            unaligned_atomic,
5782            |this, addr| this.emit_relaxed_atomic_cmpxchg(Size::S64, ret, addr, new, cmp),
5783        )
5784    }
5785
5786    fn i64_atomic_cmpxchg_8u(
5787        &mut self,
5788        new: Location,
5789        cmp: Location,
5790        target: Location,
5791        memarg: &MemArg,
5792        ret: Location,
5793        _need_check: bool,
5794        imported_memories: bool,
5795        offset: i32,
5796        heap_access_oob: Label,
5797        unaligned_atomic: Label,
5798    ) -> Result<(), CompileError> {
5799        self.memory_op(
5800            target,
5801            memarg,
5802            true,
5803            1,
5804            imported_memories,
5805            offset,
5806            heap_access_oob,
5807            unaligned_atomic,
5808            |this, addr| this.emit_relaxed_atomic_cmpxchg(Size::S8, ret, addr, new, cmp),
5809        )
5810    }
5811
5812    fn i64_atomic_cmpxchg_16u(
5813        &mut self,
5814        new: Location,
5815        cmp: Location,
5816        target: Location,
5817        memarg: &MemArg,
5818        ret: Location,
5819        _need_check: bool,
5820        imported_memories: bool,
5821        offset: i32,
5822        heap_access_oob: Label,
5823        unaligned_atomic: Label,
5824    ) -> Result<(), CompileError> {
5825        self.memory_op(
5826            target,
5827            memarg,
5828            true,
5829            2,
5830            imported_memories,
5831            offset,
5832            heap_access_oob,
5833            unaligned_atomic,
5834            |this, addr| this.emit_relaxed_atomic_cmpxchg(Size::S16, ret, addr, new, cmp),
5835        )
5836    }
5837
5838    fn i64_atomic_cmpxchg_32u(
5839        &mut self,
5840        new: Location,
5841        cmp: Location,
5842        target: Location,
5843        memarg: &MemArg,
5844        ret: Location,
5845        _need_check: bool,
5846        imported_memories: bool,
5847        offset: i32,
5848        heap_access_oob: Label,
5849        unaligned_atomic: Label,
5850    ) -> Result<(), CompileError> {
5851        self.memory_op(
5852            target,
5853            memarg,
5854            true,
5855            4,
5856            imported_memories,
5857            offset,
5858            heap_access_oob,
5859            unaligned_atomic,
5860            |this, addr| this.emit_relaxed_atomic_cmpxchg(Size::S32, ret, addr, new, cmp),
5861        )
5862    }
5863
5864    fn f32_load(
5865        &mut self,
5866        addr: Location,
5867        memarg: &MemArg,
5868        ret: Location,
5869        _need_check: bool,
5870        imported_memories: bool,
5871        offset: i32,
5872        heap_access_oob: Label,
5873        unaligned_atomic: Label,
5874    ) -> Result<(), CompileError> {
5875        self.memory_op(
5876            addr,
5877            memarg,
5878            false,
5879            4,
5880            imported_memories,
5881            offset,
5882            heap_access_oob,
5883            unaligned_atomic,
5884            |this, addr| this.emit_relaxed_load(Size::S32, false, ret, Location::Memory(addr, 0)),
5885        )
5886    }
5887
5888    fn f32_save(
5889        &mut self,
5890        value: Location,
5891        memarg: &MemArg,
5892        addr: Location,
5893        canonicalize: bool,
5894        _need_check: bool,
5895        imported_memories: bool,
5896        offset: i32,
5897        heap_access_oob: Label,
5898        unaligned_atomic: Label,
5899    ) -> Result<(), CompileError> {
5900        self.memory_op(
5901            addr,
5902            memarg,
5903            false,
5904            4,
5905            imported_memories,
5906            offset,
5907            heap_access_oob,
5908            unaligned_atomic,
5909            |this, addr| {
5910                if !canonicalize {
5911                    this.emit_relaxed_store(Size::S32, value, Location::Memory(addr, 0))
5912                } else {
5913                    this.canonicalize_nan(Size::S32, value, Location::Memory(addr, 0))
5914                }
5915            },
5916        )
5917    }
5918
5919    fn f64_load(
5920        &mut self,
5921        addr: Location,
5922        memarg: &MemArg,
5923        ret: Location,
5924        _need_check: bool,
5925        imported_memories: bool,
5926        offset: i32,
5927        heap_access_oob: Label,
5928        unaligned_atomic: Label,
5929    ) -> Result<(), CompileError> {
5930        self.memory_op(
5931            addr,
5932            memarg,
5933            false,
5934            8,
5935            imported_memories,
5936            offset,
5937            heap_access_oob,
5938            unaligned_atomic,
5939            |this, addr| this.emit_relaxed_load(Size::S64, false, ret, Location::Memory(addr, 0)),
5940        )
5941    }
5942
5943    fn f64_save(
5944        &mut self,
5945        value: Location,
5946        memarg: &MemArg,
5947        addr: Location,
5948        canonicalize: bool,
5949        _need_check: bool,
5950        imported_memories: bool,
5951        offset: i32,
5952        heap_access_oob: Label,
5953        unaligned_atomic: Label,
5954    ) -> Result<(), CompileError> {
5955        self.memory_op(
5956            addr,
5957            memarg,
5958            false,
5959            8,
5960            imported_memories,
5961            offset,
5962            heap_access_oob,
5963            unaligned_atomic,
5964            |this, addr| {
5965                if !canonicalize {
5966                    this.emit_relaxed_store(Size::S64, value, Location::Memory(addr, 0))
5967                } else {
5968                    this.canonicalize_nan(Size::S64, value, Location::Memory(addr, 0))
5969                }
5970            },
5971        )
5972    }
5973
5974    fn convert_f64_i64(
5975        &mut self,
5976        loc: Location,
5977        signed: bool,
5978        ret: Location,
5979    ) -> Result<(), CompileError> {
5980        self.convert_int_to_float(loc, Size::S64, ret, Size::S64, signed)
5981    }
5982
5983    fn convert_f64_i32(
5984        &mut self,
5985        loc: Location,
5986        signed: bool,
5987        ret: Location,
5988    ) -> Result<(), CompileError> {
5989        self.convert_int_to_float(loc, Size::S32, ret, Size::S64, signed)
5990    }
5991
5992    fn convert_f32_i64(
5993        &mut self,
5994        loc: Location,
5995        signed: bool,
5996        ret: Location,
5997    ) -> Result<(), CompileError> {
5998        self.convert_int_to_float(loc, Size::S64, ret, Size::S32, signed)
5999    }
6000
6001    fn convert_f32_i32(
6002        &mut self,
6003        loc: Location,
6004        signed: bool,
6005        ret: Location,
6006    ) -> Result<(), CompileError> {
6007        self.convert_int_to_float(loc, Size::S32, ret, Size::S32, signed)
6008    }
6009
6010    fn convert_i64_f64(
6011        &mut self,
6012        loc: Location,
6013        ret: Location,
6014        signed: bool,
6015        sat: bool,
6016    ) -> Result<(), CompileError> {
6017        self.convert_float_to_int(loc, Size::S64, ret, Size::S64, signed, sat)
6018    }
6019
6020    fn convert_i32_f64(
6021        &mut self,
6022        loc: Location,
6023        ret: Location,
6024        signed: bool,
6025        sat: bool,
6026    ) -> Result<(), CompileError> {
6027        self.convert_float_to_int(loc, Size::S64, ret, Size::S32, signed, sat)
6028    }
6029
6030    fn convert_i64_f32(
6031        &mut self,
6032        loc: Location,
6033        ret: Location,
6034        signed: bool,
6035        sat: bool,
6036    ) -> Result<(), CompileError> {
6037        self.convert_float_to_int(loc, Size::S32, ret, Size::S64, signed, sat)
6038    }
6039
6040    fn convert_i32_f32(
6041        &mut self,
6042        loc: Location,
6043        ret: Location,
6044        signed: bool,
6045        sat: bool,
6046    ) -> Result<(), CompileError> {
6047        self.convert_float_to_int(loc, Size::S32, ret, Size::S32, signed, sat)
6048    }
6049
6050    fn convert_f64_f32(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6051        self.convert_float_to_float(loc, Size::S32, ret, Size::S64)
6052    }
6053
6054    fn convert_f32_f64(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6055        self.convert_float_to_float(loc, Size::S64, ret, Size::S32)
6056    }
6057
6058    fn f64_neg(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6059        self.emit_relaxed_binop_fp(Assembler::emit_fneg, Size::S64, loc, ret, true)
6060    }
6061
6062    fn f64_abs(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6063        let tmp = self.acquire_temp_gpr().ok_or_else(|| {
6064            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6065        })?;
6066        let mask = self.acquire_temp_gpr().ok_or_else(|| {
6067            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6068        })?;
6069
6070        self.move_location(Size::S64, loc, Location::GPR(tmp))?;
6071        self.assembler
6072            .emit_mov_imm(Location::GPR(mask), 0x7fffffffffffffffi64)?;
6073        self.assembler.emit_and(
6074            Size::S64,
6075            Location::GPR(tmp),
6076            Location::GPR(mask),
6077            Location::GPR(tmp),
6078        )?;
6079        self.move_location(Size::S64, Location::GPR(tmp), ret)?;
6080
6081        self.release_gpr(tmp);
6082        self.release_gpr(mask);
6083        Ok(())
6084    }
6085
6086    fn emit_i64_copysign(&mut self, tmp1: Self::GPR, tmp2: Self::GPR) -> Result<(), CompileError> {
6087        let mask = self.acquire_temp_gpr().ok_or_else(|| {
6088            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6089        })?;
6090
6091        self.assembler
6092            .emit_mov_imm(Location::GPR(mask), 0x7fffffffffffffffu64 as _)?;
6093        self.assembler.emit_and(
6094            Size::S64,
6095            Location::GPR(tmp1),
6096            Location::GPR(mask),
6097            Location::GPR(tmp1),
6098        )?;
6099
6100        self.assembler
6101            .emit_mov_imm(Location::GPR(mask), 0x8000000000000000u64 as _)?;
6102        self.assembler.emit_and(
6103            Size::S64,
6104            Location::GPR(tmp2),
6105            Location::GPR(mask),
6106            Location::GPR(tmp2),
6107        )?;
6108
6109        self.release_gpr(mask);
6110        self.assembler.emit_or(
6111            Size::S64,
6112            Location::GPR(tmp1),
6113            Location::GPR(tmp2),
6114            Location::GPR(tmp1),
6115        )
6116    }
6117
6118    fn f64_sqrt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6119        self.emit_relaxed_binop_fp(Assembler::emit_fsqrt, Size::S64, loc, ret, true)
6120    }
6121
6122    fn f64_trunc(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6123        self.emit_relaxed_fcvt_with_rounding(RoundingMode::Rtz, Size::S64, loc, ret)
6124    }
6125
6126    fn f64_ceil(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6127        self.emit_relaxed_fcvt_with_rounding(RoundingMode::Rup, Size::S64, loc, ret)
6128    }
6129
6130    fn f64_floor(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6131        self.emit_relaxed_fcvt_with_rounding(RoundingMode::Rdn, Size::S64, loc, ret)
6132    }
6133
6134    fn f64_nearest(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6135        self.emit_relaxed_fcvt_with_rounding(RoundingMode::Rne, Size::S64, loc, ret)
6136    }
6137
6138    fn f64_cmp_ge(
6139        &mut self,
6140        loc_a: Location,
6141        loc_b: Location,
6142        ret: Location,
6143    ) -> Result<(), CompileError> {
6144        self.emit_relaxed_fcmp(Condition::Ge, Size::S64, loc_a, loc_b, ret)
6145    }
6146
6147    fn f64_cmp_gt(
6148        &mut self,
6149        loc_a: Location,
6150        loc_b: Location,
6151        ret: Location,
6152    ) -> Result<(), CompileError> {
6153        self.emit_relaxed_fcmp(Condition::Gt, Size::S64, loc_a, loc_b, ret)
6154    }
6155
6156    fn f64_cmp_le(
6157        &mut self,
6158        loc_a: Location,
6159        loc_b: Location,
6160        ret: Location,
6161    ) -> Result<(), CompileError> {
6162        self.emit_relaxed_fcmp(Condition::Le, Size::S64, loc_a, loc_b, ret)
6163    }
6164
6165    fn f64_cmp_lt(
6166        &mut self,
6167        loc_a: Location,
6168        loc_b: Location,
6169        ret: Location,
6170    ) -> Result<(), CompileError> {
6171        self.emit_relaxed_fcmp(Condition::Lt, Size::S64, loc_a, loc_b, ret)
6172    }
6173
6174    fn f64_cmp_ne(
6175        &mut self,
6176        loc_a: Location,
6177        loc_b: Location,
6178        ret: Location,
6179    ) -> Result<(), CompileError> {
6180        self.emit_relaxed_fcmp(Condition::Ne, Size::S64, loc_a, loc_b, ret)
6181    }
6182
6183    fn f64_cmp_eq(
6184        &mut self,
6185        loc_a: Location,
6186        loc_b: Location,
6187        ret: Location,
6188    ) -> Result<(), CompileError> {
6189        self.emit_relaxed_fcmp(Condition::Eq, Size::S64, loc_a, loc_b, ret)
6190    }
6191
6192    fn f64_min(
6193        &mut self,
6194        loc_a: Location,
6195        loc_b: Location,
6196        ret: Location,
6197    ) -> Result<(), CompileError> {
6198        self.emit_relaxed_binop3_fp(
6199            Assembler::emit_fmin,
6200            Size::S64,
6201            loc_a,
6202            loc_b,
6203            ret,
6204            ImmType::None,
6205            true,
6206        )
6207    }
6208
6209    fn f64_max(
6210        &mut self,
6211        loc_a: Location,
6212        loc_b: Location,
6213        ret: Location,
6214    ) -> Result<(), CompileError> {
6215        self.emit_relaxed_binop3_fp(
6216            Assembler::emit_fmax,
6217            Size::S64,
6218            loc_a,
6219            loc_b,
6220            ret,
6221            ImmType::None,
6222            true,
6223        )
6224    }
6225
6226    fn f64_add(
6227        &mut self,
6228        loc_a: Location,
6229        loc_b: Location,
6230        ret: Location,
6231    ) -> Result<(), CompileError> {
6232        self.emit_relaxed_binop3_fp(
6233            Assembler::emit_add,
6234            Size::S64,
6235            loc_a,
6236            loc_b,
6237            ret,
6238            ImmType::None,
6239            false,
6240        )
6241    }
6242
6243    fn f64_sub(
6244        &mut self,
6245        loc_a: Location,
6246        loc_b: Location,
6247        ret: Location,
6248    ) -> Result<(), CompileError> {
6249        self.emit_relaxed_binop3_fp(
6250            Assembler::emit_sub,
6251            Size::S64,
6252            loc_a,
6253            loc_b,
6254            ret,
6255            ImmType::None,
6256            false,
6257        )
6258    }
6259
6260    fn f64_mul(
6261        &mut self,
6262        loc_a: Location,
6263        loc_b: Location,
6264        ret: Location,
6265    ) -> Result<(), CompileError> {
6266        self.emit_relaxed_binop3_fp(
6267            Assembler::emit_mul,
6268            Size::S64,
6269            loc_a,
6270            loc_b,
6271            ret,
6272            ImmType::None,
6273            false,
6274        )
6275    }
6276
6277    fn f64_div(
6278        &mut self,
6279        loc_a: Location,
6280        loc_b: Location,
6281        ret: Location,
6282    ) -> Result<(), CompileError> {
6283        self.emit_relaxed_binop3_fp(
6284            Assembler::emit_fdiv,
6285            Size::S64,
6286            loc_a,
6287            loc_b,
6288            ret,
6289            ImmType::None,
6290            false,
6291        )
6292    }
6293
6294    fn f32_neg(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6295        self.emit_relaxed_binop_fp(Assembler::emit_fneg, Size::S32, loc, ret, true)
6296    }
6297
6298    fn f32_abs(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6299        let tmp = self.acquire_temp_gpr().ok_or_else(|| {
6300            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6301        })?;
6302        let mask = self.acquire_temp_gpr().ok_or_else(|| {
6303            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6304        })?;
6305
6306        self.move_location(Size::S32, loc, Location::GPR(tmp))?;
6307        self.assembler
6308            .emit_mov_imm(Location::GPR(mask), 0x7fffffffi64)?;
6309        self.assembler.emit_and(
6310            Size::S32,
6311            Location::GPR(tmp),
6312            Location::GPR(mask),
6313            Location::GPR(tmp),
6314        )?;
6315        self.move_location(Size::S32, Location::GPR(tmp), ret)?;
6316
6317        self.release_gpr(tmp);
6318        self.release_gpr(mask);
6319        Ok(())
6320    }
6321
6322    fn emit_i32_copysign(&mut self, tmp1: Self::GPR, tmp2: Self::GPR) -> Result<(), CompileError> {
6323        let mask = self.acquire_temp_gpr().ok_or_else(|| {
6324            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6325        })?;
6326
6327        self.assembler
6328            .emit_mov_imm(Location::GPR(mask), 0x7fffffffu32 as _)?;
6329        self.assembler.emit_and(
6330            Size::S32,
6331            Location::GPR(tmp1),
6332            Location::GPR(mask),
6333            Location::GPR(tmp1),
6334        )?;
6335
6336        self.assembler
6337            .emit_mov_imm(Location::GPR(mask), 0x80000000u32 as _)?;
6338        self.assembler.emit_and(
6339            Size::S32,
6340            Location::GPR(tmp2),
6341            Location::GPR(mask),
6342            Location::GPR(tmp2),
6343        )?;
6344
6345        self.release_gpr(mask);
6346        self.assembler.emit_or(
6347            Size::S32,
6348            Location::GPR(tmp1),
6349            Location::GPR(tmp2),
6350            Location::GPR(tmp1),
6351        )
6352    }
6353
6354    fn f32_sqrt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6355        self.emit_relaxed_binop_fp(Assembler::emit_fsqrt, Size::S32, loc, ret, true)
6356    }
6357
6358    fn f32_trunc(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6359        self.emit_relaxed_fcvt_with_rounding(RoundingMode::Rtz, Size::S32, loc, ret)
6360    }
6361
6362    fn f32_ceil(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6363        self.emit_relaxed_fcvt_with_rounding(RoundingMode::Rup, Size::S32, loc, ret)
6364    }
6365
6366    fn f32_floor(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6367        self.emit_relaxed_fcvt_with_rounding(RoundingMode::Rdn, Size::S32, loc, ret)
6368    }
6369
6370    fn f32_nearest(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6371        self.emit_relaxed_fcvt_with_rounding(RoundingMode::Rne, Size::S32, loc, ret)
6372    }
6373
6374    fn f32_cmp_ge(
6375        &mut self,
6376        loc_a: Location,
6377        loc_b: Location,
6378        ret: Location,
6379    ) -> Result<(), CompileError> {
6380        self.emit_relaxed_fcmp(Condition::Ge, Size::S32, loc_a, loc_b, ret)
6381    }
6382
6383    fn f32_cmp_gt(
6384        &mut self,
6385        loc_a: Location,
6386        loc_b: Location,
6387        ret: Location,
6388    ) -> Result<(), CompileError> {
6389        self.emit_relaxed_fcmp(Condition::Gt, Size::S32, loc_a, loc_b, ret)
6390    }
6391
6392    fn f32_cmp_le(
6393        &mut self,
6394        loc_a: Location,
6395        loc_b: Location,
6396        ret: Location,
6397    ) -> Result<(), CompileError> {
6398        self.emit_relaxed_fcmp(Condition::Le, Size::S32, loc_a, loc_b, ret)
6399    }
6400
6401    fn f32_cmp_lt(
6402        &mut self,
6403        loc_a: Location,
6404        loc_b: Location,
6405        ret: Location,
6406    ) -> Result<(), CompileError> {
6407        self.emit_relaxed_fcmp(Condition::Lt, Size::S32, loc_a, loc_b, ret)
6408    }
6409
6410    fn f32_cmp_ne(
6411        &mut self,
6412        loc_a: Location,
6413        loc_b: Location,
6414        ret: Location,
6415    ) -> Result<(), CompileError> {
6416        self.emit_relaxed_fcmp(Condition::Ne, Size::S32, loc_a, loc_b, ret)
6417    }
6418
6419    fn f32_cmp_eq(
6420        &mut self,
6421        loc_a: Location,
6422        loc_b: Location,
6423        ret: Location,
6424    ) -> Result<(), CompileError> {
6425        self.emit_relaxed_fcmp(Condition::Eq, Size::S32, loc_a, loc_b, ret)
6426    }
6427
6428    fn f32_min(
6429        &mut self,
6430        loc_a: Location,
6431        loc_b: Location,
6432        ret: Location,
6433    ) -> Result<(), CompileError> {
6434        self.emit_relaxed_binop3_fp(
6435            Assembler::emit_fmin,
6436            Size::S32,
6437            loc_a,
6438            loc_b,
6439            ret,
6440            ImmType::None,
6441            true,
6442        )
6443    }
6444
6445    fn f32_max(
6446        &mut self,
6447        loc_a: Location,
6448        loc_b: Location,
6449        ret: Location,
6450    ) -> Result<(), CompileError> {
6451        self.emit_relaxed_binop3_fp(
6452            Assembler::emit_fmax,
6453            Size::S32,
6454            loc_a,
6455            loc_b,
6456            ret,
6457            ImmType::None,
6458            true,
6459        )
6460    }
6461
6462    fn f32_add(
6463        &mut self,
6464        loc_a: Location,
6465        loc_b: Location,
6466        ret: Location,
6467    ) -> Result<(), CompileError> {
6468        self.emit_relaxed_binop3_fp(
6469            Assembler::emit_add,
6470            Size::S32,
6471            loc_a,
6472            loc_b,
6473            ret,
6474            ImmType::None,
6475            false,
6476        )
6477    }
6478
6479    fn f32_sub(
6480        &mut self,
6481        loc_a: Location,
6482        loc_b: Location,
6483        ret: Location,
6484    ) -> Result<(), CompileError> {
6485        self.emit_relaxed_binop3_fp(
6486            Assembler::emit_sub,
6487            Size::S32,
6488            loc_a,
6489            loc_b,
6490            ret,
6491            ImmType::None,
6492            false,
6493        )
6494    }
6495
6496    fn f32_mul(
6497        &mut self,
6498        loc_a: Location,
6499        loc_b: Location,
6500        ret: Location,
6501    ) -> Result<(), CompileError> {
6502        self.emit_relaxed_binop3_fp(
6503            Assembler::emit_mul,
6504            Size::S32,
6505            loc_a,
6506            loc_b,
6507            ret,
6508            ImmType::None,
6509            false,
6510        )
6511    }
6512
6513    fn f32_div(
6514        &mut self,
6515        loc_a: Location,
6516        loc_b: Location,
6517        ret: Location,
6518    ) -> Result<(), CompileError> {
6519        self.emit_relaxed_binop3_fp(
6520            Assembler::emit_fdiv,
6521            Size::S32,
6522            loc_a,
6523            loc_b,
6524            ret,
6525            ImmType::None,
6526            false,
6527        )
6528    }
6529
6530    fn gen_std_trampoline(
6531        &self,
6532        sig: &FunctionType,
6533        calling_convention: CallingConvention,
6534        progress_callback: Option<&CompilationProgressCallback>,
6535    ) -> Result<FunctionBody, CompileError> {
6536        gen_std_trampoline_riscv(sig, calling_convention, progress_callback)
6537    }
6538
6539    fn gen_std_dynamic_import_trampoline(
6540        &self,
6541        vmoffsets: &VMOffsets,
6542        sig: &FunctionType,
6543        _calling_convention: CallingConvention,
6544        progress_callback: Option<&CompilationProgressCallback>,
6545    ) -> Result<FunctionBody, CompileError> {
6546        gen_std_dynamic_import_trampoline_riscv(vmoffsets, sig, progress_callback)
6547    }
6548    // Singlepass calls import functions through a trampoline.
6549
6550    fn gen_import_call_trampoline(
6551        &self,
6552        vmoffsets: &VMOffsets,
6553        index: FunctionIndex,
6554        sig: &FunctionType,
6555        calling_convention: CallingConvention,
6556        progress_callback: Option<&CompilationProgressCallback>,
6557    ) -> Result<CustomSection, CompileError> {
6558        gen_import_call_trampoline_riscv(
6559            vmoffsets,
6560            index,
6561            sig,
6562            calling_convention,
6563            progress_callback,
6564        )
6565    }
6566
6567    #[cfg(feature = "unwind")]
6568    fn gen_dwarf_unwind_info(&mut self, code_len: usize) -> Option<UnwindInstructions> {
6569        let mut instructions = vec![];
6570        for &(instruction_offset, ref inst) in &self.unwind_ops {
6571            let instruction_offset = instruction_offset as u32;
6572            match *inst {
6573                UnwindOps::PushFP { up_to_sp } => {
6574                    instructions.push((
6575                        instruction_offset,
6576                        CallFrameInstruction::CfaOffset(up_to_sp as i32),
6577                    ));
6578                    instructions.push((
6579                        instruction_offset,
6580                        CallFrameInstruction::Offset(RiscV::FP, -(up_to_sp as i32)),
6581                    ));
6582                }
6583                UnwindOps::DefineNewFrame => {
6584                    instructions.push((
6585                        instruction_offset,
6586                        CallFrameInstruction::CfaRegister(RiscV::X8),
6587                    ));
6588                }
6589                UnwindOps::SaveRegister { reg, bp_neg_offset } => instructions.push((
6590                    instruction_offset,
6591                    CallFrameInstruction::Offset(reg.dwarf_index(), -bp_neg_offset),
6592                )),
6593                UnwindOps::SubtractFP { up_to_sp } => {
6594                    instructions.push((
6595                        instruction_offset,
6596                        CallFrameInstruction::CfaOffset(up_to_sp as i32),
6597                    ));
6598                }
6599                UnwindOps::Push2Regs { .. } => unimplemented!(),
6600            }
6601        }
6602        Some(UnwindInstructions {
6603            instructions,
6604            len: code_len as u32,
6605        })
6606    }
6607    #[cfg(not(feature = "unwind"))]
6608
6609    fn gen_dwarf_unwind_info(&mut self, _code_len: usize) -> Option<UnwindInstructions> {
6610        None
6611    }
6612
6613    fn gen_windows_unwind_info(&mut self, _code_len: usize) -> Option<Vec<u8>> {
6614        None
6615    }
6616}
wasmer_compiler_singlepass/machine_riscv.rs

wasmer_compiler_singlepass/
machine_riscv.rs
