wasmer_compiler_singlepass/
machine_arm64.rs

1use dynasmrt::{VecAssembler, aarch64::Aarch64Relocation};
2#[cfg(feature = "unwind")]
3use gimli::{AArch64, write::CallFrameInstruction};
4
5use wasmer_compiler::{
6    types::{
7        address_map::InstructionAddressMap,
8        function::FunctionBody,
9        relocation::{Relocation, RelocationKind, RelocationTarget},
10        section::CustomSection,
11    },
12    wasmparser::{MemArg, ValType as WpType},
13};
14use wasmer_types::{
15    CompileError, FunctionIndex, FunctionType, SourceLoc, TrapCode, TrapInformation, VMOffsets,
16    target::{CallingConvention, CpuFeature, Target},
17};
18
19use crate::{
20    arm64_decl::{GPR, NEON, new_machine_state},
21    codegen_error,
22    common_decl::*,
23    emitter_arm64::*,
24    location::{Location as AbstractLocation, Reg},
25    machine::*,
26    unwind::{UnwindInstructions, UnwindOps, UnwindRegister},
27};
28
29type Assembler = VecAssembler<Aarch64Relocation>;
30type Location = AbstractLocation<GPR, NEON>;
31
32pub struct MachineARM64 {
33    assembler: Assembler,
34    used_gprs: u32,
35    used_simd: u32,
36    trap_table: TrapTable,
37    /// Map from byte offset into wasm function to range of native instructions.
38    // Ordered by increasing InstructionAddressMap::srcloc.
39    instructions_address_map: Vec<InstructionAddressMap>,
40    /// The source location for the current operator.
41    src_loc: u32,
42    /// is last push on a 8byte multiple or 16bytes?
43    pushed: bool,
44    /// Vector of unwind operations with offset
45    unwind_ops: Vec<(usize, UnwindOps<GPR, NEON>)>,
46    /// A boolean flag signaling if this machine supports NEON.
47    has_neon: bool,
48}
49
50#[allow(dead_code)]
51#[derive(PartialEq)]
52enum ImmType {
53    None,
54    NoneXzr,
55    Bits8,
56    Bits12,
57    Shift32,
58    Shift32No0,
59    Shift64,
60    Shift64No0,
61    Logical32,
62    Logical64,
63    UnscaledOffset,
64    OffsetByte,
65    OffsetHWord,
66    OffsetWord,
67    OffsetDWord,
68}
69
70#[allow(dead_code)]
71impl MachineARM64 {
72    pub fn new(target: Option<Target>) -> Self {
73        // If and when needed, checks for other supported features should be
74        // added as boolean fields in the struct to make checking if such
75        // features are available as cheap as possible.
76        let has_neon = match target {
77            Some(ref target) => target.cpu_features().contains(CpuFeature::NEON),
78            None => false,
79        };
80
81        MachineARM64 {
82            assembler: Assembler::new(0),
83            used_gprs: 0,
84            used_simd: 0,
85            trap_table: TrapTable::default(),
86            instructions_address_map: vec![],
87            src_loc: 0,
88            pushed: false,
89            unwind_ops: vec![],
90            has_neon,
91        }
92    }
93    fn compatible_imm(&self, imm: i64, ty: ImmType) -> bool {
94        match ty {
95            ImmType::None => false,
96            ImmType::NoneXzr => false,
97            ImmType::Bits8 => (0..256).contains(&imm),
98            ImmType::Bits12 => (0..0x1000).contains(&imm),
99            ImmType::Shift32 => (0..32).contains(&imm),
100            ImmType::Shift32No0 => (1..32).contains(&imm),
101            ImmType::Shift64 => (0..64).contains(&imm),
102            ImmType::Shift64No0 => (1..64).contains(&imm),
103            ImmType::Logical32 => encode_logical_immediate_32bit(imm as u32).is_some(),
104            ImmType::Logical64 => encode_logical_immediate_64bit(imm as u64).is_some(),
105            ImmType::UnscaledOffset => (imm > -256) && (imm < 256),
106            ImmType::OffsetByte => (0..0x1000).contains(&imm),
107            ImmType::OffsetHWord => (imm & 1 == 0) && (0..0x2000).contains(&imm),
108            ImmType::OffsetWord => (imm & 3 == 0) && (0..0x4000).contains(&imm),
109            ImmType::OffsetDWord => (imm & 7 == 0) && (0..0x8000).contains(&imm),
110        }
111    }
112
113    fn location_to_reg(
114        &mut self,
115        sz: Size,
116        src: Location,
117        temps: &mut Vec<GPR>,
118        allow_imm: ImmType,
119        read_val: bool,
120        wanted: Option<GPR>,
121    ) -> Result<Location, CompileError> {
122        match src {
123            Location::GPR(_) | Location::SIMD(_) => Ok(src),
124            Location::Imm8(val) => {
125                if allow_imm == ImmType::NoneXzr && val == 0 {
126                    Ok(Location::GPR(GPR::XzrSp))
127                } else if self.compatible_imm(val as i64, allow_imm) {
128                    Ok(src)
129                } else {
130                    let tmp = if let Some(wanted) = wanted {
131                        wanted
132                    } else {
133                        let tmp = self.acquire_temp_gpr().ok_or_else(|| {
134                            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
135                        })?;
136                        temps.push(tmp);
137                        tmp
138                    };
139                    self.assembler
140                        .emit_mov_imm(Location::GPR(tmp), val as u64)?;
141                    Ok(Location::GPR(tmp))
142                }
143            }
144            Location::Imm32(val) => {
145                if allow_imm == ImmType::NoneXzr && val == 0 {
146                    Ok(Location::GPR(GPR::XzrSp))
147                } else if self.compatible_imm(val as i64, allow_imm) {
148                    Ok(src)
149                } else {
150                    let tmp = if let Some(wanted) = wanted {
151                        wanted
152                    } else {
153                        let tmp = self.acquire_temp_gpr().ok_or_else(|| {
154                            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
155                        })?;
156                        temps.push(tmp);
157                        tmp
158                    };
159                    self.assembler
160                        .emit_mov_imm(Location::GPR(tmp), (val as i64) as u64)?;
161                    Ok(Location::GPR(tmp))
162                }
163            }
164            Location::Imm64(val) => {
165                if allow_imm == ImmType::NoneXzr && val == 0 {
166                    Ok(Location::GPR(GPR::XzrSp))
167                } else if self.compatible_imm(val as i64, allow_imm) {
168                    Ok(src)
169                } else {
170                    let tmp = if let Some(wanted) = wanted {
171                        wanted
172                    } else {
173                        let tmp = self.acquire_temp_gpr().ok_or_else(|| {
174                            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
175                        })?;
176                        temps.push(tmp);
177                        tmp
178                    };
179                    self.assembler
180                        .emit_mov_imm(Location::GPR(tmp), val as u64)?;
181                    Ok(Location::GPR(tmp))
182                }
183            }
184            Location::Memory(reg, val) => {
185                let tmp = if let Some(wanted) = wanted {
186                    wanted
187                } else {
188                    let tmp = self.acquire_temp_gpr().ok_or_else(|| {
189                        CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
190                    })?;
191                    temps.push(tmp);
192                    tmp
193                };
194                if read_val {
195                    let offsize = match sz {
196                        Size::S8 => ImmType::OffsetByte,
197                        Size::S16 => ImmType::OffsetHWord,
198                        Size::S32 => ImmType::OffsetWord,
199                        Size::S64 => ImmType::OffsetDWord,
200                    };
201                    if sz == Size::S8 {
202                        if self.compatible_imm(val as i64, offsize) {
203                            self.assembler.emit_ldrb(
204                                sz,
205                                Location::GPR(tmp),
206                                Location::Memory(reg, val as _),
207                            )?;
208                        } else {
209                            if reg == tmp {
210                                codegen_error!("singlepass reg==tmp unreachable");
211                            }
212                            self.assembler
213                                .emit_mov_imm(Location::GPR(tmp), (val as i64) as u64)?;
214                            self.assembler.emit_ldrb(
215                                sz,
216                                Location::GPR(tmp),
217                                Location::Memory2(reg, tmp, Multiplier::One, 0),
218                            )?;
219                        }
220                    } else if sz == Size::S16 {
221                        if self.compatible_imm(val as i64, offsize) {
222                            self.assembler.emit_ldrh(
223                                sz,
224                                Location::GPR(tmp),
225                                Location::Memory(reg, val as _),
226                            )?;
227                        } else {
228                            if reg == tmp {
229                                codegen_error!("singlepass reg==tmp unreachable");
230                            }
231                            self.assembler
232                                .emit_mov_imm(Location::GPR(tmp), (val as i64) as u64)?;
233                            self.assembler.emit_ldrh(
234                                sz,
235                                Location::GPR(tmp),
236                                Location::Memory2(reg, tmp, Multiplier::One, 0),
237                            )?;
238                        }
239                    } else if self.compatible_imm(val as i64, offsize) {
240                        self.assembler.emit_ldr(
241                            sz,
242                            Location::GPR(tmp),
243                            Location::Memory(reg, val as _),
244                        )?;
245                    } else if self.compatible_imm(val as i64, ImmType::UnscaledOffset) {
246                        self.assembler.emit_ldur(sz, Location::GPR(tmp), reg, val)?;
247                    } else {
248                        if reg == tmp {
249                            codegen_error!("singlepass reg == tmp unreachable");
250                        }
251                        self.assembler
252                            .emit_mov_imm(Location::GPR(tmp), (val as i64) as u64)?;
253                        self.assembler.emit_ldr(
254                            sz,
255                            Location::GPR(tmp),
256                            Location::Memory2(reg, tmp, Multiplier::One, 0),
257                        )?;
258                    }
259                }
260                Ok(Location::GPR(tmp))
261            }
262            _ => codegen_error!("singlepass can't emit location_to_reg {:?} {:?}", sz, src),
263        }
264    }
265    fn location_to_neon(
266        &mut self,
267        sz: Size,
268        src: Location,
269        temps: &mut Vec<NEON>,
270        allow_imm: ImmType,
271        read_val: bool,
272    ) -> Result<Location, CompileError> {
273        match src {
274            Location::SIMD(_) => Ok(src),
275            Location::GPR(_) => {
276                let tmp = self.acquire_temp_simd().ok_or_else(|| {
277                    CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
278                })?;
279                temps.push(tmp);
280                if read_val {
281                    self.assembler.emit_mov(sz, src, Location::SIMD(tmp))?;
282                }
283                Ok(Location::SIMD(tmp))
284            }
285            Location::Imm8(val) => {
286                if self.compatible_imm(val as i64, allow_imm) {
287                    Ok(src)
288                } else {
289                    let gpr = self.acquire_temp_gpr().ok_or_else(|| {
290                        CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
291                    })?;
292                    let tmp = self.acquire_temp_simd().ok_or_else(|| {
293                        CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
294                    })?;
295                    temps.push(tmp);
296                    self.assembler
297                        .emit_mov_imm(Location::GPR(gpr), val as u64)?;
298                    self.assembler
299                        .emit_mov(sz, Location::GPR(gpr), Location::SIMD(tmp))?;
300                    self.release_gpr(gpr);
301                    Ok(Location::SIMD(tmp))
302                }
303            }
304            Location::Imm32(val) => {
305                if self.compatible_imm(val as i64, allow_imm) {
306                    Ok(src)
307                } else {
308                    let gpr = self.acquire_temp_gpr().ok_or_else(|| {
309                        CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
310                    })?;
311                    let tmp = self.acquire_temp_simd().ok_or_else(|| {
312                        CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
313                    })?;
314                    temps.push(tmp);
315                    self.assembler
316                        .emit_mov_imm(Location::GPR(gpr), (val as i64) as u64)?;
317                    self.assembler
318                        .emit_mov(sz, Location::GPR(gpr), Location::SIMD(tmp))?;
319                    self.release_gpr(gpr);
320                    Ok(Location::SIMD(tmp))
321                }
322            }
323            Location::Imm64(val) => {
324                if self.compatible_imm(val as i64, allow_imm) {
325                    Ok(src)
326                } else {
327                    let gpr = self.acquire_temp_gpr().ok_or_else(|| {
328                        CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
329                    })?;
330                    let tmp = self.acquire_temp_simd().ok_or_else(|| {
331                        CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
332                    })?;
333                    temps.push(tmp);
334                    self.assembler
335                        .emit_mov_imm(Location::GPR(gpr), val as u64)?;
336                    self.assembler
337                        .emit_mov(sz, Location::GPR(gpr), Location::SIMD(tmp))?;
338                    self.release_gpr(gpr);
339                    Ok(Location::SIMD(tmp))
340                }
341            }
342            Location::Memory(reg, val) => {
343                let tmp = self.acquire_temp_simd().ok_or_else(|| {
344                    CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
345                })?;
346                temps.push(tmp);
347                if read_val {
348                    let offsize = if sz == Size::S32 {
349                        ImmType::OffsetWord
350                    } else {
351                        ImmType::OffsetDWord
352                    };
353                    if self.compatible_imm(val as i64, offsize) {
354                        self.assembler.emit_ldr(
355                            sz,
356                            Location::SIMD(tmp),
357                            Location::Memory(reg, val as _),
358                        )?;
359                    } else if self.compatible_imm(val as i64, ImmType::UnscaledOffset) {
360                        self.assembler
361                            .emit_ldur(sz, Location::SIMD(tmp), reg, val)?;
362                    } else {
363                        let gpr = self.acquire_temp_gpr().ok_or_else(|| {
364                            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
365                        })?;
366                        self.assembler
367                            .emit_mov_imm(Location::GPR(gpr), (val as i64) as u64)?;
368                        self.assembler.emit_ldr(
369                            sz,
370                            Location::SIMD(tmp),
371                            Location::Memory2(reg, gpr, Multiplier::One, 0),
372                        )?;
373                        self.release_gpr(gpr);
374                    }
375                }
376                Ok(Location::SIMD(tmp))
377            }
378            _ => codegen_error!("singlepass can't emit location_to_neon {:?} {:?}", sz, src),
379        }
380    }
381
382    fn emit_relaxed_binop(
383        &mut self,
384        op: fn(&mut Assembler, Size, Location, Location) -> Result<(), CompileError>,
385        sz: Size,
386        src: Location,
387        dst: Location,
388        putback: bool,
389    ) -> Result<(), CompileError> {
390        let mut temps = vec![];
391        let src_imm = if putback {
392            ImmType::None
393        } else {
394            ImmType::Bits12
395        };
396        let src = self.location_to_reg(sz, src, &mut temps, src_imm, true, None)?;
397        let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, !putback, None)?;
398        op(&mut self.assembler, sz, src, dest)?;
399        if dst != dest && putback {
400            self.move_location(sz, dest, dst)?;
401        }
402        for r in temps {
403            self.release_gpr(r);
404        }
405        Ok(())
406    }
407    fn emit_relaxed_binop_neon(
408        &mut self,
409        op: fn(&mut Assembler, Size, Location, Location) -> Result<(), CompileError>,
410        sz: Size,
411        src: Location,
412        dst: Location,
413        putback: bool,
414    ) -> Result<(), CompileError> {
415        let mut temps = vec![];
416        let src = self.location_to_neon(sz, src, &mut temps, ImmType::None, true)?;
417        let dest = self.location_to_neon(sz, dst, &mut temps, ImmType::None, !putback)?;
418        op(&mut self.assembler, sz, src, dest)?;
419        if dst != dest && putback {
420            self.move_location(sz, dest, dst)?;
421        }
422        for r in temps {
423            self.release_simd(r);
424        }
425        Ok(())
426    }
427    fn emit_relaxed_binop3(
428        &mut self,
429        op: fn(&mut Assembler, Size, Location, Location, Location) -> Result<(), CompileError>,
430        sz: Size,
431        src1: Location,
432        src2: Location,
433        dst: Location,
434        allow_imm: ImmType,
435    ) -> Result<(), CompileError> {
436        let mut temps = vec![];
437        let src1 = self.location_to_reg(sz, src1, &mut temps, ImmType::None, true, None)?;
438        let src2 = self.location_to_reg(sz, src2, &mut temps, allow_imm, true, None)?;
439        let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
440        op(&mut self.assembler, sz, src1, src2, dest)?;
441        if dst != dest {
442            self.move_location(sz, dest, dst)?;
443        }
444        for r in temps {
445            self.release_gpr(r);
446        }
447        Ok(())
448    }
449    fn emit_relaxed_binop3_neon(
450        &mut self,
451        op: fn(&mut Assembler, Size, Location, Location, Location) -> Result<(), CompileError>,
452        sz: Size,
453        src1: Location,
454        src2: Location,
455        dst: Location,
456        allow_imm: ImmType,
457    ) -> Result<(), CompileError> {
458        let mut temps = vec![];
459        let src1 = self.location_to_neon(sz, src1, &mut temps, ImmType::None, true)?;
460        let src2 = self.location_to_neon(sz, src2, &mut temps, allow_imm, true)?;
461        let dest = self.location_to_neon(sz, dst, &mut temps, ImmType::None, false)?;
462        op(&mut self.assembler, sz, src1, src2, dest)?;
463        if dst != dest {
464            self.move_location(sz, dest, dst)?;
465        }
466        for r in temps {
467            self.release_simd(r);
468        }
469        Ok(())
470    }
471    fn emit_relaxed_ldr64(
472        &mut self,
473        sz: Size,
474        dst: Location,
475        src: Location,
476    ) -> Result<(), CompileError> {
477        let mut temps = vec![];
478        let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
479        match src {
480            Location::Memory(addr, offset) => {
481                if self.compatible_imm(offset as i64, ImmType::OffsetDWord) {
482                    self.assembler.emit_ldr(Size::S64, dest, src)?;
483                } else if self.compatible_imm(offset as i64, ImmType::UnscaledOffset) {
484                    self.assembler.emit_ldur(Size::S64, dest, addr, offset)?;
485                } else {
486                    let tmp = self.acquire_temp_gpr().ok_or_else(|| {
487                        CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
488                    })?;
489                    self.assembler
490                        .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
491                    self.assembler.emit_ldr(
492                        Size::S64,
493                        dest,
494                        Location::Memory2(addr, tmp, Multiplier::One, 0),
495                    )?;
496                    temps.push(tmp);
497                }
498            }
499            _ => codegen_error!("singplass emit_relaxed_ldr64 unreachable"),
500        }
501        if dst != dest {
502            self.move_location(sz, dest, dst)?;
503        }
504        for r in temps {
505            self.release_gpr(r);
506        }
507        Ok(())
508    }
509    fn emit_relaxed_ldr32(
510        &mut self,
511        sz: Size,
512        dst: Location,
513        src: Location,
514    ) -> Result<(), CompileError> {
515        let mut temps = vec![];
516        let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
517        match src {
518            Location::Memory(addr, offset) => {
519                if self.compatible_imm(offset as i64, ImmType::OffsetWord) {
520                    self.assembler.emit_ldr(Size::S32, dest, src)?;
521                } else if self.compatible_imm(offset as i64, ImmType::UnscaledOffset) {
522                    self.assembler.emit_ldur(Size::S32, dest, addr, offset)?;
523                } else {
524                    let tmp = self.acquire_temp_gpr().ok_or_else(|| {
525                        CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
526                    })?;
527                    self.assembler
528                        .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
529                    self.assembler.emit_ldr(
530                        Size::S32,
531                        dest,
532                        Location::Memory2(addr, tmp, Multiplier::One, 0),
533                    )?;
534                    temps.push(tmp);
535                }
536            }
537            _ => codegen_error!("singlepass emit_relaxed_ldr32 unreachable"),
538        }
539        if dst != dest {
540            self.move_location(sz, dest, dst)?;
541        }
542        for r in temps {
543            self.release_gpr(r);
544        }
545        Ok(())
546    }
547    fn emit_relaxed_ldr32s(
548        &mut self,
549        sz: Size,
550        dst: Location,
551        src: Location,
552    ) -> Result<(), CompileError> {
553        let mut temps = vec![];
554        let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
555        match src {
556            Location::Memory(addr, offset) => {
557                if self.compatible_imm(offset as i64, ImmType::OffsetWord) {
558                    self.assembler.emit_ldrsw(Size::S64, dest, src)?;
559                } else {
560                    let tmp = self.acquire_temp_gpr().ok_or_else(|| {
561                        CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
562                    })?;
563                    self.assembler
564                        .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
565                    self.assembler.emit_ldrsw(
566                        Size::S64,
567                        dest,
568                        Location::Memory2(addr, tmp, Multiplier::One, 0),
569                    )?;
570                    temps.push(tmp);
571                }
572            }
573            _ => codegen_error!("singplepass emit_relaxed_ldr32s unreachable"),
574        }
575        if dst != dest {
576            self.move_location(sz, dest, dst)?;
577        }
578        for r in temps {
579            self.release_gpr(r);
580        }
581        Ok(())
582    }
583    fn emit_relaxed_ldr16(
584        &mut self,
585        sz: Size,
586        dst: Location,
587        src: Location,
588    ) -> Result<(), CompileError> {
589        let mut temps = vec![];
590        let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
591        match src {
592            Location::Memory(addr, offset) => {
593                if self.compatible_imm(offset as i64, ImmType::OffsetHWord) {
594                    self.assembler.emit_ldrh(Size::S32, dest, src)?;
595                } else {
596                    let tmp = self.acquire_temp_gpr().ok_or_else(|| {
597                        CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
598                    })?;
599                    self.assembler
600                        .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
601                    self.assembler.emit_ldrh(
602                        Size::S32,
603                        dest,
604                        Location::Memory2(addr, tmp, Multiplier::One, 0),
605                    )?;
606                    temps.push(tmp);
607                }
608            }
609            _ => codegen_error!("singlpass emit_relaxed_ldr16 unreachable"),
610        }
611        if dst != dest {
612            self.move_location(sz, dest, dst)?;
613        }
614        for r in temps {
615            self.release_gpr(r);
616        }
617        Ok(())
618    }
619    fn emit_relaxed_ldr16s(
620        &mut self,
621        sz: Size,
622        dst: Location,
623        src: Location,
624    ) -> Result<(), CompileError> {
625        let mut temps = vec![];
626        let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
627        match src {
628            Location::Memory(addr, offset) => {
629                if self.compatible_imm(offset as i64, ImmType::OffsetHWord) {
630                    self.assembler.emit_ldrsh(sz, dest, src)?;
631                } else {
632                    let tmp = self.acquire_temp_gpr().ok_or_else(|| {
633                        CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
634                    })?;
635                    self.assembler
636                        .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
637                    self.assembler.emit_ldrsh(
638                        sz,
639                        dest,
640                        Location::Memory2(addr, tmp, Multiplier::One, 0),
641                    )?;
642                    temps.push(tmp);
643                }
644            }
645            _ => codegen_error!("singlepass emit_relaxed_ldr16s unreachable"),
646        }
647        if dst != dest {
648            self.move_location(sz, dest, dst)?;
649        }
650        for r in temps {
651            self.release_gpr(r);
652        }
653        Ok(())
654    }
655    fn emit_relaxed_ldr8(
656        &mut self,
657        sz: Size,
658        dst: Location,
659        src: Location,
660    ) -> Result<(), CompileError> {
661        let mut temps = vec![];
662        let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
663        match src {
664            Location::Memory(addr, offset) => {
665                if self.compatible_imm(offset as i64, ImmType::OffsetByte) {
666                    self.assembler.emit_ldrb(Size::S32, dest, src)?;
667                } else {
668                    let tmp = self.acquire_temp_gpr().ok_or_else(|| {
669                        CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
670                    })?;
671                    self.assembler
672                        .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
673                    self.assembler.emit_ldrb(
674                        Size::S32,
675                        dest,
676                        Location::Memory2(addr, tmp, Multiplier::One, 0),
677                    )?;
678                    temps.push(tmp);
679                }
680            }
681            _ => codegen_error!("singplepass emit_relaxed_ldr8 unreachable"),
682        }
683        if dst != dest {
684            self.move_location(sz, dest, dst)?;
685        }
686        for r in temps {
687            self.release_gpr(r);
688        }
689        Ok(())
690    }
691    fn emit_relaxed_ldr8s(
692        &mut self,
693        sz: Size,
694        dst: Location,
695        src: Location,
696    ) -> Result<(), CompileError> {
697        let mut temps = vec![];
698        let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
699        match src {
700            Location::Memory(addr, offset) => {
701                if self.compatible_imm(offset as i64, ImmType::OffsetByte) {
702                    self.assembler.emit_ldrsb(sz, dest, src)?;
703                } else {
704                    let tmp = self.acquire_temp_gpr().ok_or_else(|| {
705                        CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
706                    })?;
707                    self.assembler
708                        .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
709                    self.assembler.emit_ldrsb(
710                        sz,
711                        dest,
712                        Location::Memory2(addr, tmp, Multiplier::One, 0),
713                    )?;
714                    temps.push(tmp);
715                }
716            }
717            _ => codegen_error!("singlepass emit_relaxed_ldr8s unreachable"),
718        }
719        if dst != dest {
720            self.move_location(sz, dest, dst)?;
721        }
722        for r in temps {
723            self.release_gpr(r);
724        }
725        Ok(())
726    }
727    fn emit_relaxed_str64(&mut self, dst: Location, src: Location) -> Result<(), CompileError> {
728        let mut temps = vec![];
729        let dst = self.location_to_reg(Size::S64, dst, &mut temps, ImmType::NoneXzr, true, None)?;
730        match src {
731            Location::Memory(addr, offset) => {
732                if self.compatible_imm(offset as i64, ImmType::OffsetDWord) {
733                    self.assembler.emit_str(Size::S64, dst, src)?;
734                } else if self.compatible_imm(offset as i64, ImmType::UnscaledOffset) {
735                    self.assembler.emit_stur(Size::S64, dst, addr, offset)?;
736                } else {
737                    let tmp = self.acquire_temp_gpr().ok_or_else(|| {
738                        CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
739                    })?;
740                    self.assembler
741                        .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
742                    self.assembler.emit_str(
743                        Size::S64,
744                        dst,
745                        Location::Memory2(addr, tmp, Multiplier::One, 0),
746                    )?;
747                    temps.push(tmp);
748                }
749            }
750            _ => codegen_error!("singlepass can't emit str64 {:?} {:?}", dst, src),
751        }
752        for r in temps {
753            self.release_gpr(r);
754        }
755        Ok(())
756    }
757    fn emit_relaxed_str32(&mut self, dst: Location, src: Location) -> Result<(), CompileError> {
758        let mut temps = vec![];
759        let dst = self.location_to_reg(Size::S64, dst, &mut temps, ImmType::NoneXzr, true, None)?;
760        match src {
761            Location::Memory(addr, offset) => {
762                if self.compatible_imm(offset as i64, ImmType::OffsetWord) {
763                    self.assembler.emit_str(Size::S32, dst, src)?;
764                } else if self.compatible_imm(offset as i64, ImmType::UnscaledOffset) {
765                    self.assembler.emit_stur(Size::S32, dst, addr, offset)?;
766                } else {
767                    let tmp = self.acquire_temp_gpr().ok_or_else(|| {
768                        CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
769                    })?;
770                    self.assembler
771                        .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
772                    self.assembler.emit_str(
773                        Size::S32,
774                        dst,
775                        Location::Memory2(addr, tmp, Multiplier::One, 0),
776                    )?;
777                    temps.push(tmp);
778                }
779            }
780            _ => codegen_error!("singplepass emit_relaxed_str32 unreachable"),
781        }
782        for r in temps {
783            self.release_gpr(r);
784        }
785        Ok(())
786    }
787    fn emit_relaxed_str16(&mut self, dst: Location, src: Location) -> Result<(), CompileError> {
788        let mut temps = vec![];
789        let dst = self.location_to_reg(Size::S64, dst, &mut temps, ImmType::NoneXzr, true, None)?;
790        match src {
791            Location::Memory(addr, offset) => {
792                if self.compatible_imm(offset as i64, ImmType::OffsetHWord) {
793                    self.assembler.emit_strh(Size::S32, dst, src)?;
794                } else {
795                    let tmp = self.acquire_temp_gpr().ok_or_else(|| {
796                        CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
797                    })?;
798                    self.assembler
799                        .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
800                    self.assembler.emit_strh(
801                        Size::S32,
802                        dst,
803                        Location::Memory2(addr, tmp, Multiplier::One, 0),
804                    )?;
805                    temps.push(tmp);
806                }
807            }
808            _ => codegen_error!("singlepass emit_relaxed_str16 unreachable"),
809        }
810        for r in temps {
811            self.release_gpr(r);
812        }
813        Ok(())
814    }
815    fn emit_relaxed_str8(&mut self, dst: Location, src: Location) -> Result<(), CompileError> {
816        let mut temps = vec![];
817        let dst = self.location_to_reg(Size::S64, dst, &mut temps, ImmType::NoneXzr, true, None)?;
818        match src {
819            Location::Memory(addr, offset) => {
820                if self.compatible_imm(offset as i64, ImmType::OffsetByte) {
821                    self.assembler
822                        .emit_strb(Size::S32, dst, Location::Memory(addr, offset))?;
823                } else {
824                    let tmp = self.acquire_temp_gpr().ok_or_else(|| {
825                        CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
826                    })?;
827                    self.assembler
828                        .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
829                    self.assembler.emit_strb(
830                        Size::S32,
831                        dst,
832                        Location::Memory2(addr, tmp, Multiplier::One, 0),
833                    )?;
834                    temps.push(tmp);
835                }
836            }
837            _ => codegen_error!("singlepass emit_relaxed_str8 unreachable"),
838        }
839        for r in temps {
840            self.release_gpr(r);
841        }
842        Ok(())
843    }
844    /// I64 comparison with.
845    fn emit_cmpop_i64_dynamic_b(
846        &mut self,
847        c: Condition,
848        loc_a: Location,
849        loc_b: Location,
850        ret: Location,
851    ) -> Result<(), CompileError> {
852        match ret {
853            Location::GPR(_) => {
854                self.emit_relaxed_cmp(Size::S64, loc_b, loc_a)?;
855                self.assembler.emit_cset(Size::S32, ret, c)?;
856            }
857            Location::Memory(_, _) => {
858                let tmp = self.acquire_temp_gpr().ok_or_else(|| {
859                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
860                })?;
861                self.emit_relaxed_cmp(Size::S64, loc_b, loc_a)?;
862                self.assembler.emit_cset(Size::S32, Location::GPR(tmp), c)?;
863                self.move_location(Size::S32, Location::GPR(tmp), ret)?;
864                self.release_gpr(tmp);
865            }
866            _ => {
867                codegen_error!("singlepass emit_compop_i64_dynamic_b unreachable");
868            }
869        }
870        Ok(())
871    }
872    /// I32 comparison with.
873    fn emit_cmpop_i32_dynamic_b(
874        &mut self,
875        c: Condition,
876        loc_a: Location,
877        loc_b: Location,
878        ret: Location,
879    ) -> Result<(), CompileError> {
880        match ret {
881            Location::GPR(_) => {
882                self.emit_relaxed_cmp(Size::S32, loc_b, loc_a)?;
883                self.assembler.emit_cset(Size::S32, ret, c)?;
884            }
885            Location::Memory(_, _) => {
886                let tmp = self.acquire_temp_gpr().ok_or_else(|| {
887                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
888                })?;
889                self.emit_relaxed_cmp(Size::S32, loc_b, loc_a)?;
890                self.assembler.emit_cset(Size::S32, Location::GPR(tmp), c)?;
891                self.move_location(Size::S32, Location::GPR(tmp), ret)?;
892                self.release_gpr(tmp);
893            }
894            _ => {
895                codegen_error!("singlepass emit_cmpop_i32_dynamic_b unreachable");
896            }
897        }
898        Ok(())
899    }
900
901    #[allow(clippy::too_many_arguments)]
902    fn memory_op<F: FnOnce(&mut Self, GPR) -> Result<(), CompileError>>(
903        &mut self,
904        addr: Location,
905        memarg: &MemArg,
906        check_alignment: bool,
907        value_size: usize,
908        need_check: bool,
909        imported_memories: bool,
910        offset: i32,
911        heap_access_oob: Label,
912        unaligned_atomic: Label,
913        cb: F,
914    ) -> Result<(), CompileError> {
915        let tmp_addr = self.acquire_temp_gpr().ok_or_else(|| {
916            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
917        })?;
918
919        // Reusing `tmp_addr` for temporary indirection here, since it's not used before the last reference to `{base,bound}_loc`.
920        let (base_loc, bound_loc) = if imported_memories {
921            // Imported memories require one level of indirection.
922            self.emit_relaxed_binop(
923                Assembler::emit_mov,
924                Size::S64,
925                Location::Memory(self.get_vmctx_reg(), offset),
926                Location::GPR(tmp_addr),
927                true,
928            )?;
929            (Location::Memory(tmp_addr, 0), Location::Memory(tmp_addr, 8))
930        } else {
931            (
932                Location::Memory(self.get_vmctx_reg(), offset),
933                Location::Memory(self.get_vmctx_reg(), offset + 8),
934            )
935        };
936
937        let tmp_base = self.acquire_temp_gpr().ok_or_else(|| {
938            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
939        })?;
940        let tmp_bound = self.acquire_temp_gpr().ok_or_else(|| {
941            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
942        })?;
943
944        // Load base into temporary register.
945        self.emit_relaxed_ldr64(Size::S64, Location::GPR(tmp_base), base_loc)?;
946
947        // Load bound into temporary register, if needed.
948        if need_check {
949            self.emit_relaxed_ldr64(Size::S64, Location::GPR(tmp_bound), bound_loc)?;
950
951            // Wasm -> Effective.
952            // Assuming we never underflow - should always be true on Linux/macOS and Windows >=8,
953            // since the first page from 0x0 to 0x1000 is not accepted by mmap.
954            self.assembler.emit_add(
955                Size::S64,
956                Location::GPR(tmp_bound),
957                Location::GPR(tmp_base),
958                Location::GPR(tmp_bound),
959            )?;
960            if self.compatible_imm(value_size as _, ImmType::Bits12) {
961                self.assembler.emit_sub(
962                    Size::S64,
963                    Location::GPR(tmp_bound),
964                    Location::Imm32(value_size as _),
965                    Location::GPR(tmp_bound),
966                )?;
967            } else {
968                let tmp2 = self.acquire_temp_gpr().ok_or_else(|| {
969                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
970                })?;
971                self.assembler
972                    .emit_mov_imm(Location::GPR(tmp2), value_size as u64)?;
973                self.assembler.emit_sub(
974                    Size::S64,
975                    Location::GPR(tmp_bound),
976                    Location::GPR(tmp2),
977                    Location::GPR(tmp_bound),
978                )?;
979                self.release_gpr(tmp2);
980            }
981        }
982
983        // Load effective address.
984        // `base_loc` and `bound_loc` becomes INVALID after this line, because `tmp_addr`
985        // might be reused.
986        self.move_location(Size::S32, addr, Location::GPR(tmp_addr))?;
987
988        // Add offset to memory address.
989        if memarg.offset != 0 {
990            if self.compatible_imm(memarg.offset as _, ImmType::Bits12) {
991                self.assembler.emit_adds(
992                    Size::S32,
993                    Location::Imm32(memarg.offset as u32),
994                    Location::GPR(tmp_addr),
995                    Location::GPR(tmp_addr),
996                )?;
997            } else {
998                let tmp = self.acquire_temp_gpr().ok_or_else(|| {
999                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1000                })?;
1001                self.assembler
1002                    .emit_mov_imm(Location::GPR(tmp), memarg.offset as _)?;
1003                self.assembler.emit_adds(
1004                    Size::S32,
1005                    Location::GPR(tmp_addr),
1006                    Location::GPR(tmp),
1007                    Location::GPR(tmp_addr),
1008                )?;
1009                self.release_gpr(tmp);
1010            }
1011
1012            // Trap if offset calculation overflowed.
1013            self.assembler
1014                .emit_bcond_label_far(Condition::Cs, heap_access_oob)?;
1015        }
1016
1017        // Wasm linear memory -> real memory
1018        self.assembler.emit_add(
1019            Size::S64,
1020            Location::GPR(tmp_base),
1021            Location::GPR(tmp_addr),
1022            Location::GPR(tmp_addr),
1023        )?;
1024
1025        if need_check {
1026            // Trap if the end address of the requested area is above that of the linear memory.
1027            self.assembler.emit_cmp(
1028                Size::S64,
1029                Location::GPR(tmp_bound),
1030                Location::GPR(tmp_addr),
1031            )?;
1032
1033            // `tmp_bound` is inclusive. So trap only if `tmp_addr > tmp_bound`.
1034            self.assembler
1035                .emit_bcond_label_far(Condition::Hi, heap_access_oob)?;
1036        }
1037
1038        self.release_gpr(tmp_bound);
1039        self.release_gpr(tmp_base);
1040
1041        let align = value_size as u32;
1042        if check_alignment && align != 1 {
1043            self.assembler.emit_tst(
1044                Size::S64,
1045                Location::Imm32(align - 1),
1046                Location::GPR(tmp_addr),
1047            )?;
1048            self.assembler
1049                .emit_bcond_label_far(Condition::Ne, unaligned_atomic)?;
1050        }
1051        let begin = self.assembler.get_offset().0;
1052        cb(self, tmp_addr)?;
1053        let end = self.assembler.get_offset().0;
1054        self.mark_address_range_with_trap_code(TrapCode::HeapAccessOutOfBounds, begin, end);
1055
1056        self.release_gpr(tmp_addr);
1057        Ok(())
1058    }
1059
1060    /*fn emit_compare_and_swap<F: FnOnce(&mut Self, GPR, GPR)>(
1061        &mut self,
1062        _loc: Location,
1063        _target: Location,
1064        _ret: Location,
1065        _memarg: &MemArg,
1066        _value_size: usize,
1067        _memory_sz: Size,
1068        _stack_sz: Size,
1069        _need_check: bool,
1070        _imported_memories: bool,
1071        _offset: i32,
1072        _heap_access_oob: Label,
1073        _unaligned_atomic: Label,
1074        _cb: F,
1075    ) {
1076        unimplemented!();
1077    }*/
1078
1079    fn offset_is_ok(&self, size: Size, offset: i32) -> bool {
1080        if offset < 0 {
1081            return false;
1082        }
1083        let shift = match size {
1084            Size::S8 => 0,
1085            Size::S16 => 1,
1086            Size::S32 => 2,
1087            Size::S64 => 3,
1088        };
1089        if offset >= 0x1000 << shift {
1090            return false;
1091        }
1092        if (offset & ((1 << shift) - 1)) != 0 {
1093            return false;
1094        }
1095        true
1096    }
1097
1098    fn emit_push(&mut self, sz: Size, src: Location) -> Result<(), CompileError> {
1099        match (sz, src) {
1100            (Size::S64, Location::GPR(_)) | (Size::S64, Location::SIMD(_)) => {
1101                let offset = if self.pushed {
1102                    0
1103                } else {
1104                    self.assembler.emit_sub(
1105                        Size::S64,
1106                        Location::GPR(GPR::XzrSp),
1107                        Location::Imm8(16),
1108                        Location::GPR(GPR::XzrSp),
1109                    )?;
1110                    8
1111                };
1112                self.assembler
1113                    .emit_stur(Size::S64, src, GPR::XzrSp, offset)?;
1114                self.pushed = !self.pushed;
1115            }
1116            (Size::S64, _) => {
1117                let mut temps = vec![];
1118                let src = self.location_to_reg(sz, src, &mut temps, ImmType::None, true, None)?;
1119                let offset = if self.pushed {
1120                    0
1121                } else {
1122                    self.assembler.emit_sub(
1123                        Size::S64,
1124                        Location::GPR(GPR::XzrSp),
1125                        Location::Imm8(16),
1126                        Location::GPR(GPR::XzrSp),
1127                    )?;
1128                    8
1129                };
1130                self.assembler
1131                    .emit_stur(Size::S64, src, GPR::XzrSp, offset)?;
1132                self.pushed = !self.pushed;
1133                for r in temps {
1134                    self.release_gpr(r);
1135                }
1136            }
1137            _ => codegen_error!("singlepass can't emit PUSH {:?} {:?}", sz, src),
1138        }
1139        Ok(())
1140    }
1141    fn emit_double_push(
1142        &mut self,
1143        sz: Size,
1144        src1: Location,
1145        src2: Location,
1146    ) -> Result<(), CompileError> {
1147        if !self.pushed {
1148            match (sz, src1, src2) {
1149                (Size::S64, Location::GPR(_), Location::GPR(_)) => {
1150                    self.assembler
1151                        .emit_stpdb(Size::S64, src1, src2, GPR::XzrSp, 16)?;
1152                }
1153                _ => {
1154                    self.emit_push(sz, src1)?;
1155                    self.emit_push(sz, src2)?;
1156                }
1157            }
1158        } else {
1159            self.emit_push(sz, src1)?;
1160            self.emit_push(sz, src2)?;
1161        }
1162        Ok(())
1163    }
1164    fn emit_pop(&mut self, sz: Size, dst: Location) -> Result<(), CompileError> {
1165        match (sz, dst) {
1166            (Size::S64, Location::GPR(_)) | (Size::S64, Location::SIMD(_)) => {
1167                let offset = if self.pushed { 8 } else { 0 };
1168                self.assembler
1169                    .emit_ldur(Size::S64, dst, GPR::XzrSp, offset)?;
1170                if self.pushed {
1171                    self.assembler.emit_add(
1172                        Size::S64,
1173                        Location::GPR(GPR::XzrSp),
1174                        Location::Imm8(16),
1175                        Location::GPR(GPR::XzrSp),
1176                    )?;
1177                }
1178                self.pushed = !self.pushed;
1179            }
1180            _ => codegen_error!("singlepass can't emit PUSH {:?} {:?}", sz, dst),
1181        }
1182        Ok(())
1183    }
1184    fn emit_double_pop(
1185        &mut self,
1186        sz: Size,
1187        dst1: Location,
1188        dst2: Location,
1189    ) -> Result<(), CompileError> {
1190        if !self.pushed {
1191            match (sz, dst1, dst2) {
1192                (Size::S64, Location::GPR(_), Location::GPR(_)) => {
1193                    self.assembler
1194                        .emit_ldpia(Size::S64, dst1, dst2, GPR::XzrSp, 16)?;
1195                }
1196                _ => {
1197                    self.emit_pop(sz, dst2)?;
1198                    self.emit_pop(sz, dst1)?;
1199                }
1200            }
1201        } else {
1202            self.emit_pop(sz, dst2)?;
1203            self.emit_pop(sz, dst1)?;
1204        }
1205        Ok(())
1206    }
1207
1208    fn set_default_nan(&mut self, temps: &mut Vec<GPR>) -> Result<GPR, CompileError> {
1209        // temporarly set FPCR to DefaultNan
1210        let old_fpcr = self.acquire_temp_gpr().ok_or_else(|| {
1211            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1212        })?;
1213        temps.push(old_fpcr);
1214        self.assembler.emit_read_fpcr(old_fpcr)?;
1215        let new_fpcr = self.acquire_temp_gpr().ok_or_else(|| {
1216            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1217        })?;
1218        temps.push(new_fpcr);
1219        let tmp = self.acquire_temp_gpr().ok_or_else(|| {
1220            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1221        })?;
1222        temps.push(tmp);
1223        self.assembler
1224            .emit_mov(Size::S32, Location::Imm32(1), Location::GPR(tmp))?;
1225        self.assembler
1226            .emit_mov(Size::S64, Location::GPR(old_fpcr), Location::GPR(new_fpcr))?;
1227        // DN is bit 25 of FPCR
1228        self.assembler.emit_bfi(
1229            Size::S64,
1230            Location::GPR(tmp),
1231            25,
1232            1,
1233            Location::GPR(new_fpcr),
1234        )?;
1235        self.assembler.emit_write_fpcr(new_fpcr)?;
1236        Ok(old_fpcr)
1237    }
1238    fn set_trap_enabled(&mut self, temps: &mut Vec<GPR>) -> Result<GPR, CompileError> {
1239        // temporarly set FPCR to DefaultNan
1240        let old_fpcr = self.acquire_temp_gpr().ok_or_else(|| {
1241            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1242        })?;
1243        temps.push(old_fpcr);
1244        self.assembler.emit_read_fpcr(old_fpcr)?;
1245        let new_fpcr = self.acquire_temp_gpr().ok_or_else(|| {
1246            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1247        })?;
1248        temps.push(new_fpcr);
1249        self.assembler
1250            .emit_mov(Size::S64, Location::GPR(old_fpcr), Location::GPR(new_fpcr))?;
1251        // IOE is bit 8 of FPCR
1252        self.assembler
1253            .emit_bfc(Size::S64, 8, 1, Location::GPR(new_fpcr))?;
1254        self.assembler.emit_write_fpcr(new_fpcr)?;
1255        Ok(old_fpcr)
1256    }
1257    fn restore_fpcr(&mut self, old_fpcr: GPR) -> Result<(), CompileError> {
1258        self.assembler.emit_write_fpcr(old_fpcr)
1259    }
1260
1261    fn reset_exception_fpsr(&mut self) -> Result<(), CompileError> {
1262        // reset exception count in FPSR
1263        let fpsr = self.acquire_temp_gpr().ok_or_else(|| {
1264            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1265        })?;
1266        self.assembler.emit_read_fpsr(fpsr)?;
1267        // IOC is 0
1268        self.assembler
1269            .emit_bfc(Size::S64, 0, 1, Location::GPR(fpsr))?;
1270        self.assembler.emit_write_fpsr(fpsr)?;
1271        self.release_gpr(fpsr);
1272        Ok(())
1273    }
1274    fn read_fpsr(&mut self) -> Result<GPR, CompileError> {
1275        let fpsr = self.acquire_temp_gpr().ok_or_else(|| {
1276            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1277        })?;
1278        self.assembler.emit_read_fpsr(fpsr)?;
1279        Ok(fpsr)
1280    }
1281
1282    fn trap_float_convertion_errors(
1283        &mut self,
1284        old_fpcr: GPR,
1285        sz: Size,
1286        f: Location,
1287        temps: &mut Vec<GPR>,
1288    ) -> Result<(), CompileError> {
1289        let trap_badconv = self.assembler.get_label();
1290        let end = self.assembler.get_label();
1291
1292        let fpsr = self.read_fpsr()?;
1293        temps.push(fpsr);
1294        // no trap, than all good
1295        self.assembler
1296            .emit_tbz_label(Size::S32, Location::GPR(fpsr), 0, end)?;
1297        // now need to check if it's overflow or NaN
1298        self.assembler
1299            .emit_bfc(Size::S64, 0, 4, Location::GPR(fpsr))?;
1300        self.restore_fpcr(old_fpcr)?;
1301        self.assembler.emit_fcmp(sz, f, f)?;
1302        self.assembler
1303            .emit_bcond_label(Condition::Vs, trap_badconv)?;
1304        // fallthru: trap_overflow
1305        self.emit_illegal_op_internal(TrapCode::IntegerOverflow)?;
1306
1307        self.emit_label(trap_badconv)?;
1308        self.emit_illegal_op_internal(TrapCode::BadConversionToInteger)?;
1309
1310        self.emit_label(end)?;
1311        self.restore_fpcr(old_fpcr)
1312    }
1313
1314    fn used_gprs_contains(&self, r: &GPR) -> bool {
1315        self.used_gprs & (1 << r.into_index()) != 0
1316    }
1317    fn used_simd_contains(&self, r: &NEON) -> bool {
1318        self.used_simd & (1 << r.into_index()) != 0
1319    }
1320    fn used_gprs_insert(&mut self, r: GPR) {
1321        self.used_gprs |= 1 << r.into_index();
1322    }
1323    fn used_simd_insert(&mut self, r: NEON) {
1324        self.used_simd |= 1 << r.into_index();
1325    }
1326    fn used_gprs_remove(&mut self, r: &GPR) -> bool {
1327        let ret = self.used_gprs_contains(r);
1328        self.used_gprs &= !(1 << r.into_index());
1329        ret
1330    }
1331    fn used_simd_remove(&mut self, r: &NEON) -> bool {
1332        let ret = self.used_simd_contains(r);
1333        self.used_simd &= !(1 << r.into_index());
1334        ret
1335    }
1336    fn emit_unwind_op(&mut self, op: UnwindOps<GPR, NEON>) {
1337        self.unwind_ops.push((self.get_offset().0, op));
1338    }
1339    fn emit_illegal_op_internal(&mut self, trap: TrapCode) -> Result<(), CompileError> {
1340        self.assembler.emit_udf(0xc0 | (trap as u8) as u16)
1341    }
1342}
1343
1344impl Machine for MachineARM64 {
1345    type GPR = GPR;
1346    type SIMD = NEON;
1347    fn assembler_get_offset(&self) -> Offset {
1348        self.assembler.get_offset()
1349    }
1350    fn index_from_gpr(&self, x: GPR) -> RegisterIndex {
1351        RegisterIndex(x as usize)
1352    }
1353    fn index_from_simd(&self, x: NEON) -> RegisterIndex {
1354        RegisterIndex(x as usize + 32)
1355    }
1356
1357    fn get_vmctx_reg(&self) -> GPR {
1358        GPR::X28
1359    }
1360
1361    fn get_used_gprs(&self) -> Vec<GPR> {
1362        GPR::iterator()
1363            .filter(|x| self.used_gprs & (1 << x.into_index()) != 0)
1364            .cloned()
1365            .collect()
1366    }
1367
1368    fn get_used_simd(&self) -> Vec<NEON> {
1369        NEON::iterator()
1370            .filter(|x| self.used_simd & (1 << x.into_index()) != 0)
1371            .cloned()
1372            .collect()
1373    }
1374
1375    fn pick_gpr(&self) -> Option<GPR> {
1376        use GPR::*;
1377        static REGS: &[GPR] = &[X9, X10, X11, X12, X13, X14, X15];
1378        for r in REGS {
1379            if !self.used_gprs_contains(r) {
1380                return Some(*r);
1381            }
1382        }
1383        None
1384    }
1385
1386    // Picks an unused general purpose register for internal temporary use.
1387    fn pick_temp_gpr(&self) -> Option<GPR> {
1388        use GPR::*;
1389        static REGS: &[GPR] = &[X8, X7, X6, X5, X4, X3, X2, X1];
1390        for r in REGS {
1391            if !self.used_gprs_contains(r) {
1392                return Some(*r);
1393            }
1394        }
1395        None
1396    }
1397
1398    fn acquire_temp_gpr(&mut self) -> Option<GPR> {
1399        let gpr = self.pick_temp_gpr();
1400        if let Some(x) = gpr {
1401            self.used_gprs_insert(x);
1402        }
1403        gpr
1404    }
1405
1406    fn release_gpr(&mut self, gpr: GPR) {
1407        assert!(self.used_gprs_remove(&gpr));
1408    }
1409
1410    fn reserve_unused_temp_gpr(&mut self, gpr: GPR) -> GPR {
1411        assert!(!self.used_gprs_contains(&gpr));
1412        self.used_gprs_insert(gpr);
1413        gpr
1414    }
1415
1416    fn reserve_gpr(&mut self, gpr: GPR) {
1417        self.used_gprs_insert(gpr);
1418    }
1419
1420    fn push_used_gpr(&mut self, used_gprs: &[GPR]) -> Result<usize, CompileError> {
1421        if used_gprs.len() % 2 == 1 {
1422            self.emit_push(Size::S64, Location::GPR(GPR::XzrSp))?;
1423        }
1424        for r in used_gprs.iter() {
1425            self.emit_push(Size::S64, Location::GPR(*r))?;
1426        }
1427        Ok(used_gprs.len().div_ceil(2) * 16)
1428    }
1429    fn pop_used_gpr(&mut self, used_gprs: &[GPR]) -> Result<(), CompileError> {
1430        for r in used_gprs.iter().rev() {
1431            self.emit_pop(Size::S64, Location::GPR(*r))?;
1432        }
1433        if used_gprs.len() % 2 == 1 {
1434            self.emit_pop(Size::S64, Location::GPR(GPR::XzrSp))?;
1435        }
1436        Ok(())
1437    }
1438
1439    // Picks an unused NEON register.
1440    fn pick_simd(&self) -> Option<NEON> {
1441        use NEON::*;
1442        static REGS: &[NEON] = &[V8, V9, V10, V11, V12];
1443        for r in REGS {
1444            if !self.used_simd_contains(r) {
1445                return Some(*r);
1446            }
1447        }
1448        None
1449    }
1450
1451    // Picks an unused NEON register for internal temporary use.
1452    fn pick_temp_simd(&self) -> Option<NEON> {
1453        use NEON::*;
1454        static REGS: &[NEON] = &[V0, V1, V2, V3, V4, V5, V6, V7];
1455        for r in REGS {
1456            if !self.used_simd_contains(r) {
1457                return Some(*r);
1458            }
1459        }
1460        None
1461    }
1462
1463    // Acquires a temporary NEON register.
1464    fn acquire_temp_simd(&mut self) -> Option<NEON> {
1465        let simd = self.pick_temp_simd();
1466        if let Some(x) = simd {
1467            self.used_simd_insert(x);
1468        }
1469        simd
1470    }
1471
1472    fn reserve_simd(&mut self, simd: NEON) {
1473        self.used_simd_insert(simd);
1474    }
1475
1476    // Releases a temporary NEON register.
1477    fn release_simd(&mut self, simd: NEON) {
1478        assert!(self.used_simd_remove(&simd));
1479    }
1480
1481    fn push_used_simd(&mut self, used_neons: &[NEON]) -> Result<usize, CompileError> {
1482        let stack_adjust = if used_neons.len() & 1 == 1 {
1483            (used_neons.len() * 8) as u32 + 8
1484        } else {
1485            (used_neons.len() * 8) as u32
1486        };
1487        self.adjust_stack(stack_adjust)?;
1488
1489        for (i, r) in used_neons.iter().enumerate() {
1490            self.assembler.emit_str(
1491                Size::S64,
1492                Location::SIMD(*r),
1493                Location::Memory(GPR::XzrSp, (i * 8) as i32),
1494            )?;
1495        }
1496        Ok(stack_adjust as usize)
1497    }
1498    fn pop_used_simd(&mut self, used_neons: &[NEON]) -> Result<(), CompileError> {
1499        for (i, r) in used_neons.iter().enumerate() {
1500            self.assembler.emit_ldr(
1501                Size::S64,
1502                Location::SIMD(*r),
1503                Location::Memory(GPR::XzrSp, (i * 8) as i32),
1504            )?;
1505        }
1506        let stack_adjust = if used_neons.len() & 1 == 1 {
1507            (used_neons.len() * 8) as u32 + 8
1508        } else {
1509            (used_neons.len() * 8) as u32
1510        };
1511        self.assembler.emit_add(
1512            Size::S64,
1513            Location::GPR(GPR::XzrSp),
1514            Location::Imm32(stack_adjust as _),
1515            Location::GPR(GPR::XzrSp),
1516        )
1517    }
1518
1519    /// Set the source location of the Wasm to the given offset.
1520    fn set_srcloc(&mut self, offset: u32) {
1521        self.src_loc = offset;
1522    }
1523    /// Marks each address in the code range emitted by `f` with the trap code `code`.
1524    fn mark_address_range_with_trap_code(&mut self, code: TrapCode, begin: usize, end: usize) {
1525        for i in begin..end {
1526            self.trap_table.offset_to_code.insert(i, code);
1527        }
1528        self.mark_instruction_address_end(begin);
1529    }
1530
1531    /// Marks one address as trappable with trap code `code`.
1532    fn mark_address_with_trap_code(&mut self, code: TrapCode) {
1533        let offset = self.assembler.get_offset().0;
1534        self.trap_table.offset_to_code.insert(offset, code);
1535        self.mark_instruction_address_end(offset);
1536    }
1537    /// Marks the instruction as trappable with trap code `code`. return "begin" offset
1538    fn mark_instruction_with_trap_code(&mut self, code: TrapCode) -> usize {
1539        let offset = self.assembler.get_offset().0;
1540        self.trap_table.offset_to_code.insert(offset, code);
1541        offset
1542    }
1543    /// Pushes the instruction to the address map, calculating the offset from a
1544    /// provided beginning address.
1545    fn mark_instruction_address_end(&mut self, begin: usize) {
1546        self.instructions_address_map.push(InstructionAddressMap {
1547            srcloc: SourceLoc::new(self.src_loc),
1548            code_offset: begin,
1549            code_len: self.assembler.get_offset().0 - begin,
1550        });
1551    }
1552
1553    /// Insert a StackOverflow (at offset 0)
1554    fn insert_stackoverflow(&mut self) {
1555        let offset = 0;
1556        self.trap_table
1557            .offset_to_code
1558            .insert(offset, TrapCode::StackOverflow);
1559        self.mark_instruction_address_end(offset);
1560    }
1561
1562    /// Get all current TrapInformation
1563    fn collect_trap_information(&self) -> Vec<TrapInformation> {
1564        self.trap_table
1565            .offset_to_code
1566            .clone()
1567            .into_iter()
1568            .map(|(offset, code)| TrapInformation {
1569                code_offset: offset as u32,
1570                trap_code: code,
1571            })
1572            .collect()
1573    }
1574
1575    fn instructions_address_map(&self) -> Vec<InstructionAddressMap> {
1576        self.instructions_address_map.clone()
1577    }
1578
1579    // Return a rounded stack adjustement value (must be multiple of 16bytes on ARM64 for example)
1580    fn round_stack_adjust(&self, value: usize) -> usize {
1581        if value & 0xf != 0 {
1582            ((value >> 4) + 1) << 4
1583        } else {
1584            value
1585        }
1586    }
1587
1588    // Memory location for a local on the stack
1589    fn local_on_stack(&mut self, stack_offset: i32) -> Location {
1590        Location::Memory(GPR::X29, -stack_offset)
1591    }
1592
1593    // Adjust stack for locals
1594    fn adjust_stack(&mut self, delta_stack_offset: u32) -> Result<(), CompileError> {
1595        let delta = if self.compatible_imm(delta_stack_offset as _, ImmType::Bits12) {
1596            Location::Imm32(delta_stack_offset as _)
1597        } else {
1598            let tmp = GPR::X17;
1599            self.assembler
1600                .emit_mov_imm(Location::GPR(tmp), delta_stack_offset as u64)?;
1601            Location::GPR(tmp)
1602        };
1603        self.assembler.emit_sub(
1604            Size::S64,
1605            Location::GPR(GPR::XzrSp),
1606            delta,
1607            Location::GPR(GPR::XzrSp),
1608        )
1609    }
1610    // restore stack
1611    fn restore_stack(&mut self, delta_stack_offset: u32) -> Result<(), CompileError> {
1612        let delta = if self.compatible_imm(delta_stack_offset as _, ImmType::Bits12) {
1613            Location::Imm32(delta_stack_offset as _)
1614        } else {
1615            let tmp = GPR::X17;
1616            self.assembler
1617                .emit_mov_imm(Location::GPR(tmp), delta_stack_offset as u64)?;
1618            Location::GPR(tmp)
1619        };
1620        self.assembler.emit_add(
1621            Size::S64,
1622            Location::GPR(GPR::XzrSp),
1623            delta,
1624            Location::GPR(GPR::XzrSp),
1625        )
1626    }
1627    fn pop_stack_locals(&mut self, delta_stack_offset: u32) -> Result<(), CompileError> {
1628        let real_delta = if delta_stack_offset & 15 != 0 {
1629            delta_stack_offset + 8
1630        } else {
1631            delta_stack_offset
1632        };
1633        let delta = if self.compatible_imm(real_delta as i64, ImmType::Bits12) {
1634            Location::Imm32(real_delta as _)
1635        } else {
1636            let tmp = GPR::X17;
1637            self.assembler
1638                .emit_mov_imm(Location::GPR(tmp), real_delta as u64)?;
1639            Location::GPR(tmp)
1640        };
1641        self.assembler.emit_add(
1642            Size::S64,
1643            Location::GPR(GPR::XzrSp),
1644            delta,
1645            Location::GPR(GPR::XzrSp),
1646        )
1647    }
1648    // push a value on the stack for a native call
1649    fn move_location_for_native(
1650        &mut self,
1651        size: Size,
1652        loc: Location,
1653        dest: Location,
1654    ) -> Result<(), CompileError> {
1655        match loc {
1656            Location::Imm64(_)
1657            | Location::Imm32(_)
1658            | Location::Imm8(_)
1659            | Location::Memory(_, _)
1660            | Location::Memory2(_, _, _, _) => {
1661                self.move_location(size, loc, Location::GPR(GPR::X17))?;
1662                self.move_location(size, Location::GPR(GPR::X17), dest)
1663            }
1664            _ => self.move_location(size, loc, dest),
1665        }
1666    }
1667
1668    // Zero a location that is 32bits
1669    fn zero_location(&mut self, size: Size, location: Location) -> Result<(), CompileError> {
1670        self.move_location(size, Location::GPR(GPR::XzrSp), location)
1671    }
1672
1673    // GPR Reg used for local pointer on the stack
1674    fn local_pointer(&self) -> GPR {
1675        GPR::X29
1676    }
1677
1678    // Determine whether a local should be allocated on the stack.
1679    fn is_local_on_stack(&self, idx: usize) -> bool {
1680        idx > 7
1681    }
1682
1683    // Determine a local's location.
1684    fn get_local_location(&self, idx: usize, callee_saved_regs_size: usize) -> Location {
1685        // Use callee-saved registers for the first locals.
1686        match idx {
1687            0 => Location::GPR(GPR::X19),
1688            1 => Location::GPR(GPR::X20),
1689            2 => Location::GPR(GPR::X21),
1690            3 => Location::GPR(GPR::X22),
1691            4 => Location::GPR(GPR::X23),
1692            5 => Location::GPR(GPR::X24),
1693            6 => Location::GPR(GPR::X25),
1694            7 => Location::GPR(GPR::X26),
1695            _ => Location::Memory(GPR::X29, -(((idx - 7) * 8 + callee_saved_regs_size) as i32)),
1696        }
1697    }
1698    // Move a local to the stack
1699    fn move_local(&mut self, stack_offset: i32, location: Location) -> Result<(), CompileError> {
1700        if stack_offset < 256 {
1701            self.assembler
1702                .emit_stur(Size::S64, location, GPR::X29, -stack_offset)?;
1703        } else {
1704            let tmp = GPR::X17;
1705            if stack_offset < 0x1_0000 {
1706                self.assembler
1707                    .emit_mov_imm(Location::GPR(tmp), (-stack_offset as i64) as u64)?;
1708                self.assembler.emit_str(
1709                    Size::S64,
1710                    location,
1711                    Location::Memory2(GPR::X29, tmp, Multiplier::One, 0),
1712                )?;
1713            } else {
1714                self.assembler
1715                    .emit_mov_imm(Location::GPR(tmp), (stack_offset as i64) as u64)?;
1716                self.assembler.emit_sub(
1717                    Size::S64,
1718                    Location::GPR(GPR::X29),
1719                    Location::GPR(tmp),
1720                    Location::GPR(tmp),
1721                )?;
1722                self.assembler
1723                    .emit_str(Size::S64, location, Location::GPR(tmp))?;
1724            }
1725        }
1726        match location {
1727            Location::GPR(x) => self.emit_unwind_op(UnwindOps::SaveRegister {
1728                reg: UnwindRegister::<GPR, NEON>::GPR(x),
1729                bp_neg_offset: stack_offset,
1730            }),
1731            Location::SIMD(x) => self.emit_unwind_op(UnwindOps::SaveRegister {
1732                reg: UnwindRegister::FPR(x),
1733                bp_neg_offset: stack_offset,
1734            }),
1735            _ => (),
1736        }
1737        Ok(())
1738    }
1739
1740    // List of register to save, depending on the CallingConvention
1741    fn list_to_save(&self, _calling_convention: CallingConvention) -> Vec<Location> {
1742        vec![]
1743    }
1744
1745    // Get param location, MUST be called in order!
1746    fn get_param_location(
1747        &self,
1748        idx: usize,
1749        sz: Size,
1750        stack_args: &mut usize,
1751        calling_convention: CallingConvention,
1752    ) -> Location {
1753        match calling_convention {
1754            CallingConvention::AppleAarch64 => match idx {
1755                0 => Location::GPR(GPR::X0),
1756                1 => Location::GPR(GPR::X1),
1757                2 => Location::GPR(GPR::X2),
1758                3 => Location::GPR(GPR::X3),
1759                4 => Location::GPR(GPR::X4),
1760                5 => Location::GPR(GPR::X5),
1761                6 => Location::GPR(GPR::X6),
1762                7 => Location::GPR(GPR::X7),
1763                _ => {
1764                    let sz = 1
1765                        << match sz {
1766                            Size::S8 => 0,
1767                            Size::S16 => 1,
1768                            Size::S32 => 2,
1769                            Size::S64 => 3,
1770                        };
1771                    // align first
1772                    if sz > 1 && *stack_args & (sz - 1) != 0 {
1773                        *stack_args = (*stack_args + (sz - 1)) & !(sz - 1);
1774                    }
1775                    let loc = Location::Memory(GPR::XzrSp, *stack_args as i32);
1776                    *stack_args += sz;
1777                    loc
1778                }
1779            },
1780            _ => match idx {
1781                0 => Location::GPR(GPR::X0),
1782                1 => Location::GPR(GPR::X1),
1783                2 => Location::GPR(GPR::X2),
1784                3 => Location::GPR(GPR::X3),
1785                4 => Location::GPR(GPR::X4),
1786                5 => Location::GPR(GPR::X5),
1787                6 => Location::GPR(GPR::X6),
1788                7 => Location::GPR(GPR::X7),
1789                _ => {
1790                    let loc = Location::Memory(GPR::XzrSp, *stack_args as i32);
1791                    *stack_args += 8;
1792                    loc
1793                }
1794            },
1795        }
1796    }
1797    // Get call param location, MUST be called in order!
1798    fn get_call_param_location(
1799        &self,
1800        idx: usize,
1801        sz: Size,
1802        stack_args: &mut usize,
1803        calling_convention: CallingConvention,
1804    ) -> Location {
1805        match calling_convention {
1806            CallingConvention::AppleAarch64 => match idx {
1807                0 => Location::GPR(GPR::X0),
1808                1 => Location::GPR(GPR::X1),
1809                2 => Location::GPR(GPR::X2),
1810                3 => Location::GPR(GPR::X3),
1811                4 => Location::GPR(GPR::X4),
1812                5 => Location::GPR(GPR::X5),
1813                6 => Location::GPR(GPR::X6),
1814                7 => Location::GPR(GPR::X7),
1815                _ => {
1816                    let sz = 1
1817                        << match sz {
1818                            Size::S8 => 0,
1819                            Size::S16 => 1,
1820                            Size::S32 => 2,
1821                            Size::S64 => 3,
1822                        };
1823                    // align first
1824                    if sz > 1 && *stack_args & (sz - 1) != 0 {
1825                        *stack_args = (*stack_args + (sz - 1)) & !(sz - 1);
1826                    }
1827                    let loc = Location::Memory(GPR::X29, 16 * 2 + *stack_args as i32);
1828                    *stack_args += sz;
1829                    loc
1830                }
1831            },
1832            _ => match idx {
1833                0 => Location::GPR(GPR::X0),
1834                1 => Location::GPR(GPR::X1),
1835                2 => Location::GPR(GPR::X2),
1836                3 => Location::GPR(GPR::X3),
1837                4 => Location::GPR(GPR::X4),
1838                5 => Location::GPR(GPR::X5),
1839                6 => Location::GPR(GPR::X6),
1840                7 => Location::GPR(GPR::X7),
1841                _ => {
1842                    let loc = Location::Memory(GPR::X29, 16 * 2 + *stack_args as i32);
1843                    *stack_args += 8;
1844                    loc
1845                }
1846            },
1847        }
1848    }
1849    // Get simple param location, Will not be accurate for Apple calling convention on "stack" arguments
1850    fn get_simple_param_location(
1851        &self,
1852        idx: usize,
1853        calling_convention: CallingConvention,
1854    ) -> Location {
1855        #[allow(clippy::match_single_binding)]
1856        match calling_convention {
1857            _ => match idx {
1858                0 => Location::GPR(GPR::X0),
1859                1 => Location::GPR(GPR::X1),
1860                2 => Location::GPR(GPR::X2),
1861                3 => Location::GPR(GPR::X3),
1862                4 => Location::GPR(GPR::X4),
1863                5 => Location::GPR(GPR::X5),
1864                6 => Location::GPR(GPR::X6),
1865                7 => Location::GPR(GPR::X7),
1866                _ => Location::Memory(GPR::X29, (16 * 2 + (idx - 8) * 8) as i32),
1867            },
1868        }
1869    }
1870    // move a location to another
1871    fn move_location(
1872        &mut self,
1873        size: Size,
1874        source: Location,
1875        dest: Location,
1876    ) -> Result<(), CompileError> {
1877        match source {
1878            Location::GPR(_) | Location::SIMD(_) => match dest {
1879                Location::GPR(_) | Location::SIMD(_) => self.assembler.emit_mov(size, source, dest),
1880                Location::Memory(addr, offs) => {
1881                    if self.offset_is_ok(size, offs) {
1882                        self.assembler.emit_str(size, source, dest)
1883                    } else if self.compatible_imm(offs as i64, ImmType::UnscaledOffset) {
1884                        self.assembler.emit_stur(size, source, addr, offs)
1885                    } else {
1886                        let tmp = GPR::X17;
1887                        if offs < 0 {
1888                            self.assembler
1889                                .emit_mov_imm(Location::GPR(tmp), (-offs) as u64)?;
1890                            self.assembler.emit_sub(
1891                                Size::S64,
1892                                Location::GPR(addr),
1893                                Location::GPR(tmp),
1894                                Location::GPR(tmp),
1895                            )?;
1896                        } else {
1897                            self.assembler
1898                                .emit_mov_imm(Location::GPR(tmp), offs as u64)?;
1899                            self.assembler.emit_add(
1900                                Size::S64,
1901                                Location::GPR(addr),
1902                                Location::GPR(tmp),
1903                                Location::GPR(tmp),
1904                            )?;
1905                        }
1906                        self.assembler
1907                            .emit_str(size, source, Location::Memory(tmp, 0))
1908                    }
1909                }
1910                _ => codegen_error!(
1911                    "singlepass can't emit move_location {:?} {:?} => {:?}",
1912                    size,
1913                    source,
1914                    dest
1915                ),
1916            },
1917            Location::Imm8(_) => match dest {
1918                Location::GPR(_) => self.assembler.emit_mov(size, source, dest),
1919                Location::Memory(_, _) => match size {
1920                    Size::S64 => self.emit_relaxed_str64(source, dest),
1921                    Size::S32 => self.emit_relaxed_str32(source, dest),
1922                    Size::S16 => self.emit_relaxed_str16(source, dest),
1923                    Size::S8 => self.emit_relaxed_str8(source, dest),
1924                },
1925                _ => codegen_error!(
1926                    "singlepass can't emit move_location {:?} {:?} => {:?}",
1927                    size,
1928                    source,
1929                    dest
1930                ),
1931            },
1932            Location::Imm32(val) => match dest {
1933                Location::GPR(_) => self.assembler.emit_mov_imm(dest, val as u64),
1934                Location::Memory(_, _) => match size {
1935                    Size::S64 => self.emit_relaxed_str64(source, dest),
1936                    Size::S32 => self.emit_relaxed_str32(source, dest),
1937                    Size::S16 => self.emit_relaxed_str16(source, dest),
1938                    Size::S8 => self.emit_relaxed_str8(source, dest),
1939                },
1940                _ => codegen_error!(
1941                    "singlepass can't emit move_location {:?} {:?} => {:?}",
1942                    size,
1943                    source,
1944                    dest
1945                ),
1946            },
1947            Location::Imm64(val) => match dest {
1948                Location::GPR(_) => self.assembler.emit_mov_imm(dest, val),
1949                Location::Memory(_, _) => match size {
1950                    Size::S64 => self.emit_relaxed_str64(source, dest),
1951                    Size::S32 => self.emit_relaxed_str32(source, dest),
1952                    Size::S16 => self.emit_relaxed_str16(source, dest),
1953                    Size::S8 => self.emit_relaxed_str8(source, dest),
1954                },
1955                _ => codegen_error!(
1956                    "singlepass can't emit move_location {:?} {:?} => {:?}",
1957                    size,
1958                    source,
1959                    dest
1960                ),
1961            },
1962            Location::Memory(addr, offs) => match dest {
1963                Location::GPR(_) | Location::SIMD(_) => {
1964                    if self.offset_is_ok(size, offs) {
1965                        self.assembler.emit_ldr(size, dest, source)
1966                    } else if offs > -256 && offs < 256 {
1967                        self.assembler.emit_ldur(size, dest, addr, offs)
1968                    } else {
1969                        let tmp = GPR::X17;
1970                        if offs < 0 {
1971                            self.assembler
1972                                .emit_mov_imm(Location::GPR(tmp), (-offs) as u64)?;
1973                            self.assembler.emit_sub(
1974                                Size::S64,
1975                                Location::GPR(addr),
1976                                Location::GPR(tmp),
1977                                Location::GPR(tmp),
1978                            )?;
1979                        } else {
1980                            self.assembler
1981                                .emit_mov_imm(Location::GPR(tmp), offs as u64)?;
1982                            self.assembler.emit_add(
1983                                Size::S64,
1984                                Location::GPR(addr),
1985                                Location::GPR(tmp),
1986                                Location::GPR(tmp),
1987                            )?;
1988                        }
1989                        self.assembler
1990                            .emit_ldr(size, dest, Location::Memory(tmp, 0))
1991                    }
1992                }
1993                _ => {
1994                    let mut temps = vec![];
1995                    let src =
1996                        self.location_to_reg(size, source, &mut temps, ImmType::None, true, None)?;
1997                    self.move_location(size, src, dest)?;
1998                    for r in temps {
1999                        self.release_gpr(r);
2000                    }
2001                    Ok(())
2002                }
2003            },
2004            _ => codegen_error!(
2005                "singlepass can't emit move_location {:?} {:?} => {:?}",
2006                size,
2007                source,
2008                dest
2009            ),
2010        }
2011    }
2012    // move a location to another
2013    fn move_location_extend(
2014        &mut self,
2015        size_val: Size,
2016        signed: bool,
2017        source: Location,
2018        size_op: Size,
2019        dest: Location,
2020    ) -> Result<(), CompileError> {
2021        if size_op != Size::S64 {
2022            codegen_error!("singlepass move_location_extend unreachable");
2023        }
2024        let mut temps = vec![];
2025        let dst = self.location_to_reg(size_op, dest, &mut temps, ImmType::None, false, None)?;
2026        let src = match (size_val, signed, source) {
2027            (Size::S64, _, _) => source,
2028            (Size::S32, false, Location::GPR(_)) => {
2029                self.assembler.emit_mov(size_val, source, dst)?;
2030                dst
2031            }
2032            (Size::S8, false, Location::GPR(_)) => {
2033                self.assembler.emit_uxtb(size_op, source, dst)?;
2034                dst
2035            }
2036            (Size::S16, false, Location::GPR(_)) => {
2037                self.assembler.emit_uxth(size_op, source, dst)?;
2038                dst
2039            }
2040            (Size::S8, true, Location::GPR(_)) => {
2041                self.assembler.emit_sxtb(size_op, source, dst)?;
2042                dst
2043            }
2044            (Size::S16, true, Location::GPR(_)) => {
2045                self.assembler.emit_sxth(size_op, source, dst)?;
2046                dst
2047            }
2048            (Size::S32, true, Location::GPR(_)) => {
2049                self.assembler.emit_sxtw(size_op, source, dst)?;
2050                dst
2051            }
2052            (Size::S32, false, Location::Memory(_, _)) => {
2053                self.emit_relaxed_ldr32(size_op, dst, source)?;
2054                dst
2055            }
2056            (Size::S32, true, Location::Memory(_, _)) => {
2057                self.emit_relaxed_ldr32s(size_op, dst, source)?;
2058                dst
2059            }
2060            (Size::S16, false, Location::Memory(_, _)) => {
2061                self.emit_relaxed_ldr16(size_op, dst, source)?;
2062                dst
2063            }
2064            (Size::S16, true, Location::Memory(_, _)) => {
2065                self.emit_relaxed_ldr16s(size_op, dst, source)?;
2066                dst
2067            }
2068            (Size::S8, false, Location::Memory(_, _)) => {
2069                self.emit_relaxed_ldr8(size_op, dst, source)?;
2070                dst
2071            }
2072            (Size::S8, true, Location::Memory(_, _)) => {
2073                self.emit_relaxed_ldr8s(size_op, dst, source)?;
2074                dst
2075            }
2076            _ => codegen_error!(
2077                "singlepass can't emit move_location_extend {:?} {:?} {:?} => {:?} {:?}",
2078                size_val,
2079                signed,
2080                source,
2081                size_op,
2082                dest
2083            ),
2084        };
2085        if src != dst {
2086            self.move_location(size_op, src, dst)?;
2087        }
2088        if dst != dest {
2089            self.move_location(size_op, dst, dest)?;
2090        }
2091        for r in temps {
2092            self.release_gpr(r);
2093        }
2094        Ok(())
2095    }
2096    fn load_address(
2097        &mut self,
2098        _size: Size,
2099        _reg: Location,
2100        _mem: Location,
2101    ) -> Result<(), CompileError> {
2102        codegen_error!("singlepass load_address unimplemented");
2103    }
2104    // Init the stack loc counter
2105    fn init_stack_loc(
2106        &mut self,
2107        init_stack_loc_cnt: u64,
2108        last_stack_loc: Location,
2109    ) -> Result<(), CompileError> {
2110        let label = self.assembler.get_label();
2111        let mut temps = vec![];
2112        let dest = self.acquire_temp_gpr().ok_or_else(|| {
2113            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2114        })?;
2115        temps.push(dest);
2116        let cnt = self.location_to_reg(
2117            Size::S64,
2118            Location::Imm64(init_stack_loc_cnt),
2119            &mut temps,
2120            ImmType::None,
2121            true,
2122            None,
2123        )?;
2124        let dest = match last_stack_loc {
2125            Location::GPR(_) => codegen_error!("singlepass init_stack_loc unreachable"),
2126            Location::SIMD(_) => codegen_error!("singlepass init_stack_loc unreachable"),
2127            Location::Memory(reg, offset) => {
2128                if offset < 0 {
2129                    let offset = (-offset) as u32;
2130                    if self.compatible_imm(offset as i64, ImmType::Bits12) {
2131                        self.assembler.emit_sub(
2132                            Size::S64,
2133                            Location::GPR(reg),
2134                            Location::Imm32(offset),
2135                            Location::GPR(dest),
2136                        )?;
2137                    } else {
2138                        let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2139                            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2140                        })?;
2141                        self.assembler
2142                            .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
2143                        self.assembler.emit_sub(
2144                            Size::S64,
2145                            Location::GPR(reg),
2146                            Location::GPR(tmp),
2147                            Location::GPR(dest),
2148                        )?;
2149                        temps.push(tmp);
2150                    }
2151                    dest
2152                } else {
2153                    let offset = offset as u32;
2154                    if self.compatible_imm(offset as i64, ImmType::Bits12) {
2155                        self.assembler.emit_add(
2156                            Size::S64,
2157                            Location::GPR(reg),
2158                            Location::Imm32(offset),
2159                            Location::GPR(dest),
2160                        )?;
2161                    } else {
2162                        let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2163                            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2164                        })?;
2165                        self.assembler
2166                            .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
2167                        self.assembler.emit_add(
2168                            Size::S64,
2169                            Location::GPR(reg),
2170                            Location::GPR(tmp),
2171                            Location::GPR(dest),
2172                        )?;
2173                        temps.push(tmp);
2174                    }
2175                    dest
2176                }
2177            }
2178            _ => codegen_error!("singlepass can't emit init_stack_loc {:?}", last_stack_loc),
2179        };
2180        self.assembler.emit_label(label)?;
2181        self.assembler
2182            .emit_stria(Size::S64, Location::GPR(GPR::XzrSp), dest, 8)?;
2183        self.assembler
2184            .emit_sub(Size::S64, cnt, Location::Imm8(1), cnt)?;
2185        self.assembler.emit_cbnz_label(Size::S64, cnt, label)?;
2186        for r in temps {
2187            self.release_gpr(r);
2188        }
2189        Ok(())
2190    }
2191    // Restore save_area
2192    fn restore_saved_area(&mut self, saved_area_offset: i32) -> Result<(), CompileError> {
2193        let real_delta = if saved_area_offset & 15 != 0 {
2194            self.pushed = true;
2195            saved_area_offset + 8
2196        } else {
2197            self.pushed = false;
2198            saved_area_offset
2199        };
2200        if self.compatible_imm(real_delta as _, ImmType::Bits12) {
2201            self.assembler.emit_sub(
2202                Size::S64,
2203                Location::GPR(GPR::X29),
2204                Location::Imm32(real_delta as _),
2205                Location::GPR(GPR::XzrSp),
2206            )?;
2207        } else {
2208            let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2209                CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2210            })?;
2211            self.assembler
2212                .emit_mov_imm(Location::GPR(tmp), real_delta as u64)?;
2213            self.assembler.emit_sub(
2214                Size::S64,
2215                Location::GPR(GPR::X29),
2216                Location::GPR(tmp),
2217                Location::GPR(GPR::XzrSp),
2218            )?;
2219            self.release_gpr(tmp);
2220        }
2221        Ok(())
2222    }
2223    // Pop a location
2224    fn pop_location(&mut self, location: Location) -> Result<(), CompileError> {
2225        self.emit_pop(Size::S64, location)
2226    }
2227    // Create a new `MachineState` with default values.
2228    fn new_machine_state(&self) -> MachineState {
2229        new_machine_state()
2230    }
2231
2232    // assembler finalize
2233    fn assembler_finalize(self) -> Result<Vec<u8>, CompileError> {
2234        self.assembler.finalize().map_err(|e| {
2235            CompileError::Codegen(format!("Assembler failed finalization with: {e:?}"))
2236        })
2237    }
2238
2239    fn get_offset(&self) -> Offset {
2240        self.assembler.get_offset()
2241    }
2242
2243    fn finalize_function(&mut self) -> Result<(), CompileError> {
2244        self.assembler.finalize_function();
2245        Ok(())
2246    }
2247
2248    fn emit_function_prolog(&mut self) -> Result<(), CompileError> {
2249        self.emit_double_push(Size::S64, Location::GPR(GPR::X29), Location::GPR(GPR::X30))?; // save LR too
2250        self.emit_unwind_op(UnwindOps::Push2Regs {
2251            reg1: UnwindRegister::GPR(GPR::X29),
2252            reg2: UnwindRegister::GPR(GPR::X30),
2253            up_to_sp: 16,
2254        });
2255        self.emit_double_push(Size::S64, Location::GPR(GPR::X27), Location::GPR(GPR::X28))?;
2256        self.emit_unwind_op(UnwindOps::Push2Regs {
2257            reg1: UnwindRegister::GPR(GPR::X27),
2258            reg2: UnwindRegister::GPR(GPR::X28),
2259            up_to_sp: 32,
2260        });
2261        // cannot use mov, because XSP is XZR there. Need to use ADD with #0
2262        self.assembler.emit_add(
2263            Size::S64,
2264            Location::GPR(GPR::XzrSp),
2265            Location::Imm8(0),
2266            Location::GPR(GPR::X29),
2267        )?;
2268        self.emit_unwind_op(UnwindOps::DefineNewFrame);
2269        Ok(())
2270    }
2271
2272    fn emit_function_epilog(&mut self) -> Result<(), CompileError> {
2273        // cannot use mov, because XSP is XZR there. Need to use ADD with #0
2274        self.assembler.emit_add(
2275            Size::S64,
2276            Location::GPR(GPR::X29),
2277            Location::Imm8(0),
2278            Location::GPR(GPR::XzrSp),
2279        )?;
2280        self.pushed = false; // SP is restored, consider it aligned
2281        self.emit_double_pop(Size::S64, Location::GPR(GPR::X27), Location::GPR(GPR::X28))?;
2282        self.emit_double_pop(Size::S64, Location::GPR(GPR::X29), Location::GPR(GPR::X30))?;
2283        Ok(())
2284    }
2285
2286    fn emit_function_return_value(
2287        &mut self,
2288        ty: WpType,
2289        canonicalize: bool,
2290        loc: Location,
2291    ) -> Result<(), CompileError> {
2292        if canonicalize {
2293            self.canonicalize_nan(
2294                match ty {
2295                    WpType::F32 => Size::S32,
2296                    WpType::F64 => Size::S64,
2297                    _ => unreachable!(),
2298                },
2299                loc,
2300                Location::GPR(GPR::X0),
2301            )?;
2302        } else {
2303            self.emit_relaxed_mov(Size::S64, loc, Location::GPR(GPR::X0))?;
2304        }
2305        Ok(())
2306    }
2307
2308    fn emit_function_return_float(&mut self) -> Result<(), CompileError> {
2309        self.assembler
2310            .emit_mov(Size::S64, Location::GPR(GPR::X0), Location::SIMD(NEON::V0))
2311    }
2312
2313    fn arch_supports_canonicalize_nan(&self) -> bool {
2314        self.assembler.arch_supports_canonicalize_nan()
2315    }
2316    fn canonicalize_nan(
2317        &mut self,
2318        sz: Size,
2319        input: Location,
2320        output: Location,
2321    ) -> Result<(), CompileError> {
2322        let mut tempn = vec![];
2323        let mut temps = vec![];
2324        let old_fpcr = self.set_default_nan(&mut temps)?;
2325        // use FMAX (input, intput) => output to automaticaly normalize the NaN
2326        match (sz, input, output) {
2327            (Size::S32, Location::SIMD(_), Location::SIMD(_)) => {
2328                self.assembler.emit_fmax(sz, input, input, output)?;
2329            }
2330            (Size::S64, Location::SIMD(_), Location::SIMD(_)) => {
2331                self.assembler.emit_fmax(sz, input, input, output)?;
2332            }
2333            (Size::S32, Location::SIMD(_), _) | (Size::S64, Location::SIMD(_), _) => {
2334                let tmp = self.location_to_neon(sz, output, &mut tempn, ImmType::None, false)?;
2335                self.assembler.emit_fmax(sz, input, input, tmp)?;
2336                self.move_location(sz, tmp, output)?;
2337            }
2338            (Size::S32, Location::Memory(_, _), _) | (Size::S64, Location::Memory(_, _), _) => {
2339                let src = self.location_to_neon(sz, input, &mut tempn, ImmType::None, true)?;
2340                let tmp = self.location_to_neon(sz, output, &mut tempn, ImmType::None, false)?;
2341                self.assembler.emit_fmax(sz, src, src, tmp)?;
2342                if tmp != output {
2343                    self.move_location(sz, tmp, output)?;
2344                }
2345            }
2346            _ => codegen_error!(
2347                "singlepass can't emit canonicalize_nan {:?} {:?} {:?}",
2348                sz,
2349                input,
2350                output
2351            ),
2352        }
2353
2354        self.restore_fpcr(old_fpcr)?;
2355        for r in temps {
2356            self.release_gpr(r);
2357        }
2358        for r in tempn {
2359            self.release_simd(r);
2360        }
2361        Ok(())
2362    }
2363
2364    fn emit_illegal_op(&mut self, trap: TrapCode) -> Result<(), CompileError> {
2365        let offset = self.assembler.get_offset().0;
2366        self.assembler.emit_udf(0xc0 | (trap as u8) as u16)?;
2367        self.mark_instruction_address_end(offset);
2368        Ok(())
2369    }
2370    fn get_label(&mut self) -> Label {
2371        self.assembler.new_dynamic_label()
2372    }
2373    fn emit_label(&mut self, label: Label) -> Result<(), CompileError> {
2374        self.assembler.emit_label(label)
2375    }
2376    fn get_grp_for_call(&self) -> GPR {
2377        GPR::X27
2378    }
2379    fn emit_call_register(&mut self, reg: GPR) -> Result<(), CompileError> {
2380        self.assembler.emit_call_register(reg)
2381    }
2382    fn emit_call_label(&mut self, label: Label) -> Result<(), CompileError> {
2383        self.assembler.emit_call_label(label)
2384    }
2385    fn get_gpr_for_ret(&self) -> GPR {
2386        GPR::X0
2387    }
2388    fn get_simd_for_ret(&self) -> NEON {
2389        NEON::V0
2390    }
2391
2392    fn arch_requires_indirect_call_trampoline(&self) -> bool {
2393        self.assembler.arch_requires_indirect_call_trampoline()
2394    }
2395
2396    fn arch_emit_indirect_call_with_trampoline(
2397        &mut self,
2398        location: Location,
2399    ) -> Result<(), CompileError> {
2400        self.assembler
2401            .arch_emit_indirect_call_with_trampoline(location)
2402    }
2403
2404    fn emit_debug_breakpoint(&mut self) -> Result<(), CompileError> {
2405        self.assembler.emit_brk()
2406    }
2407
2408    fn emit_call_location(&mut self, location: Location) -> Result<(), CompileError> {
2409        let mut temps = vec![];
2410        let loc = self.location_to_reg(
2411            Size::S64,
2412            location,
2413            &mut temps,
2414            ImmType::None,
2415            true,
2416            Some(GPR::X27),
2417        )?;
2418        match loc {
2419            Location::GPR(reg) => self.assembler.emit_call_register(reg),
2420            _ => codegen_error!("singlepass can't emit CALL Location"),
2421        }?;
2422        for r in temps {
2423            self.release_gpr(r);
2424        }
2425        Ok(())
2426    }
2427
2428    fn location_address(
2429        &mut self,
2430        _size: Size,
2431        _source: Location,
2432        _dest: Location,
2433    ) -> Result<(), CompileError> {
2434        codegen_error!("singlepass location_address not implemented")
2435    }
2436    // logic
2437    fn location_and(
2438        &mut self,
2439        _size: Size,
2440        _source: Location,
2441        _dest: Location,
2442        _flags: bool,
2443    ) -> Result<(), CompileError> {
2444        codegen_error!("singlepass location_and not implemented")
2445    }
2446    fn location_xor(
2447        &mut self,
2448        _size: Size,
2449        _source: Location,
2450        _dest: Location,
2451        _flags: bool,
2452    ) -> Result<(), CompileError> {
2453        codegen_error!("singlepass location_xor not implemented")
2454    }
2455    fn location_or(
2456        &mut self,
2457        _size: Size,
2458        _source: Location,
2459        _dest: Location,
2460        _flags: bool,
2461    ) -> Result<(), CompileError> {
2462        codegen_error!("singlepass location_or not implemented")
2463    }
2464    fn location_test(
2465        &mut self,
2466        _size: Size,
2467        _source: Location,
2468        _dest: Location,
2469    ) -> Result<(), CompileError> {
2470        codegen_error!("singlepass location_test not implemented")
2471    }
2472    // math
2473    fn location_add(
2474        &mut self,
2475        size: Size,
2476        source: Location,
2477        dest: Location,
2478        flags: bool,
2479    ) -> Result<(), CompileError> {
2480        let mut temps = vec![];
2481        let src = self.location_to_reg(size, source, &mut temps, ImmType::Bits12, true, None)?;
2482        let dst = self.location_to_reg(size, dest, &mut temps, ImmType::None, true, None)?;
2483        if flags {
2484            self.assembler.emit_adds(size, dst, src, dst)?;
2485        } else {
2486            self.assembler.emit_add(size, dst, src, dst)?;
2487        }
2488        if dst != dest {
2489            self.move_location(size, dst, dest)?;
2490        }
2491        for r in temps {
2492            self.release_gpr(r);
2493        }
2494        Ok(())
2495    }
2496    fn location_sub(
2497        &mut self,
2498        size: Size,
2499        source: Location,
2500        dest: Location,
2501        flags: bool,
2502    ) -> Result<(), CompileError> {
2503        let mut temps = vec![];
2504        let src = self.location_to_reg(size, source, &mut temps, ImmType::Bits12, true, None)?;
2505        let dst = self.location_to_reg(size, dest, &mut temps, ImmType::None, true, None)?;
2506        if flags {
2507            self.assembler.emit_subs(size, dst, src, dst)?;
2508        } else {
2509            self.assembler.emit_sub(size, dst, src, dst)?;
2510        }
2511        if dst != dest {
2512            self.move_location(size, dst, dest)?;
2513        }
2514        for r in temps {
2515            self.release_gpr(r);
2516        }
2517        Ok(())
2518    }
2519    fn location_cmp(
2520        &mut self,
2521        size: Size,
2522        source: Location,
2523        dest: Location,
2524    ) -> Result<(), CompileError> {
2525        self.emit_relaxed_binop(Assembler::emit_cmp, size, source, dest, false)
2526    }
2527    fn jmp_unconditionnal(&mut self, label: Label) -> Result<(), CompileError> {
2528        self.assembler.emit_b_label(label)
2529    }
2530
2531    fn jmp_on_condition(
2532        &mut self,
2533        cond: UnsignedCondition,
2534        size: Size,
2535        source: AbstractLocation<Self::GPR, Self::SIMD>,
2536        dest: AbstractLocation<Self::GPR, Self::SIMD>,
2537        label: Label,
2538    ) -> Result<(), CompileError> {
2539        self.emit_relaxed_binop(Assembler::emit_cmp, size, source, dest, false)?;
2540        let cond = match cond {
2541            UnsignedCondition::Equal => Condition::Eq,
2542            UnsignedCondition::NotEqual => Condition::Ne,
2543            UnsignedCondition::Above => Condition::Hi,
2544            UnsignedCondition::AboveEqual => Condition::Cs,
2545            UnsignedCondition::Below => Condition::Cc,
2546            UnsignedCondition::BelowEqual => Condition::Ls,
2547        };
2548        self.assembler.emit_bcond_label_far(cond, label)
2549    }
2550
2551    // jmp table
2552    fn emit_jmp_to_jumptable(&mut self, label: Label, cond: Location) -> Result<(), CompileError> {
2553        let tmp1 = self.acquire_temp_gpr().ok_or_else(|| {
2554            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2555        })?;
2556        let tmp2 = self.acquire_temp_gpr().ok_or_else(|| {
2557            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2558        })?;
2559
2560        self.assembler.emit_load_label(tmp1, label)?;
2561        self.move_location(Size::S32, cond, Location::GPR(tmp2))?;
2562
2563        self.assembler.emit_add_lsl(
2564            Size::S64,
2565            Location::GPR(tmp1),
2566            Location::GPR(tmp2),
2567            2,
2568            Location::GPR(tmp2),
2569        )?;
2570        self.assembler.emit_b_register(tmp2)?;
2571        self.release_gpr(tmp2);
2572        self.release_gpr(tmp1);
2573        Ok(())
2574    }
2575
2576    fn align_for_loop(&mut self) -> Result<(), CompileError> {
2577        // noting to do on ARM64
2578        Ok(())
2579    }
2580
2581    fn emit_ret(&mut self) -> Result<(), CompileError> {
2582        self.assembler.emit_ret()
2583    }
2584
2585    fn emit_push(&mut self, size: Size, loc: Location) -> Result<(), CompileError> {
2586        self.emit_push(size, loc)
2587    }
2588    fn emit_pop(&mut self, size: Size, loc: Location) -> Result<(), CompileError> {
2589        self.emit_pop(size, loc)
2590    }
2591
2592    fn emit_memory_fence(&mut self) -> Result<(), CompileError> {
2593        self.assembler.emit_dmb()
2594    }
2595
2596    fn location_neg(
2597        &mut self,
2598        _size_val: Size, // size of src
2599        _signed: bool,
2600        _source: Location,
2601        _size_op: Size,
2602        _dest: Location,
2603    ) -> Result<(), CompileError> {
2604        codegen_error!("singlepass location_neg unimplemented");
2605    }
2606
2607    fn emit_imul_imm32(&mut self, size: Size, imm32: u32, gpr: GPR) -> Result<(), CompileError> {
2608        let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2609            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2610        })?;
2611        self.assembler
2612            .emit_mov_imm(Location::GPR(tmp), imm32 as u64)?;
2613        self.assembler.emit_mul(
2614            size,
2615            Location::GPR(gpr),
2616            Location::GPR(tmp),
2617            Location::GPR(gpr),
2618        )?;
2619        self.release_gpr(tmp);
2620        Ok(())
2621    }
2622
2623    // relaxed binop based...
2624    fn emit_relaxed_mov(
2625        &mut self,
2626        sz: Size,
2627        src: Location,
2628        dst: Location,
2629    ) -> Result<(), CompileError> {
2630        self.emit_relaxed_binop(Assembler::emit_mov, sz, src, dst, true)
2631    }
2632    fn emit_relaxed_cmp(
2633        &mut self,
2634        sz: Size,
2635        src: Location,
2636        dst: Location,
2637    ) -> Result<(), CompileError> {
2638        self.emit_relaxed_binop(Assembler::emit_cmp, sz, src, dst, false)
2639    }
2640    fn emit_relaxed_zero_extension(
2641        &mut self,
2642        _sz_src: Size,
2643        _src: Location,
2644        _sz_dst: Size,
2645        _dst: Location,
2646    ) -> Result<(), CompileError> {
2647        codegen_error!("singlepass emit_relaxed_zero_extension unimplemented");
2648    }
2649    fn emit_relaxed_sign_extension(
2650        &mut self,
2651        sz_src: Size,
2652        src: Location,
2653        sz_dst: Size,
2654        dst: Location,
2655    ) -> Result<(), CompileError> {
2656        match (src, dst) {
2657            (Location::Memory(_, _), Location::GPR(_)) => match sz_src {
2658                Size::S8 => self.emit_relaxed_ldr8s(sz_dst, dst, src),
2659                Size::S16 => self.emit_relaxed_ldr16s(sz_dst, dst, src),
2660                Size::S32 => self.emit_relaxed_ldr32s(sz_dst, dst, src),
2661                _ => codegen_error!("singlepass emit_relaxed_sign_extension unreachable"),
2662            },
2663            _ => {
2664                let mut temps = vec![];
2665                let src =
2666                    self.location_to_reg(sz_src, src, &mut temps, ImmType::None, true, None)?;
2667                let dest =
2668                    self.location_to_reg(sz_dst, dst, &mut temps, ImmType::None, false, None)?;
2669                match sz_src {
2670                    Size::S8 => self.assembler.emit_sxtb(sz_dst, src, dest),
2671                    Size::S16 => self.assembler.emit_sxth(sz_dst, src, dest),
2672                    Size::S32 => self.assembler.emit_sxtw(sz_dst, src, dest),
2673                    _ => codegen_error!("singlepass emit_relaxed_sign_extension unreachable"),
2674                }?;
2675                if dst != dest {
2676                    self.move_location(sz_dst, dest, dst)?;
2677                }
2678                for r in temps {
2679                    self.release_gpr(r);
2680                }
2681                Ok(())
2682            }
2683        }
2684    }
2685
2686    fn emit_binop_add32(
2687        &mut self,
2688        loc_a: Location,
2689        loc_b: Location,
2690        ret: Location,
2691    ) -> Result<(), CompileError> {
2692        self.emit_relaxed_binop3(
2693            Assembler::emit_add,
2694            Size::S32,
2695            loc_a,
2696            loc_b,
2697            ret,
2698            ImmType::Bits12,
2699        )
2700    }
2701    fn emit_binop_sub32(
2702        &mut self,
2703        loc_a: Location,
2704        loc_b: Location,
2705        ret: Location,
2706    ) -> Result<(), CompileError> {
2707        self.emit_relaxed_binop3(
2708            Assembler::emit_sub,
2709            Size::S32,
2710            loc_a,
2711            loc_b,
2712            ret,
2713            ImmType::Bits12,
2714        )
2715    }
2716    fn emit_binop_mul32(
2717        &mut self,
2718        loc_a: Location,
2719        loc_b: Location,
2720        ret: Location,
2721    ) -> Result<(), CompileError> {
2722        self.emit_relaxed_binop3(
2723            Assembler::emit_mul,
2724            Size::S32,
2725            loc_a,
2726            loc_b,
2727            ret,
2728            ImmType::None,
2729        )
2730    }
2731    fn emit_binop_udiv32(
2732        &mut self,
2733        loc_a: Location,
2734        loc_b: Location,
2735        ret: Location,
2736        integer_division_by_zero: Label,
2737        _integer_overflow: Label,
2738    ) -> Result<usize, CompileError> {
2739        let mut temps = vec![];
2740        let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
2741        let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
2742        let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
2743
2744        self.assembler
2745            .emit_cbz_label_far(Size::S32, src2, integer_division_by_zero)?;
2746        let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
2747        self.assembler.emit_udiv(Size::S32, src1, src2, dest)?;
2748        if ret != dest {
2749            self.move_location(Size::S32, dest, ret)?;
2750        }
2751        for r in temps {
2752            self.release_gpr(r);
2753        }
2754        Ok(offset)
2755    }
2756    fn emit_binop_sdiv32(
2757        &mut self,
2758        loc_a: Location,
2759        loc_b: Location,
2760        ret: Location,
2761        integer_division_by_zero: Label,
2762        integer_overflow: Label,
2763    ) -> Result<usize, CompileError> {
2764        let mut temps = vec![];
2765        let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
2766        let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
2767        let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
2768
2769        self.assembler
2770            .emit_cbz_label_far(Size::S32, src2, integer_division_by_zero)?;
2771        let label_nooverflow = self.assembler.get_label();
2772        let tmp = self.location_to_reg(
2773            Size::S32,
2774            Location::Imm32(0x80000000),
2775            &mut temps,
2776            ImmType::None,
2777            true,
2778            None,
2779        )?;
2780        self.assembler.emit_cmp(Size::S32, tmp, src1)?;
2781        self.assembler
2782            .emit_bcond_label(Condition::Ne, label_nooverflow)?;
2783        self.assembler.emit_movn(Size::S32, tmp, 0)?;
2784        self.assembler.emit_cmp(Size::S32, tmp, src2)?;
2785        self.assembler
2786            .emit_bcond_label_far(Condition::Eq, integer_overflow)?;
2787        let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
2788        self.assembler.emit_label(label_nooverflow)?;
2789        self.assembler.emit_sdiv(Size::S32, src1, src2, dest)?;
2790        if ret != dest {
2791            self.move_location(Size::S32, dest, ret)?;
2792        }
2793        for r in temps {
2794            self.release_gpr(r);
2795        }
2796        Ok(offset)
2797    }
2798    fn emit_binop_urem32(
2799        &mut self,
2800        loc_a: Location,
2801        loc_b: Location,
2802        ret: Location,
2803        integer_division_by_zero: Label,
2804        _integer_overflow: Label,
2805    ) -> Result<usize, CompileError> {
2806        let mut temps = vec![];
2807        let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
2808        let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
2809        let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
2810        let dest = if dest == src1 || dest == src2 {
2811            let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2812                CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2813            })?;
2814            temps.push(tmp);
2815            self.assembler
2816                .emit_mov(Size::S32, dest, Location::GPR(tmp))?;
2817            Location::GPR(tmp)
2818        } else {
2819            dest
2820        };
2821        self.assembler
2822            .emit_cbz_label_far(Size::S32, src2, integer_division_by_zero)?;
2823        let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
2824        self.assembler.emit_udiv(Size::S32, src1, src2, dest)?;
2825        // unsigned remainder : src1 - (src1/src2)*src2
2826        self.assembler
2827            .emit_msub(Size::S32, dest, src2, src1, dest)?;
2828        if ret != dest {
2829            self.move_location(Size::S32, dest, ret)?;
2830        }
2831        for r in temps {
2832            self.release_gpr(r);
2833        }
2834        Ok(offset)
2835    }
2836    fn emit_binop_srem32(
2837        &mut self,
2838        loc_a: Location,
2839        loc_b: Location,
2840        ret: Location,
2841        integer_division_by_zero: Label,
2842        _integer_overflow: Label,
2843    ) -> Result<usize, CompileError> {
2844        let mut temps = vec![];
2845        let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
2846        let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
2847        let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
2848        let dest = if dest == src1 || dest == src2 {
2849            let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2850                CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2851            })?;
2852            temps.push(tmp);
2853            self.assembler
2854                .emit_mov(Size::S32, dest, Location::GPR(tmp))?;
2855            Location::GPR(tmp)
2856        } else {
2857            dest
2858        };
2859        self.assembler
2860            .emit_cbz_label_far(Size::S32, src2, integer_division_by_zero)?;
2861        let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
2862        self.assembler.emit_sdiv(Size::S32, src1, src2, dest)?;
2863        // unsigned remainder : src1 - (src1/src2)*src2
2864        self.assembler
2865            .emit_msub(Size::S32, dest, src2, src1, dest)?;
2866        if ret != dest {
2867            self.move_location(Size::S32, dest, ret)?;
2868        }
2869        for r in temps {
2870            self.release_gpr(r);
2871        }
2872        Ok(offset)
2873    }
2874    fn emit_binop_and32(
2875        &mut self,
2876        loc_a: Location,
2877        loc_b: Location,
2878        ret: Location,
2879    ) -> Result<(), CompileError> {
2880        self.emit_relaxed_binop3(
2881            Assembler::emit_and,
2882            Size::S32,
2883            loc_a,
2884            loc_b,
2885            ret,
2886            ImmType::Logical32,
2887        )
2888    }
2889    fn emit_binop_or32(
2890        &mut self,
2891        loc_a: Location,
2892        loc_b: Location,
2893        ret: Location,
2894    ) -> Result<(), CompileError> {
2895        self.emit_relaxed_binop3(
2896            Assembler::emit_or,
2897            Size::S32,
2898            loc_a,
2899            loc_b,
2900            ret,
2901            ImmType::Logical32,
2902        )
2903    }
2904    fn emit_binop_xor32(
2905        &mut self,
2906        loc_a: Location,
2907        loc_b: Location,
2908        ret: Location,
2909    ) -> Result<(), CompileError> {
2910        self.emit_relaxed_binop3(
2911            Assembler::emit_eor,
2912            Size::S32,
2913            loc_a,
2914            loc_b,
2915            ret,
2916            ImmType::Logical32,
2917        )
2918    }
2919    fn i32_cmp_ge_s(
2920        &mut self,
2921        loc_a: Location,
2922        loc_b: Location,
2923        ret: Location,
2924    ) -> Result<(), CompileError> {
2925        self.emit_cmpop_i32_dynamic_b(Condition::Ge, loc_a, loc_b, ret)
2926    }
2927    fn i32_cmp_gt_s(
2928        &mut self,
2929        loc_a: Location,
2930        loc_b: Location,
2931        ret: Location,
2932    ) -> Result<(), CompileError> {
2933        self.emit_cmpop_i32_dynamic_b(Condition::Gt, loc_a, loc_b, ret)
2934    }
2935    fn i32_cmp_le_s(
2936        &mut self,
2937        loc_a: Location,
2938        loc_b: Location,
2939        ret: Location,
2940    ) -> Result<(), CompileError> {
2941        self.emit_cmpop_i32_dynamic_b(Condition::Le, loc_a, loc_b, ret)
2942    }
2943    fn i32_cmp_lt_s(
2944        &mut self,
2945        loc_a: Location,
2946        loc_b: Location,
2947        ret: Location,
2948    ) -> Result<(), CompileError> {
2949        self.emit_cmpop_i32_dynamic_b(Condition::Lt, loc_a, loc_b, ret)
2950    }
2951    fn i32_cmp_ge_u(
2952        &mut self,
2953        loc_a: Location,
2954        loc_b: Location,
2955        ret: Location,
2956    ) -> Result<(), CompileError> {
2957        self.emit_cmpop_i32_dynamic_b(Condition::Cs, loc_a, loc_b, ret)
2958    }
2959    fn i32_cmp_gt_u(
2960        &mut self,
2961        loc_a: Location,
2962        loc_b: Location,
2963        ret: Location,
2964    ) -> Result<(), CompileError> {
2965        self.emit_cmpop_i32_dynamic_b(Condition::Hi, loc_a, loc_b, ret)
2966    }
2967    fn i32_cmp_le_u(
2968        &mut self,
2969        loc_a: Location,
2970        loc_b: Location,
2971        ret: Location,
2972    ) -> Result<(), CompileError> {
2973        self.emit_cmpop_i32_dynamic_b(Condition::Ls, loc_a, loc_b, ret)
2974    }
2975    fn i32_cmp_lt_u(
2976        &mut self,
2977        loc_a: Location,
2978        loc_b: Location,
2979        ret: Location,
2980    ) -> Result<(), CompileError> {
2981        self.emit_cmpop_i32_dynamic_b(Condition::Cc, loc_a, loc_b, ret)
2982    }
2983    fn i32_cmp_ne(
2984        &mut self,
2985        loc_a: Location,
2986        loc_b: Location,
2987        ret: Location,
2988    ) -> Result<(), CompileError> {
2989        self.emit_cmpop_i32_dynamic_b(Condition::Ne, loc_a, loc_b, ret)
2990    }
2991    fn i32_cmp_eq(
2992        &mut self,
2993        loc_a: Location,
2994        loc_b: Location,
2995        ret: Location,
2996    ) -> Result<(), CompileError> {
2997        self.emit_cmpop_i32_dynamic_b(Condition::Eq, loc_a, loc_b, ret)
2998    }
2999    fn i32_clz(&mut self, src: Location, dst: Location) -> Result<(), CompileError> {
3000        self.emit_relaxed_binop(Assembler::emit_clz, Size::S32, src, dst, true)
3001    }
3002    fn i32_ctz(&mut self, src: Location, dst: Location) -> Result<(), CompileError> {
3003        let mut temps = vec![];
3004        let src = self.location_to_reg(Size::S32, src, &mut temps, ImmType::None, true, None)?;
3005        let dest = self.location_to_reg(Size::S32, dst, &mut temps, ImmType::None, false, None)?;
3006        self.assembler.emit_rbit(Size::S32, src, dest)?;
3007        self.assembler.emit_clz(Size::S32, dest, dest)?;
3008        if dst != dest {
3009            self.move_location(Size::S32, dest, dst)?;
3010        }
3011        for r in temps {
3012            self.release_gpr(r);
3013        }
3014        Ok(())
3015    }
3016    fn i32_popcnt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
3017        if self.has_neon {
3018            let mut temps = vec![];
3019
3020            let src_gpr =
3021                self.location_to_reg(Size::S32, loc, &mut temps, ImmType::None, true, None)?;
3022            let dst_gpr =
3023                self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3024
3025            let mut neon_temps = vec![];
3026            let neon_temp = self.acquire_temp_simd().ok_or_else(|| {
3027                CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3028            })?;
3029            neon_temps.push(neon_temp);
3030
3031            self.assembler
3032                .emit_fmov(Size::S32, src_gpr, Size::S32, Location::SIMD(neon_temp))?;
3033            self.assembler.emit_cnt(neon_temp, neon_temp)?;
3034            self.assembler.emit_addv(neon_temp, neon_temp)?;
3035            self.assembler
3036                .emit_fmov(Size::S32, Location::SIMD(neon_temp), Size::S32, dst_gpr)?;
3037
3038            if ret != dst_gpr {
3039                self.move_location(Size::S32, dst_gpr, ret)?;
3040            }
3041
3042            for r in temps {
3043                self.release_gpr(r);
3044            }
3045
3046            for r in neon_temps {
3047                self.release_simd(r);
3048            }
3049        } else {
3050            let mut temps = vec![];
3051            let src =
3052                self.location_to_reg(Size::S32, loc, &mut temps, ImmType::None, true, None)?;
3053            let dest =
3054                self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3055            let src = if src == loc {
3056                let tmp = self.acquire_temp_gpr().ok_or_else(|| {
3057                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3058                })?;
3059                temps.push(tmp);
3060                self.assembler
3061                    .emit_mov(Size::S32, src, Location::GPR(tmp))?;
3062                Location::GPR(tmp)
3063            } else {
3064                src
3065            };
3066            let tmp = {
3067                let tmp = self.acquire_temp_gpr().ok_or_else(|| {
3068                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3069                })?;
3070                temps.push(tmp);
3071                Location::GPR(tmp)
3072            };
3073            let label_loop = self.assembler.get_label();
3074            let label_exit = self.assembler.get_label();
3075            self.assembler
3076                .emit_mov(Size::S32, Location::GPR(GPR::XzrSp), dest)?; // 0 => dest
3077            self.assembler.emit_cbz_label(Size::S32, src, label_exit)?; // src==0, exit
3078            self.assembler.emit_label(label_loop)?; // loop:
3079            self.assembler
3080                .emit_add(Size::S32, dest, Location::Imm8(1), dest)?; // dest += 1
3081            self.assembler.emit_clz(Size::S32, src, tmp)?; // clz src => tmp
3082            self.assembler.emit_lsl(Size::S32, src, tmp, src)?; // src << tmp => src
3083            self.assembler
3084                .emit_lsl(Size::S32, src, Location::Imm8(1), src)?; // src << 1 => src
3085            self.assembler.emit_cbnz_label(Size::S32, src, label_loop)?; // if src!=0 goto loop
3086            self.assembler.emit_label(label_exit)?;
3087            if ret != dest {
3088                self.move_location(Size::S32, dest, ret)?;
3089            }
3090            for r in temps {
3091                self.release_gpr(r);
3092            }
3093        }
3094        Ok(())
3095    }
3096    fn i32_shl(
3097        &mut self,
3098        loc_a: Location,
3099        loc_b: Location,
3100        ret: Location,
3101    ) -> Result<(), CompileError> {
3102        self.emit_relaxed_binop3(
3103            Assembler::emit_lsl,
3104            Size::S32,
3105            loc_a,
3106            loc_b,
3107            ret,
3108            ImmType::Shift32No0,
3109        )
3110    }
3111    fn i32_shr(
3112        &mut self,
3113        loc_a: Location,
3114        loc_b: Location,
3115        ret: Location,
3116    ) -> Result<(), CompileError> {
3117        self.emit_relaxed_binop3(
3118            Assembler::emit_lsr,
3119            Size::S32,
3120            loc_a,
3121            loc_b,
3122            ret,
3123            ImmType::Shift32No0,
3124        )
3125    }
3126    fn i32_sar(
3127        &mut self,
3128        loc_a: Location,
3129        loc_b: Location,
3130        ret: Location,
3131    ) -> Result<(), CompileError> {
3132        self.emit_relaxed_binop3(
3133            Assembler::emit_asr,
3134            Size::S32,
3135            loc_a,
3136            loc_b,
3137            ret,
3138            ImmType::Shift32No0,
3139        )
3140    }
3141    fn i32_rol(
3142        &mut self,
3143        loc_a: Location,
3144        loc_b: Location,
3145        ret: Location,
3146    ) -> Result<(), CompileError> {
3147        let mut temps = vec![];
3148        let src2 = match loc_b {
3149            Location::Imm8(imm) => Location::Imm8(32 - (imm & 31)),
3150            Location::Imm32(imm) => Location::Imm8(32 - (imm & 31) as u8),
3151            Location::Imm64(imm) => Location::Imm8(32 - (imm & 31) as u8),
3152            _ => {
3153                let tmp1 = self.location_to_reg(
3154                    Size::S32,
3155                    Location::Imm32(32),
3156                    &mut temps,
3157                    ImmType::None,
3158                    true,
3159                    None,
3160                )?;
3161                let tmp2 =
3162                    self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
3163                self.assembler.emit_sub(Size::S32, tmp1, tmp2, tmp1)?;
3164                tmp1
3165            }
3166        };
3167        self.emit_relaxed_binop3(
3168            Assembler::emit_ror,
3169            Size::S32,
3170            loc_a,
3171            src2,
3172            ret,
3173            ImmType::Shift32No0,
3174        )?;
3175        for r in temps {
3176            self.release_gpr(r);
3177        }
3178        Ok(())
3179    }
3180    fn i32_ror(
3181        &mut self,
3182        loc_a: Location,
3183        loc_b: Location,
3184        ret: Location,
3185    ) -> Result<(), CompileError> {
3186        self.emit_relaxed_binop3(
3187            Assembler::emit_ror,
3188            Size::S32,
3189            loc_a,
3190            loc_b,
3191            ret,
3192            ImmType::Shift32No0,
3193        )
3194    }
3195    fn i32_load(
3196        &mut self,
3197        addr: Location,
3198        memarg: &MemArg,
3199        ret: Location,
3200        need_check: bool,
3201        imported_memories: bool,
3202        offset: i32,
3203        heap_access_oob: Label,
3204        unaligned_atomic: Label,
3205    ) -> Result<(), CompileError> {
3206        self.memory_op(
3207            addr,
3208            memarg,
3209            false,
3210            4,
3211            need_check,
3212            imported_memories,
3213            offset,
3214            heap_access_oob,
3215            unaligned_atomic,
3216            |this, addr| this.emit_relaxed_ldr32(Size::S32, ret, Location::Memory(addr, 0)),
3217        )
3218    }
3219    fn i32_load_8u(
3220        &mut self,
3221        addr: Location,
3222        memarg: &MemArg,
3223        ret: Location,
3224        need_check: bool,
3225        imported_memories: bool,
3226        offset: i32,
3227        heap_access_oob: Label,
3228        unaligned_atomic: Label,
3229    ) -> Result<(), CompileError> {
3230        self.memory_op(
3231            addr,
3232            memarg,
3233            false,
3234            1,
3235            need_check,
3236            imported_memories,
3237            offset,
3238            heap_access_oob,
3239            unaligned_atomic,
3240            |this, addr| this.emit_relaxed_ldr8(Size::S32, ret, Location::Memory(addr, 0)),
3241        )
3242    }
3243    fn i32_load_8s(
3244        &mut self,
3245        addr: Location,
3246        memarg: &MemArg,
3247        ret: Location,
3248        need_check: bool,
3249        imported_memories: bool,
3250        offset: i32,
3251        heap_access_oob: Label,
3252        unaligned_atomic: Label,
3253    ) -> Result<(), CompileError> {
3254        self.memory_op(
3255            addr,
3256            memarg,
3257            false,
3258            1,
3259            need_check,
3260            imported_memories,
3261            offset,
3262            heap_access_oob,
3263            unaligned_atomic,
3264            |this, addr| this.emit_relaxed_ldr8s(Size::S32, ret, Location::Memory(addr, 0)),
3265        )
3266    }
3267    fn i32_load_16u(
3268        &mut self,
3269        addr: Location,
3270        memarg: &MemArg,
3271        ret: Location,
3272        need_check: bool,
3273        imported_memories: bool,
3274        offset: i32,
3275        heap_access_oob: Label,
3276        unaligned_atomic: Label,
3277    ) -> Result<(), CompileError> {
3278        self.memory_op(
3279            addr,
3280            memarg,
3281            false,
3282            2,
3283            need_check,
3284            imported_memories,
3285            offset,
3286            heap_access_oob,
3287            unaligned_atomic,
3288            |this, addr| this.emit_relaxed_ldr16(Size::S32, ret, Location::Memory(addr, 0)),
3289        )
3290    }
3291    fn i32_load_16s(
3292        &mut self,
3293        addr: Location,
3294        memarg: &MemArg,
3295        ret: Location,
3296        need_check: bool,
3297        imported_memories: bool,
3298        offset: i32,
3299        heap_access_oob: Label,
3300        unaligned_atomic: Label,
3301    ) -> Result<(), CompileError> {
3302        self.memory_op(
3303            addr,
3304            memarg,
3305            false,
3306            2,
3307            need_check,
3308            imported_memories,
3309            offset,
3310            heap_access_oob,
3311            unaligned_atomic,
3312            |this, addr| this.emit_relaxed_ldr16s(Size::S32, ret, Location::Memory(addr, 0)),
3313        )
3314    }
3315    fn i32_atomic_load(
3316        &mut self,
3317        addr: Location,
3318        memarg: &MemArg,
3319        ret: Location,
3320        need_check: bool,
3321        imported_memories: bool,
3322        offset: i32,
3323        heap_access_oob: Label,
3324        unaligned_atomic: Label,
3325    ) -> Result<(), CompileError> {
3326        self.memory_op(
3327            addr,
3328            memarg,
3329            true,
3330            4,
3331            need_check,
3332            imported_memories,
3333            offset,
3334            heap_access_oob,
3335            unaligned_atomic,
3336            |this, addr| this.emit_relaxed_ldr32(Size::S32, ret, Location::Memory(addr, 0)),
3337        )
3338    }
3339    fn i32_atomic_load_8u(
3340        &mut self,
3341        addr: Location,
3342        memarg: &MemArg,
3343        ret: Location,
3344        need_check: bool,
3345        imported_memories: bool,
3346        offset: i32,
3347        heap_access_oob: Label,
3348        unaligned_atomic: Label,
3349    ) -> Result<(), CompileError> {
3350        self.memory_op(
3351            addr,
3352            memarg,
3353            true,
3354            1,
3355            need_check,
3356            imported_memories,
3357            offset,
3358            heap_access_oob,
3359            unaligned_atomic,
3360            |this, addr| this.emit_relaxed_ldr8(Size::S32, ret, Location::Memory(addr, 0)),
3361        )
3362    }
3363    fn i32_atomic_load_16u(
3364        &mut self,
3365        addr: Location,
3366        memarg: &MemArg,
3367        ret: Location,
3368        need_check: bool,
3369        imported_memories: bool,
3370        offset: i32,
3371        heap_access_oob: Label,
3372        unaligned_atomic: Label,
3373    ) -> Result<(), CompileError> {
3374        self.memory_op(
3375            addr,
3376            memarg,
3377            true,
3378            2,
3379            need_check,
3380            imported_memories,
3381            offset,
3382            heap_access_oob,
3383            unaligned_atomic,
3384            |this, addr| this.emit_relaxed_ldr16(Size::S32, ret, Location::Memory(addr, 0)),
3385        )
3386    }
3387    fn i32_save(
3388        &mut self,
3389        target_value: Location,
3390        memarg: &MemArg,
3391        target_addr: Location,
3392        need_check: bool,
3393        imported_memories: bool,
3394        offset: i32,
3395        heap_access_oob: Label,
3396        unaligned_atomic: Label,
3397    ) -> Result<(), CompileError> {
3398        self.memory_op(
3399            target_addr,
3400            memarg,
3401            false,
3402            4,
3403            need_check,
3404            imported_memories,
3405            offset,
3406            heap_access_oob,
3407            unaligned_atomic,
3408            |this, addr| this.emit_relaxed_str32(target_value, Location::Memory(addr, 0)),
3409        )
3410    }
3411    fn i32_save_8(
3412        &mut self,
3413        target_value: Location,
3414        memarg: &MemArg,
3415        target_addr: Location,
3416        need_check: bool,
3417        imported_memories: bool,
3418        offset: i32,
3419        heap_access_oob: Label,
3420        unaligned_atomic: Label,
3421    ) -> Result<(), CompileError> {
3422        self.memory_op(
3423            target_addr,
3424            memarg,
3425            false,
3426            4,
3427            need_check,
3428            imported_memories,
3429            offset,
3430            heap_access_oob,
3431            unaligned_atomic,
3432            |this, addr| this.emit_relaxed_str8(target_value, Location::Memory(addr, 0)),
3433        )
3434    }
3435    fn i32_save_16(
3436        &mut self,
3437        target_value: Location,
3438        memarg: &MemArg,
3439        target_addr: Location,
3440        need_check: bool,
3441        imported_memories: bool,
3442        offset: i32,
3443        heap_access_oob: Label,
3444        unaligned_atomic: Label,
3445    ) -> Result<(), CompileError> {
3446        self.memory_op(
3447            target_addr,
3448            memarg,
3449            false,
3450            4,
3451            need_check,
3452            imported_memories,
3453            offset,
3454            heap_access_oob,
3455            unaligned_atomic,
3456            |this, addr| this.emit_relaxed_str16(target_value, Location::Memory(addr, 0)),
3457        )
3458    }
3459    fn i32_atomic_save(
3460        &mut self,
3461        target_value: Location,
3462        memarg: &MemArg,
3463        target_addr: Location,
3464        need_check: bool,
3465        imported_memories: bool,
3466        offset: i32,
3467        heap_access_oob: Label,
3468        unaligned_atomic: Label,
3469    ) -> Result<(), CompileError> {
3470        self.memory_op(
3471            target_addr,
3472            memarg,
3473            true,
3474            4,
3475            need_check,
3476            imported_memories,
3477            offset,
3478            heap_access_oob,
3479            unaligned_atomic,
3480            |this, addr| this.emit_relaxed_str32(target_value, Location::Memory(addr, 0)),
3481        )?;
3482        self.assembler.emit_dmb()
3483    }
3484    fn i32_atomic_save_8(
3485        &mut self,
3486        target_value: Location,
3487        memarg: &MemArg,
3488        target_addr: Location,
3489        need_check: bool,
3490        imported_memories: bool,
3491        offset: i32,
3492        heap_access_oob: Label,
3493        unaligned_atomic: Label,
3494    ) -> Result<(), CompileError> {
3495        self.memory_op(
3496            target_addr,
3497            memarg,
3498            true,
3499            1,
3500            need_check,
3501            imported_memories,
3502            offset,
3503            heap_access_oob,
3504            unaligned_atomic,
3505            |this, addr| this.emit_relaxed_str8(target_value, Location::Memory(addr, 0)),
3506        )?;
3507        self.assembler.emit_dmb()
3508    }
3509    fn i32_atomic_save_16(
3510        &mut self,
3511        target_value: Location,
3512        memarg: &MemArg,
3513        target_addr: Location,
3514        need_check: bool,
3515        imported_memories: bool,
3516        offset: i32,
3517        heap_access_oob: Label,
3518        unaligned_atomic: Label,
3519    ) -> Result<(), CompileError> {
3520        self.memory_op(
3521            target_addr,
3522            memarg,
3523            true,
3524            2,
3525            need_check,
3526            imported_memories,
3527            offset,
3528            heap_access_oob,
3529            unaligned_atomic,
3530            |this, addr| this.emit_relaxed_str16(target_value, Location::Memory(addr, 0)),
3531        )?;
3532        self.assembler.emit_dmb()
3533    }
3534    // i32 atomic Add with i32
3535    fn i32_atomic_add(
3536        &mut self,
3537        loc: Location,
3538        target: Location,
3539        memarg: &MemArg,
3540        ret: Location,
3541        need_check: bool,
3542        imported_memories: bool,
3543        offset: i32,
3544        heap_access_oob: Label,
3545        unaligned_atomic: Label,
3546    ) -> Result<(), CompileError> {
3547        self.memory_op(
3548            target,
3549            memarg,
3550            true,
3551            4,
3552            need_check,
3553            imported_memories,
3554            offset,
3555            heap_access_oob,
3556            unaligned_atomic,
3557            |this, addr| {
3558                let mut temps = vec![];
3559                let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3560                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3561                })?;
3562                let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3563                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3564                })?;
3565                let dst =
3566                    this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3567                let reread = this.get_label();
3568
3569                this.emit_label(reread)?;
3570                this.assembler
3571                    .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
3572                this.emit_binop_add32(dst, loc, Location::GPR(tmp1))?;
3573                this.assembler.emit_stlxr(
3574                    Size::S32,
3575                    Location::GPR(tmp2),
3576                    Location::GPR(tmp1),
3577                    Location::GPR(addr),
3578                )?;
3579                this.assembler
3580                    .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3581                this.assembler.emit_dmb()?;
3582
3583                if dst != ret {
3584                    this.move_location(Size::S32, ret, dst)?;
3585                }
3586                for r in temps {
3587                    this.release_gpr(r);
3588                }
3589                this.release_gpr(tmp1);
3590                this.release_gpr(tmp2);
3591                Ok(())
3592            },
3593        )
3594    }
3595    // i32 atomic Add with u8
3596    fn i32_atomic_add_8u(
3597        &mut self,
3598        loc: Location,
3599        target: Location,
3600        memarg: &MemArg,
3601        ret: Location,
3602        need_check: bool,
3603        imported_memories: bool,
3604        offset: i32,
3605        heap_access_oob: Label,
3606        unaligned_atomic: Label,
3607    ) -> Result<(), CompileError> {
3608        self.memory_op(
3609            target,
3610            memarg,
3611            true,
3612            1,
3613            need_check,
3614            imported_memories,
3615            offset,
3616            heap_access_oob,
3617            unaligned_atomic,
3618            |this, addr| {
3619                let mut temps = vec![];
3620                let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3621                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3622                })?;
3623                let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3624                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3625                })?;
3626                let dst =
3627                    this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3628                let reread = this.get_label();
3629
3630                this.emit_label(reread)?;
3631                this.assembler
3632                    .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?;
3633                this.emit_binop_add32(dst, loc, Location::GPR(tmp1))?;
3634                this.assembler.emit_stlxrb(
3635                    Size::S32,
3636                    Location::GPR(tmp2),
3637                    Location::GPR(tmp1),
3638                    Location::GPR(addr),
3639                )?;
3640                this.assembler
3641                    .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3642                this.assembler.emit_dmb()?;
3643
3644                if dst != ret {
3645                    this.move_location(Size::S32, ret, dst)?;
3646                }
3647                for r in temps {
3648                    this.release_gpr(r);
3649                }
3650                this.release_gpr(tmp1);
3651                this.release_gpr(tmp2);
3652                Ok(())
3653            },
3654        )
3655    }
3656    // i32 atomic Add with u16
3657    fn i32_atomic_add_16u(
3658        &mut self,
3659        loc: Location,
3660        target: Location,
3661        memarg: &MemArg,
3662        ret: Location,
3663        need_check: bool,
3664        imported_memories: bool,
3665        offset: i32,
3666        heap_access_oob: Label,
3667        unaligned_atomic: Label,
3668    ) -> Result<(), CompileError> {
3669        self.memory_op(
3670            target,
3671            memarg,
3672            true,
3673            2,
3674            need_check,
3675            imported_memories,
3676            offset,
3677            heap_access_oob,
3678            unaligned_atomic,
3679            |this, addr| {
3680                let mut temps = vec![];
3681                let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3682                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3683                })?;
3684                let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3685                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3686                })?;
3687                let dst =
3688                    this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3689                let reread = this.get_label();
3690
3691                this.emit_label(reread)?;
3692                this.assembler
3693                    .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?;
3694                this.emit_binop_add32(dst, loc, Location::GPR(tmp1))?;
3695                this.assembler.emit_stlxrh(
3696                    Size::S32,
3697                    Location::GPR(tmp2),
3698                    Location::GPR(tmp1),
3699                    Location::GPR(addr),
3700                )?;
3701                this.assembler
3702                    .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3703                this.assembler.emit_dmb()?;
3704
3705                if dst != ret {
3706                    this.move_location(Size::S32, ret, dst)?;
3707                }
3708                for r in temps {
3709                    this.release_gpr(r);
3710                }
3711                this.release_gpr(tmp1);
3712                this.release_gpr(tmp2);
3713                Ok(())
3714            },
3715        )
3716    }
3717    // i32 atomic Sub with i32
3718    fn i32_atomic_sub(
3719        &mut self,
3720        loc: Location,
3721        target: Location,
3722        memarg: &MemArg,
3723        ret: Location,
3724        need_check: bool,
3725        imported_memories: bool,
3726        offset: i32,
3727        heap_access_oob: Label,
3728        unaligned_atomic: Label,
3729    ) -> Result<(), CompileError> {
3730        self.memory_op(
3731            target,
3732            memarg,
3733            true,
3734            4,
3735            need_check,
3736            imported_memories,
3737            offset,
3738            heap_access_oob,
3739            unaligned_atomic,
3740            |this, addr| {
3741                let mut temps = vec![];
3742                let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3743                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3744                })?;
3745                let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3746                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3747                })?;
3748                let dst =
3749                    this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3750                let reread = this.get_label();
3751
3752                this.emit_label(reread)?;
3753                this.assembler
3754                    .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
3755                this.emit_binop_sub32(dst, loc, Location::GPR(tmp1))?;
3756                this.assembler.emit_stlxr(
3757                    Size::S32,
3758                    Location::GPR(tmp2),
3759                    Location::GPR(tmp1),
3760                    Location::GPR(addr),
3761                )?;
3762                this.assembler
3763                    .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3764                this.assembler.emit_dmb()?;
3765
3766                if dst != ret {
3767                    this.move_location(Size::S32, ret, dst)?;
3768                }
3769                for r in temps {
3770                    this.release_gpr(r);
3771                }
3772                this.release_gpr(tmp1);
3773                this.release_gpr(tmp2);
3774                Ok(())
3775            },
3776        )
3777    }
3778    // i32 atomic Sub with u8
3779    fn i32_atomic_sub_8u(
3780        &mut self,
3781        loc: Location,
3782        target: Location,
3783        memarg: &MemArg,
3784        ret: Location,
3785        need_check: bool,
3786        imported_memories: bool,
3787        offset: i32,
3788        heap_access_oob: Label,
3789        unaligned_atomic: Label,
3790    ) -> Result<(), CompileError> {
3791        self.memory_op(
3792            target,
3793            memarg,
3794            true,
3795            1,
3796            need_check,
3797            imported_memories,
3798            offset,
3799            heap_access_oob,
3800            unaligned_atomic,
3801            |this, addr| {
3802                let mut temps = vec![];
3803                let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3804                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3805                })?;
3806                let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3807                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3808                })?;
3809                let dst =
3810                    this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3811                let reread = this.get_label();
3812
3813                this.emit_label(reread)?;
3814                this.assembler
3815                    .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?;
3816                this.emit_binop_sub32(dst, loc, Location::GPR(tmp1))?;
3817                this.assembler.emit_stlxrb(
3818                    Size::S32,
3819                    Location::GPR(tmp2),
3820                    Location::GPR(tmp1),
3821                    Location::GPR(addr),
3822                )?;
3823                this.assembler
3824                    .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3825                this.assembler.emit_dmb()?;
3826
3827                if dst != ret {
3828                    this.move_location(Size::S32, ret, dst)?;
3829                }
3830                for r in temps {
3831                    this.release_gpr(r);
3832                }
3833                this.release_gpr(tmp1);
3834                this.release_gpr(tmp2);
3835                Ok(())
3836            },
3837        )
3838    }
3839    // i32 atomic Sub with u16
3840    fn i32_atomic_sub_16u(
3841        &mut self,
3842        loc: Location,
3843        target: Location,
3844        memarg: &MemArg,
3845        ret: Location,
3846        need_check: bool,
3847        imported_memories: bool,
3848        offset: i32,
3849        heap_access_oob: Label,
3850        unaligned_atomic: Label,
3851    ) -> Result<(), CompileError> {
3852        self.memory_op(
3853            target,
3854            memarg,
3855            true,
3856            2,
3857            need_check,
3858            imported_memories,
3859            offset,
3860            heap_access_oob,
3861            unaligned_atomic,
3862            |this, addr| {
3863                let mut temps = vec![];
3864                let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3865                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3866                })?;
3867                let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3868                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3869                })?;
3870                let dst =
3871                    this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3872                let reread = this.get_label();
3873
3874                this.emit_label(reread)?;
3875                this.assembler
3876                    .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?;
3877                this.emit_binop_sub32(dst, loc, Location::GPR(tmp1))?;
3878                this.assembler.emit_stlxrh(
3879                    Size::S32,
3880                    Location::GPR(tmp2),
3881                    Location::GPR(tmp1),
3882                    Location::GPR(addr),
3883                )?;
3884                this.assembler
3885                    .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3886                this.assembler.emit_dmb()?;
3887
3888                if dst != ret {
3889                    this.move_location(Size::S32, ret, dst)?;
3890                }
3891                for r in temps {
3892                    this.release_gpr(r);
3893                }
3894                this.release_gpr(tmp1);
3895                this.release_gpr(tmp2);
3896                Ok(())
3897            },
3898        )
3899    }
3900    // i32 atomic And with i32
3901    fn i32_atomic_and(
3902        &mut self,
3903        loc: Location,
3904        target: Location,
3905        memarg: &MemArg,
3906        ret: Location,
3907        need_check: bool,
3908        imported_memories: bool,
3909        offset: i32,
3910        heap_access_oob: Label,
3911        unaligned_atomic: Label,
3912    ) -> Result<(), CompileError> {
3913        self.memory_op(
3914            target,
3915            memarg,
3916            true,
3917            4,
3918            need_check,
3919            imported_memories,
3920            offset,
3921            heap_access_oob,
3922            unaligned_atomic,
3923            |this, addr| {
3924                let mut temps = vec![];
3925                let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3926                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3927                })?;
3928                let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3929                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3930                })?;
3931                let dst =
3932                    this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3933                let reread = this.get_label();
3934
3935                this.emit_label(reread)?;
3936                this.assembler
3937                    .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
3938                this.emit_binop_and32(dst, loc, Location::GPR(tmp1))?;
3939                this.assembler.emit_stlxr(
3940                    Size::S32,
3941                    Location::GPR(tmp2),
3942                    Location::GPR(tmp1),
3943                    Location::GPR(addr),
3944                )?;
3945                this.assembler
3946                    .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3947                this.assembler.emit_dmb()?;
3948
3949                if dst != ret {
3950                    this.move_location(Size::S32, ret, dst)?;
3951                }
3952                for r in temps {
3953                    this.release_gpr(r);
3954                }
3955                this.release_gpr(tmp1);
3956                this.release_gpr(tmp2);
3957                Ok(())
3958            },
3959        )
3960    }
3961    // i32 atomic And with u8
3962    fn i32_atomic_and_8u(
3963        &mut self,
3964        loc: Location,
3965        target: Location,
3966        memarg: &MemArg,
3967        ret: Location,
3968        need_check: bool,
3969        imported_memories: bool,
3970        offset: i32,
3971        heap_access_oob: Label,
3972        unaligned_atomic: Label,
3973    ) -> Result<(), CompileError> {
3974        self.memory_op(
3975            target,
3976            memarg,
3977            true,
3978            1,
3979            need_check,
3980            imported_memories,
3981            offset,
3982            heap_access_oob,
3983            unaligned_atomic,
3984            |this, addr| {
3985                let mut temps = vec![];
3986                let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3987                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3988                })?;
3989                let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3990                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3991                })?;
3992                let dst =
3993                    this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3994                let reread = this.get_label();
3995
3996                this.emit_label(reread)?;
3997                this.assembler
3998                    .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?;
3999                this.emit_binop_and32(dst, loc, Location::GPR(tmp1))?;
4000                this.assembler.emit_stlxrb(
4001                    Size::S32,
4002                    Location::GPR(tmp2),
4003                    Location::GPR(tmp1),
4004                    Location::GPR(addr),
4005                )?;
4006                this.assembler
4007                    .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
4008                this.assembler.emit_dmb()?;
4009
4010                if dst != ret {
4011                    this.move_location(Size::S32, ret, dst)?;
4012                }
4013                for r in temps {
4014                    this.release_gpr(r);
4015                }
4016                this.release_gpr(tmp1);
4017                this.release_gpr(tmp2);
4018                Ok(())
4019            },
4020        )
4021    }
4022    // i32 atomic And with u16
4023    fn i32_atomic_and_16u(
4024        &mut self,
4025        loc: Location,
4026        target: Location,
4027        memarg: &MemArg,
4028        ret: Location,
4029        need_check: bool,
4030        imported_memories: bool,
4031        offset: i32,
4032        heap_access_oob: Label,
4033        unaligned_atomic: Label,
4034    ) -> Result<(), CompileError> {
4035        self.memory_op(
4036            target,
4037            memarg,
4038            true,
4039            2,
4040            need_check,
4041            imported_memories,
4042            offset,
4043            heap_access_oob,
4044            unaligned_atomic,
4045            |this, addr| {
4046                let mut temps = vec![];
4047                let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
4048                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4049                })?;
4050                let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
4051                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4052                })?;
4053                let dst =
4054                    this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4055                let reread = this.get_label();
4056
4057                this.emit_label(reread)?;
4058                this.assembler
4059                    .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?;
4060                this.emit_binop_and32(dst, loc, Location::GPR(tmp1))?;
4061                this.assembler.emit_stlxrh(
4062                    Size::S32,
4063                    Location::GPR(tmp2),
4064                    Location::GPR(tmp1),
4065                    Location::GPR(addr),
4066                )?;
4067                this.assembler
4068                    .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
4069                this.assembler.emit_dmb()?;
4070
4071                if dst != ret {
4072                    this.move_location(Size::S32, ret, dst)?;
4073                }
4074                for r in temps {
4075                    this.release_gpr(r);
4076                }
4077                this.release_gpr(tmp1);
4078                this.release_gpr(tmp2);
4079                Ok(())
4080            },
4081        )
4082    }
4083    // i32 atomic Or with i32
4084    fn i32_atomic_or(
4085        &mut self,
4086        loc: Location,
4087        target: Location,
4088        memarg: &MemArg,
4089        ret: Location,
4090        need_check: bool,
4091        imported_memories: bool,
4092        offset: i32,
4093        heap_access_oob: Label,
4094        unaligned_atomic: Label,
4095    ) -> Result<(), CompileError> {
4096        self.memory_op(
4097            target,
4098            memarg,
4099            true,
4100            4,
4101            need_check,
4102            imported_memories,
4103            offset,
4104            heap_access_oob,
4105            unaligned_atomic,
4106            |this, addr| {
4107                let mut temps = vec![];
4108                let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
4109                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4110                })?;
4111                let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
4112                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4113                })?;
4114                let dst =
4115                    this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4116                let reread = this.get_label();
4117
4118                this.emit_label(reread)?;
4119                this.assembler
4120                    .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
4121                this.emit_binop_or32(dst, loc, Location::GPR(tmp1))?;
4122                this.assembler.emit_stlxr(
4123                    Size::S32,
4124                    Location::GPR(tmp2),
4125                    Location::GPR(tmp1),
4126                    Location::GPR(addr),
4127                )?;
4128                this.assembler
4129                    .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
4130                this.assembler.emit_dmb()?;
4131
4132                if dst != ret {
4133                    this.move_location(Size::S32, ret, dst)?;
4134                }
4135                for r in temps {
4136                    this.release_gpr(r);
4137                }
4138                this.release_gpr(tmp1);
4139                this.release_gpr(tmp2);
4140                Ok(())
4141            },
4142        )
4143    }
4144    // i32 atomic Or with u8
4145    fn i32_atomic_or_8u(
4146        &mut self,
4147        loc: Location,
4148        target: Location,
4149        memarg: &MemArg,
4150        ret: Location,
4151        need_check: bool,
4152        imported_memories: bool,
4153        offset: i32,
4154        heap_access_oob: Label,
4155        unaligned_atomic: Label,
4156    ) -> Result<(), CompileError> {
4157        self.memory_op(
4158            target,
4159            memarg,
4160            true,
4161            1,
4162            need_check,
4163            imported_memories,
4164            offset,
4165            heap_access_oob,
4166            unaligned_atomic,
4167            |this, addr| {
4168                let mut temps = vec![];
4169                let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
4170                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4171                })?;
4172                let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
4173                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4174                })?;
4175                let dst =
4176                    this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4177                let reread = this.get_label();
4178
4179                this.emit_label(reread)?;
4180                this.assembler
4181                    .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?;
4182                this.emit_binop_or32(dst, loc, Location::GPR(tmp1))?;
4183                this.assembler.emit_stlxrb(
4184                    Size::S32,
4185                    Location::GPR(tmp2),
4186                    Location::GPR(tmp1),
4187                    Location::GPR(addr),
4188                )?;
4189                this.assembler
4190                    .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
4191                this.assembler.emit_dmb()?;
4192
4193                if dst != ret {
4194                    this.move_location(Size::S32, ret, dst)?;
4195                }
4196                for r in temps {
4197                    this.release_gpr(r);
4198                }
4199                this.release_gpr(tmp1);
4200                this.release_gpr(tmp2);
4201                Ok(())
4202            },
4203        )
4204    }
4205    // i32 atomic Or with u16
4206    fn i32_atomic_or_16u(
4207        &mut self,
4208        loc: Location,
4209        target: Location,
4210        memarg: &MemArg,
4211        ret: Location,
4212        need_check: bool,
4213        imported_memories: bool,
4214        offset: i32,
4215        heap_access_oob: Label,
4216        unaligned_atomic: Label,
4217    ) -> Result<(), CompileError> {
4218        self.memory_op(
4219            target,
4220            memarg,
4221            true,
4222            2,
4223            need_check,
4224            imported_memories,
4225            offset,
4226            heap_access_oob,
4227            unaligned_atomic,
4228            |this, addr| {
4229                let mut temps = vec![];
4230                let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
4231                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4232                })?;
4233                let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
4234                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4235                })?;
4236                let dst =
4237                    this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4238                let reread = this.get_label();
4239
4240                this.emit_label(reread)?;
4241                this.assembler
4242                    .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?;
4243                this.emit_binop_or32(dst, loc, Location::GPR(tmp1))?;
4244                this.assembler.emit_stlxrh(
4245                    Size::S32,
4246                    Location::GPR(tmp2),
4247                    Location::GPR(tmp1),
4248                    Location::GPR(addr),
4249                )?;
4250                this.assembler
4251                    .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
4252                this.assembler.emit_dmb()?;
4253
4254                if dst != ret {
4255                    this.move_location(Size::S32, ret, dst)?;
4256                }
4257                for r in temps {
4258                    this.release_gpr(r);
4259                }
4260                this.release_gpr(tmp1);
4261                this.release_gpr(tmp2);
4262                Ok(())
4263            },
4264        )
4265    }
4266    // i32 atomic Xor with i32
4267    fn i32_atomic_xor(
4268        &mut self,
4269        loc: Location,
4270        target: Location,
4271        memarg: &MemArg,
4272        ret: Location,
4273        need_check: bool,
4274        imported_memories: bool,
4275        offset: i32,
4276        heap_access_oob: Label,
4277        unaligned_atomic: Label,
4278    ) -> Result<(), CompileError> {
4279        self.memory_op(
4280            target,
4281            memarg,
4282            true,
4283            4,
4284            need_check,
4285            imported_memories,
4286            offset,
4287            heap_access_oob,
4288            unaligned_atomic,
4289            |this, addr| {
4290                let mut temps = vec![];
4291                let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
4292                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4293                })?;
4294                let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
4295                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4296                })?;
4297                let dst =
4298                    this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4299                let reread = this.get_label();
4300
4301                this.emit_label(reread)?;
4302                this.assembler
4303                    .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
4304                this.emit_binop_xor32(dst, loc, Location::GPR(tmp1))?;
4305                this.assembler.emit_stlxr(
4306                    Size::S32,
4307                    Location::GPR(tmp2),
4308                    Location::GPR(tmp1),
4309                    Location::GPR(addr),
4310                )?;
4311                this.assembler
4312                    .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
4313                this.assembler.emit_dmb()?;
4314
4315                if dst != ret {
4316                    this.move_location(Size::S32, ret, dst)?;
4317                }
4318                for r in temps {
4319                    this.release_gpr(r);
4320                }
4321                this.release_gpr(tmp1);
4322                this.release_gpr(tmp2);
4323                Ok(())
4324            },
4325        )
4326    }
4327    // i32 atomic Xor with u8
4328    fn i32_atomic_xor_8u(
4329        &mut self,
4330        loc: Location,
4331        target: Location,
4332        memarg: &MemArg,
4333        ret: Location,
4334        need_check: bool,
4335        imported_memories: bool,
4336        offset: i32,
4337        heap_access_oob: Label,
4338        unaligned_atomic: Label,
4339    ) -> Result<(), CompileError> {
4340        self.memory_op(
4341            target,
4342            memarg,
4343            true,
4344            1,
4345            need_check,
4346            imported_memories,
4347            offset,
4348            heap_access_oob,
4349            unaligned_atomic,
4350            |this, addr| {
4351                let mut temps = vec![];
4352                let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
4353                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4354                })?;
4355                let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
4356                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4357                })?;
4358                let dst =
4359                    this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4360                let reread = this.get_label();
4361
4362                this.emit_label(reread)?;
4363                this.assembler
4364                    .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?;
4365                this.emit_binop_xor32(dst, loc, Location::GPR(tmp1))?;
4366                this.assembler.emit_stlxrb(
4367                    Size::S32,
4368                    Location::GPR(tmp2),
4369                    Location::GPR(tmp1),
4370                    Location::GPR(addr),
4371                )?;
4372                this.assembler
4373                    .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
4374                this.assembler.emit_dmb()?;
4375
4376                if dst != ret {
4377                    this.move_location(Size::S32, ret, dst)?;
4378                }
4379                for r in temps {
4380                    this.release_gpr(r);
4381                }
4382                this.release_gpr(tmp1);
4383                this.release_gpr(tmp2);
4384                Ok(())
4385            },
4386        )
4387    }
4388    // i32 atomic Xor with u16
4389    fn i32_atomic_xor_16u(
4390        &mut self,
4391        loc: Location,
4392        target: Location,
4393        memarg: &MemArg,
4394        ret: Location,
4395        need_check: bool,
4396        imported_memories: bool,
4397        offset: i32,
4398        heap_access_oob: Label,
4399        unaligned_atomic: Label,
4400    ) -> Result<(), CompileError> {
4401        self.memory_op(
4402            target,
4403            memarg,
4404            true,
4405            2,
4406            need_check,
4407            imported_memories,
4408            offset,
4409            heap_access_oob,
4410            unaligned_atomic,
4411            |this, addr| {
4412                let mut temps = vec![];
4413                let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
4414                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4415                })?;
4416                let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
4417                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4418                })?;
4419                let dst =
4420                    this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4421                let reread = this.get_label();
4422
4423                this.emit_label(reread)?;
4424                this.assembler
4425                    .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?;
4426                this.emit_binop_xor32(dst, loc, Location::GPR(tmp1))?;
4427                this.assembler.emit_stlxrh(
4428                    Size::S32,
4429                    Location::GPR(tmp2),
4430                    Location::GPR(tmp1),
4431                    Location::GPR(addr),
4432                )?;
4433                this.assembler
4434                    .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
4435                this.assembler.emit_dmb()?;
4436
4437                if dst != ret {
4438                    this.move_location(Size::S32, ret, dst)?;
4439                }
4440                for r in temps {
4441                    this.release_gpr(r);
4442                }
4443                this.release_gpr(tmp1);
4444                this.release_gpr(tmp2);
4445                Ok(())
4446            },
4447        )
4448    }
4449    // i32 atomic Exchange with i32
4450    fn i32_atomic_xchg(
4451        &mut self,
4452        loc: Location,
4453        target: Location,
4454        memarg: &MemArg,
4455        ret: Location,
4456        need_check: bool,
4457        imported_memories: bool,
4458        offset: i32,
4459        heap_access_oob: Label,
4460        unaligned_atomic: Label,
4461    ) -> Result<(), CompileError> {
4462        self.memory_op(
4463            target,
4464            memarg,
4465            true,
4466            4,
4467            need_check,
4468            imported_memories,
4469            offset,
4470            heap_access_oob,
4471            unaligned_atomic,
4472            |this, addr| {
4473                let mut temps = vec![];
4474                let tmp = this.acquire_temp_gpr().ok_or_else(|| {
4475                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4476                })?;
4477                let dst =
4478                    this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4479                let org =
4480                    this.location_to_reg(Size::S32, loc, &mut temps, ImmType::None, false, None)?;
4481                let reread = this.get_label();
4482
4483                this.emit_label(reread)?;
4484                this.assembler
4485                    .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
4486                this.assembler.emit_stlxr(
4487                    Size::S32,
4488                    Location::GPR(tmp),
4489                    org,
4490                    Location::GPR(addr),
4491                )?;
4492                this.assembler
4493                    .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
4494                this.assembler.emit_dmb()?;
4495
4496                if dst != ret {
4497                    this.move_location(Size::S32, ret, dst)?;
4498                }
4499                for r in temps {
4500                    this.release_gpr(r);
4501                }
4502                this.release_gpr(tmp);
4503                Ok(())
4504            },
4505        )
4506    }
4507    // i32 atomic Exchange with u8
4508    fn i32_atomic_xchg_8u(
4509        &mut self,
4510        loc: Location,
4511        target: Location,
4512        memarg: &MemArg,
4513        ret: Location,
4514        need_check: bool,
4515        imported_memories: bool,
4516        offset: i32,
4517        heap_access_oob: Label,
4518        unaligned_atomic: Label,
4519    ) -> Result<(), CompileError> {
4520        self.memory_op(
4521            target,
4522            memarg,
4523            true,
4524            1,
4525            need_check,
4526            imported_memories,
4527            offset,
4528            heap_access_oob,
4529            unaligned_atomic,
4530            |this, addr| {
4531                let mut temps = vec![];
4532                let tmp = this.acquire_temp_gpr().ok_or_else(|| {
4533                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4534                })?;
4535                let dst =
4536                    this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4537                let org =
4538                    this.location_to_reg(Size::S32, loc, &mut temps, ImmType::None, false, None)?;
4539                let reread = this.get_label();
4540
4541                this.emit_label(reread)?;
4542                this.assembler
4543                    .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?;
4544                this.assembler.emit_stlxrb(
4545                    Size::S32,
4546                    Location::GPR(tmp),
4547                    org,
4548                    Location::GPR(addr),
4549                )?;
4550                this.assembler
4551                    .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
4552                this.assembler.emit_dmb()?;
4553
4554                if dst != ret {
4555                    this.move_location(Size::S32, ret, dst)?;
4556                }
4557                for r in temps {
4558                    this.release_gpr(r);
4559                }
4560                this.release_gpr(tmp);
4561                Ok(())
4562            },
4563        )
4564    }
4565    // i32 atomic Exchange with u16
4566    fn i32_atomic_xchg_16u(
4567        &mut self,
4568        loc: Location,
4569        target: Location,
4570        memarg: &MemArg,
4571        ret: Location,
4572        need_check: bool,
4573        imported_memories: bool,
4574        offset: i32,
4575        heap_access_oob: Label,
4576        unaligned_atomic: Label,
4577    ) -> Result<(), CompileError> {
4578        self.memory_op(
4579            target,
4580            memarg,
4581            true,
4582            2,
4583            need_check,
4584            imported_memories,
4585            offset,
4586            heap_access_oob,
4587            unaligned_atomic,
4588            |this, addr| {
4589                let mut temps = vec![];
4590                let tmp = this.acquire_temp_gpr().ok_or_else(|| {
4591                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4592                })?;
4593                let dst =
4594                    this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4595                let org =
4596                    this.location_to_reg(Size::S32, loc, &mut temps, ImmType::None, false, None)?;
4597                let reread = this.get_label();
4598
4599                this.emit_label(reread)?;
4600                this.assembler
4601                    .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?;
4602                this.assembler.emit_stlxrh(
4603                    Size::S32,
4604                    Location::GPR(tmp),
4605                    org,
4606                    Location::GPR(addr),
4607                )?;
4608                this.assembler
4609                    .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
4610                this.assembler.emit_dmb()?;
4611
4612                if dst != ret {
4613                    this.move_location(Size::S32, ret, dst)?;
4614                }
4615                for r in temps {
4616                    this.release_gpr(r);
4617                }
4618                this.release_gpr(tmp);
4619                Ok(())
4620            },
4621        )
4622    }
4623    // i32 atomic Exchange with i32
4624    fn i32_atomic_cmpxchg(
4625        &mut self,
4626        new: Location,
4627        cmp: Location,
4628        target: Location,
4629        memarg: &MemArg,
4630        ret: Location,
4631        need_check: bool,
4632        imported_memories: bool,
4633        offset: i32,
4634        heap_access_oob: Label,
4635        unaligned_atomic: Label,
4636    ) -> Result<(), CompileError> {
4637        self.memory_op(
4638            target,
4639            memarg,
4640            true,
4641            4,
4642            need_check,
4643            imported_memories,
4644            offset,
4645            heap_access_oob,
4646            unaligned_atomic,
4647            |this, addr| {
4648                let mut temps = vec![];
4649                let tmp = this.acquire_temp_gpr().ok_or_else(|| {
4650                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4651                })?;
4652                let dst =
4653                    this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4654                let org =
4655                    this.location_to_reg(Size::S32, new, &mut temps, ImmType::None, false, None)?;
4656                let reread = this.get_label();
4657                let nosame = this.get_label();
4658
4659                this.emit_label(reread)?;
4660                this.assembler
4661                    .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
4662                this.emit_relaxed_cmp(Size::S32, dst, cmp)?;
4663                this.assembler.emit_bcond_label(Condition::Ne, nosame)?;
4664                this.assembler.emit_stlxr(
4665                    Size::S32,
4666                    Location::GPR(tmp),
4667                    org,
4668                    Location::GPR(addr),
4669                )?;
4670                this.assembler
4671                    .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
4672                this.assembler.emit_dmb()?;
4673
4674                this.emit_label(nosame)?;
4675                if dst != ret {
4676                    this.move_location(Size::S32, ret, dst)?;
4677                }
4678                for r in temps {
4679                    this.release_gpr(r);
4680                }
4681                this.release_gpr(tmp);
4682                Ok(())
4683            },
4684        )
4685    }
4686    // i32 atomic Exchange with u8
4687    fn i32_atomic_cmpxchg_8u(
4688        &mut self,
4689        new: Location,
4690        cmp: Location,
4691        target: Location,
4692        memarg: &MemArg,
4693        ret: Location,
4694        need_check: bool,
4695        imported_memories: bool,
4696        offset: i32,
4697        heap_access_oob: Label,
4698        unaligned_atomic: Label,
4699    ) -> Result<(), CompileError> {
4700        self.memory_op(
4701            target,
4702            memarg,
4703            true,
4704            1,
4705            need_check,
4706            imported_memories,
4707            offset,
4708            heap_access_oob,
4709            unaligned_atomic,
4710            |this, addr| {
4711                let mut temps = vec![];
4712                let tmp = this.acquire_temp_gpr().ok_or_else(|| {
4713                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4714                })?;
4715                let dst =
4716                    this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4717                let org =
4718                    this.location_to_reg(Size::S32, new, &mut temps, ImmType::None, false, None)?;
4719                let reread = this.get_label();
4720                let nosame = this.get_label();
4721
4722                this.emit_label(reread)?;
4723                this.assembler
4724                    .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?;
4725                this.emit_relaxed_cmp(Size::S32, dst, cmp)?;
4726                this.assembler.emit_bcond_label(Condition::Ne, nosame)?;
4727                this.assembler.emit_stlxrb(
4728                    Size::S32,
4729                    Location::GPR(tmp),
4730                    org,
4731                    Location::GPR(addr),
4732                )?;
4733                this.assembler
4734                    .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
4735                this.assembler.emit_dmb()?;
4736
4737                this.emit_label(nosame)?;
4738                if dst != ret {
4739                    this.move_location(Size::S32, ret, dst)?;
4740                }
4741                for r in temps {
4742                    this.release_gpr(r);
4743                }
4744                this.release_gpr(tmp);
4745                Ok(())
4746            },
4747        )
4748    }
4749    // i32 atomic Exchange with u16
4750    fn i32_atomic_cmpxchg_16u(
4751        &mut self,
4752        new: Location,
4753        cmp: Location,
4754        target: Location,
4755        memarg: &MemArg,
4756        ret: Location,
4757        need_check: bool,
4758        imported_memories: bool,
4759        offset: i32,
4760        heap_access_oob: Label,
4761        unaligned_atomic: Label,
4762    ) -> Result<(), CompileError> {
4763        self.memory_op(
4764            target,
4765            memarg,
4766            true,
4767            2,
4768            need_check,
4769            imported_memories,
4770            offset,
4771            heap_access_oob,
4772            unaligned_atomic,
4773            |this, addr| {
4774                let mut temps = vec![];
4775                let tmp = this.acquire_temp_gpr().ok_or_else(|| {
4776                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4777                })?;
4778                let dst =
4779                    this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4780                let org =
4781                    this.location_to_reg(Size::S32, new, &mut temps, ImmType::None, false, None)?;
4782                let reread = this.get_label();
4783                let nosame = this.get_label();
4784
4785                this.emit_label(reread)?;
4786                this.assembler
4787                    .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?;
4788                this.emit_relaxed_cmp(Size::S32, dst, cmp)?;
4789                this.assembler.emit_bcond_label(Condition::Ne, nosame)?;
4790                this.assembler.emit_stlxrh(
4791                    Size::S32,
4792                    Location::GPR(tmp),
4793                    org,
4794                    Location::GPR(addr),
4795                )?;
4796                this.assembler
4797                    .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
4798                this.assembler.emit_dmb()?;
4799
4800                this.emit_label(nosame)?;
4801                if dst != ret {
4802                    this.move_location(Size::S32, ret, dst)?;
4803                }
4804                for r in temps {
4805                    this.release_gpr(r);
4806                }
4807                this.release_gpr(tmp);
4808                Ok(())
4809            },
4810        )
4811    }
4812
4813    fn emit_call_with_reloc(
4814        &mut self,
4815        _calling_convention: CallingConvention,
4816        reloc_target: RelocationTarget,
4817    ) -> Result<Vec<Relocation>, CompileError> {
4818        let mut relocations = vec![];
4819        let next = self.get_label();
4820        let reloc_at = self.assembler.get_offset().0;
4821        self.emit_label(next)?; // this is to be sure the current imm26 value is 0
4822        self.assembler.emit_call_label(next)?;
4823        relocations.push(Relocation {
4824            kind: RelocationKind::Arm64Call,
4825            reloc_target,
4826            offset: reloc_at as u32,
4827            addend: 0,
4828        });
4829        Ok(relocations)
4830    }
4831
4832    fn emit_binop_add64(
4833        &mut self,
4834        loc_a: Location,
4835        loc_b: Location,
4836        ret: Location,
4837    ) -> Result<(), CompileError> {
4838        self.emit_relaxed_binop3(
4839            Assembler::emit_add,
4840            Size::S64,
4841            loc_a,
4842            loc_b,
4843            ret,
4844            ImmType::Bits12,
4845        )
4846    }
4847    fn emit_binop_sub64(
4848        &mut self,
4849        loc_a: Location,
4850        loc_b: Location,
4851        ret: Location,
4852    ) -> Result<(), CompileError> {
4853        self.emit_relaxed_binop3(
4854            Assembler::emit_sub,
4855            Size::S64,
4856            loc_a,
4857            loc_b,
4858            ret,
4859            ImmType::Bits12,
4860        )
4861    }
4862    fn emit_binop_mul64(
4863        &mut self,
4864        loc_a: Location,
4865        loc_b: Location,
4866        ret: Location,
4867    ) -> Result<(), CompileError> {
4868        self.emit_relaxed_binop3(
4869            Assembler::emit_mul,
4870            Size::S64,
4871            loc_a,
4872            loc_b,
4873            ret,
4874            ImmType::None,
4875        )
4876    }
4877    fn emit_binop_udiv64(
4878        &mut self,
4879        loc_a: Location,
4880        loc_b: Location,
4881        ret: Location,
4882        integer_division_by_zero: Label,
4883        _integer_overflow: Label,
4884    ) -> Result<usize, CompileError> {
4885        let mut temps = vec![];
4886        let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4887        let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4888        let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4889
4890        self.assembler
4891            .emit_cbz_label(Size::S64, src2, integer_division_by_zero)?;
4892        let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
4893        self.assembler.emit_udiv(Size::S64, src1, src2, dest)?;
4894        if ret != dest {
4895            self.move_location(Size::S64, dest, ret)?;
4896        }
4897        for r in temps {
4898            self.release_gpr(r);
4899        }
4900        Ok(offset)
4901    }
4902    fn emit_binop_sdiv64(
4903        &mut self,
4904        loc_a: Location,
4905        loc_b: Location,
4906        ret: Location,
4907        integer_division_by_zero: Label,
4908        integer_overflow: Label,
4909    ) -> Result<usize, CompileError> {
4910        let mut temps = vec![];
4911        let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4912        let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4913        let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4914
4915        self.assembler
4916            .emit_cbz_label_far(Size::S64, src2, integer_division_by_zero)?;
4917        let label_nooverflow = self.assembler.get_label();
4918        let tmp = self.location_to_reg(
4919            Size::S64,
4920            Location::Imm64(0x8000000000000000),
4921            &mut temps,
4922            ImmType::None,
4923            true,
4924            None,
4925        )?;
4926        self.assembler.emit_cmp(Size::S64, tmp, src1)?;
4927        self.assembler
4928            .emit_bcond_label(Condition::Ne, label_nooverflow)?;
4929        self.assembler.emit_movn(Size::S64, tmp, 0)?;
4930        self.assembler.emit_cmp(Size::S64, tmp, src2)?;
4931        self.assembler
4932            .emit_bcond_label_far(Condition::Eq, integer_overflow)?;
4933        let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
4934        self.assembler.emit_label(label_nooverflow)?;
4935        self.assembler.emit_sdiv(Size::S64, src1, src2, dest)?;
4936        if ret != dest {
4937            self.move_location(Size::S64, dest, ret)?;
4938        }
4939        for r in temps {
4940            self.release_gpr(r);
4941        }
4942        Ok(offset)
4943    }
4944    fn emit_binop_urem64(
4945        &mut self,
4946        loc_a: Location,
4947        loc_b: Location,
4948        ret: Location,
4949        integer_division_by_zero: Label,
4950        _integer_overflow: Label,
4951    ) -> Result<usize, CompileError> {
4952        let mut temps = vec![];
4953        let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4954        let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4955        let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4956        let dest = if dest == src1 || dest == src2 {
4957            let tmp = self.acquire_temp_gpr().ok_or_else(|| {
4958                CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4959            })?;
4960            temps.push(tmp);
4961            self.assembler
4962                .emit_mov(Size::S32, dest, Location::GPR(tmp))?;
4963            Location::GPR(tmp)
4964        } else {
4965            dest
4966        };
4967        self.assembler
4968            .emit_cbz_label_far(Size::S64, src2, integer_division_by_zero)?;
4969        let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
4970        self.assembler.emit_udiv(Size::S64, src1, src2, dest)?;
4971        // unsigned remainder : src1 - (src1/src2)*src2
4972        self.assembler
4973            .emit_msub(Size::S64, dest, src2, src1, dest)?;
4974        if ret != dest {
4975            self.move_location(Size::S64, dest, ret)?;
4976        }
4977        for r in temps {
4978            self.release_gpr(r);
4979        }
4980        Ok(offset)
4981    }
4982    fn emit_binop_srem64(
4983        &mut self,
4984        loc_a: Location,
4985        loc_b: Location,
4986        ret: Location,
4987        integer_division_by_zero: Label,
4988        _integer_overflow: Label,
4989    ) -> Result<usize, CompileError> {
4990        let mut temps = vec![];
4991        let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4992        let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4993        let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4994        let dest = if dest == src1 || dest == src2 {
4995            let tmp = self.acquire_temp_gpr().ok_or_else(|| {
4996                CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4997            })?;
4998            temps.push(tmp);
4999            self.assembler
5000                .emit_mov(Size::S64, dest, Location::GPR(tmp))?;
5001            Location::GPR(tmp)
5002        } else {
5003            dest
5004        };
5005        self.assembler
5006            .emit_cbz_label_far(Size::S64, src2, integer_division_by_zero)?;
5007        let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
5008        self.assembler.emit_sdiv(Size::S64, src1, src2, dest)?;
5009        // unsigned remainder : src1 - (src1/src2)*src2
5010        self.assembler
5011            .emit_msub(Size::S64, dest, src2, src1, dest)?;
5012        if ret != dest {
5013            self.move_location(Size::S64, dest, ret)?;
5014        }
5015        for r in temps {
5016            self.release_gpr(r);
5017        }
5018        Ok(offset)
5019    }
5020    fn emit_binop_and64(
5021        &mut self,
5022        loc_a: Location,
5023        loc_b: Location,
5024        ret: Location,
5025    ) -> Result<(), CompileError> {
5026        self.emit_relaxed_binop3(
5027            Assembler::emit_and,
5028            Size::S64,
5029            loc_a,
5030            loc_b,
5031            ret,
5032            ImmType::Logical64,
5033        )
5034    }
5035    fn emit_binop_or64(
5036        &mut self,
5037        loc_a: Location,
5038        loc_b: Location,
5039        ret: Location,
5040    ) -> Result<(), CompileError> {
5041        self.emit_relaxed_binop3(
5042            Assembler::emit_or,
5043            Size::S64,
5044            loc_a,
5045            loc_b,
5046            ret,
5047            ImmType::Logical64,
5048        )
5049    }
5050    fn emit_binop_xor64(
5051        &mut self,
5052        loc_a: Location,
5053        loc_b: Location,
5054        ret: Location,
5055    ) -> Result<(), CompileError> {
5056        self.emit_relaxed_binop3(
5057            Assembler::emit_eor,
5058            Size::S64,
5059            loc_a,
5060            loc_b,
5061            ret,
5062            ImmType::Logical64,
5063        )
5064    }
5065    fn i64_cmp_ge_s(
5066        &mut self,
5067        loc_a: Location,
5068        loc_b: Location,
5069        ret: Location,
5070    ) -> Result<(), CompileError> {
5071        self.emit_cmpop_i64_dynamic_b(Condition::Ge, loc_a, loc_b, ret)
5072    }
5073    fn i64_cmp_gt_s(
5074        &mut self,
5075        loc_a: Location,
5076        loc_b: Location,
5077        ret: Location,
5078    ) -> Result<(), CompileError> {
5079        self.emit_cmpop_i64_dynamic_b(Condition::Gt, loc_a, loc_b, ret)
5080    }
5081    fn i64_cmp_le_s(
5082        &mut self,
5083        loc_a: Location,
5084        loc_b: Location,
5085        ret: Location,
5086    ) -> Result<(), CompileError> {
5087        self.emit_cmpop_i64_dynamic_b(Condition::Le, loc_a, loc_b, ret)
5088    }
5089    fn i64_cmp_lt_s(
5090        &mut self,
5091        loc_a: Location,
5092        loc_b: Location,
5093        ret: Location,
5094    ) -> Result<(), CompileError> {
5095        self.emit_cmpop_i64_dynamic_b(Condition::Lt, loc_a, loc_b, ret)
5096    }
5097    fn i64_cmp_ge_u(
5098        &mut self,
5099        loc_a: Location,
5100        loc_b: Location,
5101        ret: Location,
5102    ) -> Result<(), CompileError> {
5103        self.emit_cmpop_i64_dynamic_b(Condition::Cs, loc_a, loc_b, ret)
5104    }
5105    fn i64_cmp_gt_u(
5106        &mut self,
5107        loc_a: Location,
5108        loc_b: Location,
5109        ret: Location,
5110    ) -> Result<(), CompileError> {
5111        self.emit_cmpop_i64_dynamic_b(Condition::Hi, loc_a, loc_b, ret)
5112    }
5113    fn i64_cmp_le_u(
5114        &mut self,
5115        loc_a: Location,
5116        loc_b: Location,
5117        ret: Location,
5118    ) -> Result<(), CompileError> {
5119        self.emit_cmpop_i64_dynamic_b(Condition::Ls, loc_a, loc_b, ret)
5120    }
5121    fn i64_cmp_lt_u(
5122        &mut self,
5123        loc_a: Location,
5124        loc_b: Location,
5125        ret: Location,
5126    ) -> Result<(), CompileError> {
5127        self.emit_cmpop_i64_dynamic_b(Condition::Cc, loc_a, loc_b, ret)
5128    }
5129    fn i64_cmp_ne(
5130        &mut self,
5131        loc_a: Location,
5132        loc_b: Location,
5133        ret: Location,
5134    ) -> Result<(), CompileError> {
5135        self.emit_cmpop_i64_dynamic_b(Condition::Ne, loc_a, loc_b, ret)
5136    }
5137    fn i64_cmp_eq(
5138        &mut self,
5139        loc_a: Location,
5140        loc_b: Location,
5141        ret: Location,
5142    ) -> Result<(), CompileError> {
5143        self.emit_cmpop_i64_dynamic_b(Condition::Eq, loc_a, loc_b, ret)
5144    }
5145    fn i64_clz(&mut self, src: Location, dst: Location) -> Result<(), CompileError> {
5146        self.emit_relaxed_binop(Assembler::emit_clz, Size::S64, src, dst, true)
5147    }
5148    fn i64_ctz(&mut self, src: Location, dst: Location) -> Result<(), CompileError> {
5149        let mut temps = vec![];
5150        let src = self.location_to_reg(Size::S64, src, &mut temps, ImmType::None, true, None)?;
5151        let dest = self.location_to_reg(Size::S64, dst, &mut temps, ImmType::None, false, None)?;
5152        self.assembler.emit_rbit(Size::S64, src, dest)?;
5153        self.assembler.emit_clz(Size::S64, dest, dest)?;
5154        if dst != dest {
5155            self.move_location(Size::S64, dest, dst)?;
5156        }
5157        for r in temps {
5158            self.release_gpr(r);
5159        }
5160        Ok(())
5161    }
5162    fn i64_popcnt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
5163        if self.has_neon {
5164            let mut temps = vec![];
5165
5166            let src_gpr =
5167                self.location_to_reg(Size::S64, loc, &mut temps, ImmType::None, true, None)?;
5168            let dst_gpr =
5169                self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
5170
5171            let mut neon_temps = vec![];
5172            let neon_temp = self.acquire_temp_simd().ok_or_else(|| {
5173                CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5174            })?;
5175            neon_temps.push(neon_temp);
5176
5177            self.assembler
5178                .emit_fmov(Size::S64, src_gpr, Size::S64, Location::SIMD(neon_temp))?;
5179            self.assembler.emit_cnt(neon_temp, neon_temp)?;
5180            self.assembler.emit_addv(neon_temp, neon_temp)?;
5181            self.assembler
5182                .emit_fmov(Size::S64, Location::SIMD(neon_temp), Size::S64, dst_gpr)?;
5183
5184            if ret != dst_gpr {
5185                self.move_location(Size::S64, dst_gpr, ret)?;
5186            }
5187
5188            for r in temps {
5189                self.release_gpr(r);
5190            }
5191
5192            for r in neon_temps {
5193                self.release_simd(r);
5194            }
5195        } else {
5196            let mut temps = vec![];
5197            let src =
5198                self.location_to_reg(Size::S64, loc, &mut temps, ImmType::None, true, None)?;
5199            let dest =
5200                self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
5201            let src = if src == loc {
5202                let tmp = self.acquire_temp_gpr().ok_or_else(|| {
5203                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5204                })?;
5205                temps.push(tmp);
5206                self.assembler
5207                    .emit_mov(Size::S64, src, Location::GPR(tmp))?;
5208                Location::GPR(tmp)
5209            } else {
5210                src
5211            };
5212            let tmp = {
5213                let tmp = self.acquire_temp_gpr().ok_or_else(|| {
5214                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5215                })?;
5216                temps.push(tmp);
5217                Location::GPR(tmp)
5218            };
5219            let label_loop = self.assembler.get_label();
5220            let label_exit = self.assembler.get_label();
5221            self.assembler
5222                .emit_mov(Size::S32, Location::GPR(GPR::XzrSp), dest)?; // dest <= 0
5223            self.assembler.emit_cbz_label(Size::S64, src, label_exit)?; // src == 0, then goto label_exit
5224            self.assembler.emit_label(label_loop)?;
5225            self.assembler
5226                .emit_add(Size::S32, dest, Location::Imm8(1), dest)?; // dest += 1
5227            self.assembler.emit_clz(Size::S64, src, tmp)?; // clz src => tmp
5228            self.assembler.emit_lsl(Size::S64, src, tmp, src)?; // src << tmp => src
5229            self.assembler
5230                .emit_lsl(Size::S64, src, Location::Imm8(1), src)?; // src << 1 => src
5231            self.assembler.emit_cbnz_label(Size::S64, src, label_loop)?; // src != 0, then goto label_loop
5232            self.assembler.emit_label(label_exit)?;
5233            if ret != dest {
5234                self.move_location(Size::S64, dest, ret)?;
5235            }
5236            for r in temps {
5237                self.release_gpr(r);
5238            }
5239        }
5240
5241        Ok(())
5242    }
5243    fn i64_shl(
5244        &mut self,
5245        loc_a: Location,
5246        loc_b: Location,
5247        ret: Location,
5248    ) -> Result<(), CompileError> {
5249        self.emit_relaxed_binop3(
5250            Assembler::emit_lsl,
5251            Size::S64,
5252            loc_a,
5253            loc_b,
5254            ret,
5255            ImmType::Shift64No0,
5256        )
5257    }
5258    fn i64_shr(
5259        &mut self,
5260        loc_a: Location,
5261        loc_b: Location,
5262        ret: Location,
5263    ) -> Result<(), CompileError> {
5264        self.emit_relaxed_binop3(
5265            Assembler::emit_lsr,
5266            Size::S64,
5267            loc_a,
5268            loc_b,
5269            ret,
5270            ImmType::Shift64No0,
5271        )
5272    }
5273    fn i64_sar(
5274        &mut self,
5275        loc_a: Location,
5276        loc_b: Location,
5277        ret: Location,
5278    ) -> Result<(), CompileError> {
5279        self.emit_relaxed_binop3(
5280            Assembler::emit_asr,
5281            Size::S64,
5282            loc_a,
5283            loc_b,
5284            ret,
5285            ImmType::Shift64No0,
5286        )
5287    }
5288    fn i64_rol(
5289        &mut self,
5290        loc_a: Location,
5291        loc_b: Location,
5292        ret: Location,
5293    ) -> Result<(), CompileError> {
5294        // there is no ROL on ARM64. We use ROR with 64-value instead
5295        let mut temps = vec![];
5296        let src2 = match loc_b {
5297            Location::Imm8(imm) => Location::Imm8(64 - (imm & 63)),
5298            Location::Imm32(imm) => Location::Imm8(64 - (imm & 63) as u8),
5299            Location::Imm64(imm) => Location::Imm8(64 - (imm & 63) as u8),
5300            _ => {
5301                let tmp1 = self.location_to_reg(
5302                    Size::S64,
5303                    Location::Imm32(64),
5304                    &mut temps,
5305                    ImmType::None,
5306                    true,
5307                    None,
5308                )?;
5309                let tmp2 =
5310                    self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
5311                self.assembler.emit_sub(Size::S64, tmp1, tmp2, tmp1)?;
5312                tmp1
5313            }
5314        };
5315        self.emit_relaxed_binop3(
5316            Assembler::emit_ror,
5317            Size::S64,
5318            loc_a,
5319            src2,
5320            ret,
5321            ImmType::Shift64No0,
5322        )?;
5323        for r in temps {
5324            self.release_gpr(r);
5325        }
5326        Ok(())
5327    }
5328    fn i64_ror(
5329        &mut self,
5330        loc_a: Location,
5331        loc_b: Location,
5332        ret: Location,
5333    ) -> Result<(), CompileError> {
5334        self.emit_relaxed_binop3(
5335            Assembler::emit_ror,
5336            Size::S64,
5337            loc_a,
5338            loc_b,
5339            ret,
5340            ImmType::Shift64No0,
5341        )
5342    }
5343    fn i64_load(
5344        &mut self,
5345        addr: Location,
5346        memarg: &MemArg,
5347        ret: Location,
5348        need_check: bool,
5349        imported_memories: bool,
5350        offset: i32,
5351        heap_access_oob: Label,
5352        unaligned_atomic: Label,
5353    ) -> Result<(), CompileError> {
5354        self.memory_op(
5355            addr,
5356            memarg,
5357            false,
5358            8,
5359            need_check,
5360            imported_memories,
5361            offset,
5362            heap_access_oob,
5363            unaligned_atomic,
5364            |this, addr| this.emit_relaxed_ldr64(Size::S64, ret, Location::Memory(addr, 0)),
5365        )
5366    }
5367    fn i64_load_8u(
5368        &mut self,
5369        addr: Location,
5370        memarg: &MemArg,
5371        ret: Location,
5372        need_check: bool,
5373        imported_memories: bool,
5374        offset: i32,
5375        heap_access_oob: Label,
5376        unaligned_atomic: Label,
5377    ) -> Result<(), CompileError> {
5378        self.memory_op(
5379            addr,
5380            memarg,
5381            false,
5382            1,
5383            need_check,
5384            imported_memories,
5385            offset,
5386            heap_access_oob,
5387            unaligned_atomic,
5388            |this, addr| this.emit_relaxed_ldr8(Size::S64, ret, Location::Memory(addr, 0)),
5389        )
5390    }
5391    fn i64_load_8s(
5392        &mut self,
5393        addr: Location,
5394        memarg: &MemArg,
5395        ret: Location,
5396        need_check: bool,
5397        imported_memories: bool,
5398        offset: i32,
5399        heap_access_oob: Label,
5400        unaligned_atomic: Label,
5401    ) -> Result<(), CompileError> {
5402        self.memory_op(
5403            addr,
5404            memarg,
5405            false,
5406            1,
5407            need_check,
5408            imported_memories,
5409            offset,
5410            heap_access_oob,
5411            unaligned_atomic,
5412            |this, addr| this.emit_relaxed_ldr8s(Size::S64, ret, Location::Memory(addr, 0)),
5413        )
5414    }
5415    fn i64_load_16u(
5416        &mut self,
5417        addr: Location,
5418        memarg: &MemArg,
5419        ret: Location,
5420        need_check: bool,
5421        imported_memories: bool,
5422        offset: i32,
5423        heap_access_oob: Label,
5424        unaligned_atomic: Label,
5425    ) -> Result<(), CompileError> {
5426        self.memory_op(
5427            addr,
5428            memarg,
5429            false,
5430            2,
5431            need_check,
5432            imported_memories,
5433            offset,
5434            heap_access_oob,
5435            unaligned_atomic,
5436            |this, addr| this.emit_relaxed_ldr16(Size::S64, ret, Location::Memory(addr, 0)),
5437        )
5438    }
5439    fn i64_load_16s(
5440        &mut self,
5441        addr: Location,
5442        memarg: &MemArg,
5443        ret: Location,
5444        need_check: bool,
5445        imported_memories: bool,
5446        offset: i32,
5447        heap_access_oob: Label,
5448        unaligned_atomic: Label,
5449    ) -> Result<(), CompileError> {
5450        self.memory_op(
5451            addr,
5452            memarg,
5453            false,
5454            2,
5455            need_check,
5456            imported_memories,
5457            offset,
5458            heap_access_oob,
5459            unaligned_atomic,
5460            |this, addr| this.emit_relaxed_ldr16s(Size::S64, ret, Location::Memory(addr, 0)),
5461        )
5462    }
5463    fn i64_load_32u(
5464        &mut self,
5465        addr: Location,
5466        memarg: &MemArg,
5467        ret: Location,
5468        need_check: bool,
5469        imported_memories: bool,
5470        offset: i32,
5471        heap_access_oob: Label,
5472        unaligned_atomic: Label,
5473    ) -> Result<(), CompileError> {
5474        self.memory_op(
5475            addr,
5476            memarg,
5477            false,
5478            4,
5479            need_check,
5480            imported_memories,
5481            offset,
5482            heap_access_oob,
5483            unaligned_atomic,
5484            |this, addr| this.emit_relaxed_ldr32(Size::S64, ret, Location::Memory(addr, 0)),
5485        )
5486    }
5487    fn i64_load_32s(
5488        &mut self,
5489        addr: Location,
5490        memarg: &MemArg,
5491        ret: Location,
5492        need_check: bool,
5493        imported_memories: bool,
5494        offset: i32,
5495        heap_access_oob: Label,
5496        unaligned_atomic: Label,
5497    ) -> Result<(), CompileError> {
5498        self.memory_op(
5499            addr,
5500            memarg,
5501            false,
5502            4,
5503            need_check,
5504            imported_memories,
5505            offset,
5506            heap_access_oob,
5507            unaligned_atomic,
5508            |this, addr| this.emit_relaxed_ldr32s(Size::S64, ret, Location::Memory(addr, 0)),
5509        )
5510    }
5511    fn i64_atomic_load(
5512        &mut self,
5513        addr: Location,
5514        memarg: &MemArg,
5515        ret: Location,
5516        need_check: bool,
5517        imported_memories: bool,
5518        offset: i32,
5519        heap_access_oob: Label,
5520        unaligned_atomic: Label,
5521    ) -> Result<(), CompileError> {
5522        self.memory_op(
5523            addr,
5524            memarg,
5525            true,
5526            8,
5527            need_check,
5528            imported_memories,
5529            offset,
5530            heap_access_oob,
5531            unaligned_atomic,
5532            |this, addr| this.emit_relaxed_ldr64(Size::S64, ret, Location::Memory(addr, 0)),
5533        )
5534    }
5535    fn i64_atomic_load_8u(
5536        &mut self,
5537        addr: Location,
5538        memarg: &MemArg,
5539        ret: Location,
5540        need_check: bool,
5541        imported_memories: bool,
5542        offset: i32,
5543        heap_access_oob: Label,
5544        unaligned_atomic: Label,
5545    ) -> Result<(), CompileError> {
5546        self.memory_op(
5547            addr,
5548            memarg,
5549            true,
5550            1,
5551            need_check,
5552            imported_memories,
5553            offset,
5554            heap_access_oob,
5555            unaligned_atomic,
5556            |this, addr| this.emit_relaxed_ldr8(Size::S64, ret, Location::Memory(addr, 0)),
5557        )
5558    }
5559    fn i64_atomic_load_16u(
5560        &mut self,
5561        addr: Location,
5562        memarg: &MemArg,
5563        ret: Location,
5564        need_check: bool,
5565        imported_memories: bool,
5566        offset: i32,
5567        heap_access_oob: Label,
5568        unaligned_atomic: Label,
5569    ) -> Result<(), CompileError> {
5570        self.memory_op(
5571            addr,
5572            memarg,
5573            true,
5574            2,
5575            need_check,
5576            imported_memories,
5577            offset,
5578            heap_access_oob,
5579            unaligned_atomic,
5580            |this, addr| this.emit_relaxed_ldr16(Size::S64, ret, Location::Memory(addr, 0)),
5581        )
5582    }
5583    fn i64_atomic_load_32u(
5584        &mut self,
5585        addr: Location,
5586        memarg: &MemArg,
5587        ret: Location,
5588        need_check: bool,
5589        imported_memories: bool,
5590        offset: i32,
5591        heap_access_oob: Label,
5592        unaligned_atomic: Label,
5593    ) -> Result<(), CompileError> {
5594        self.memory_op(
5595            addr,
5596            memarg,
5597            true,
5598            4,
5599            need_check,
5600            imported_memories,
5601            offset,
5602            heap_access_oob,
5603            unaligned_atomic,
5604            |this, addr| this.emit_relaxed_ldr32(Size::S64, ret, Location::Memory(addr, 0)),
5605        )
5606    }
5607    fn i64_save(
5608        &mut self,
5609        target_value: Location,
5610        memarg: &MemArg,
5611        target_addr: Location,
5612        need_check: bool,
5613        imported_memories: bool,
5614        offset: i32,
5615        heap_access_oob: Label,
5616        unaligned_atomic: Label,
5617    ) -> Result<(), CompileError> {
5618        self.memory_op(
5619            target_addr,
5620            memarg,
5621            false,
5622            8,
5623            need_check,
5624            imported_memories,
5625            offset,
5626            heap_access_oob,
5627            unaligned_atomic,
5628            |this, addr| this.emit_relaxed_str64(target_value, Location::Memory(addr, 0)),
5629        )
5630    }
5631    fn i64_save_8(
5632        &mut self,
5633        target_value: Location,
5634        memarg: &MemArg,
5635        target_addr: Location,
5636        need_check: bool,
5637        imported_memories: bool,
5638        offset: i32,
5639        heap_access_oob: Label,
5640        unaligned_atomic: Label,
5641    ) -> Result<(), CompileError> {
5642        self.memory_op(
5643            target_addr,
5644            memarg,
5645            false,
5646            1,
5647            need_check,
5648            imported_memories,
5649            offset,
5650            heap_access_oob,
5651            unaligned_atomic,
5652            |this, addr| this.emit_relaxed_str8(target_value, Location::Memory(addr, 0)),
5653        )
5654    }
5655    fn i64_save_16(
5656        &mut self,
5657        target_value: Location,
5658        memarg: &MemArg,
5659        target_addr: Location,
5660        need_check: bool,
5661        imported_memories: bool,
5662        offset: i32,
5663        heap_access_oob: Label,
5664        unaligned_atomic: Label,
5665    ) -> Result<(), CompileError> {
5666        self.memory_op(
5667            target_addr,
5668            memarg,
5669            false,
5670            2,
5671            need_check,
5672            imported_memories,
5673            offset,
5674            heap_access_oob,
5675            unaligned_atomic,
5676            |this, addr| this.emit_relaxed_str16(target_value, Location::Memory(addr, 0)),
5677        )
5678    }
5679    fn i64_save_32(
5680        &mut self,
5681        target_value: Location,
5682        memarg: &MemArg,
5683        target_addr: Location,
5684        need_check: bool,
5685        imported_memories: bool,
5686        offset: i32,
5687        heap_access_oob: Label,
5688        unaligned_atomic: Label,
5689    ) -> Result<(), CompileError> {
5690        self.memory_op(
5691            target_addr,
5692            memarg,
5693            false,
5694            4,
5695            need_check,
5696            imported_memories,
5697            offset,
5698            heap_access_oob,
5699            unaligned_atomic,
5700            |this, addr| this.emit_relaxed_str32(target_value, Location::Memory(addr, 0)),
5701        )
5702    }
5703    fn i64_atomic_save(
5704        &mut self,
5705        target_value: Location,
5706        memarg: &MemArg,
5707        target_addr: Location,
5708        need_check: bool,
5709        imported_memories: bool,
5710        offset: i32,
5711        heap_access_oob: Label,
5712        unaligned_atomic: Label,
5713    ) -> Result<(), CompileError> {
5714        self.memory_op(
5715            target_addr,
5716            memarg,
5717            true,
5718            8,
5719            need_check,
5720            imported_memories,
5721            offset,
5722            heap_access_oob,
5723            unaligned_atomic,
5724            |this, addr| this.emit_relaxed_str64(target_value, Location::Memory(addr, 0)),
5725        )?;
5726        self.assembler.emit_dmb()
5727    }
5728    fn i64_atomic_save_8(
5729        &mut self,
5730        target_value: Location,
5731        memarg: &MemArg,
5732        target_addr: Location,
5733        need_check: bool,
5734        imported_memories: bool,
5735        offset: i32,
5736        heap_access_oob: Label,
5737        unaligned_atomic: Label,
5738    ) -> Result<(), CompileError> {
5739        self.memory_op(
5740            target_addr,
5741            memarg,
5742            true,
5743            1,
5744            need_check,
5745            imported_memories,
5746            offset,
5747            heap_access_oob,
5748            unaligned_atomic,
5749            |this, addr| this.emit_relaxed_str8(target_value, Location::Memory(addr, 0)),
5750        )?;
5751        self.assembler.emit_dmb()
5752    }
5753    fn i64_atomic_save_16(
5754        &mut self,
5755        target_value: Location,
5756        memarg: &MemArg,
5757        target_addr: Location,
5758        need_check: bool,
5759        imported_memories: bool,
5760        offset: i32,
5761        heap_access_oob: Label,
5762        unaligned_atomic: Label,
5763    ) -> Result<(), CompileError> {
5764        self.memory_op(
5765            target_addr,
5766            memarg,
5767            true,
5768            2,
5769            need_check,
5770            imported_memories,
5771            offset,
5772            heap_access_oob,
5773            unaligned_atomic,
5774            |this, addr| this.emit_relaxed_str16(target_value, Location::Memory(addr, 0)),
5775        )?;
5776        self.assembler.emit_dmb()
5777    }
5778    fn i64_atomic_save_32(
5779        &mut self,
5780        target_value: Location,
5781        memarg: &MemArg,
5782        target_addr: Location,
5783        need_check: bool,
5784        imported_memories: bool,
5785        offset: i32,
5786        heap_access_oob: Label,
5787        unaligned_atomic: Label,
5788    ) -> Result<(), CompileError> {
5789        self.memory_op(
5790            target_addr,
5791            memarg,
5792            true,
5793            4,
5794            need_check,
5795            imported_memories,
5796            offset,
5797            heap_access_oob,
5798            unaligned_atomic,
5799            |this, addr| this.emit_relaxed_str32(target_value, Location::Memory(addr, 0)),
5800        )?;
5801        self.assembler.emit_dmb()
5802    }
5803    // i64 atomic Add with i64
5804    fn i64_atomic_add(
5805        &mut self,
5806        loc: Location,
5807        target: Location,
5808        memarg: &MemArg,
5809        ret: Location,
5810        need_check: bool,
5811        imported_memories: bool,
5812        offset: i32,
5813        heap_access_oob: Label,
5814        unaligned_atomic: Label,
5815    ) -> Result<(), CompileError> {
5816        self.memory_op(
5817            target,
5818            memarg,
5819            true,
5820            8,
5821            need_check,
5822            imported_memories,
5823            offset,
5824            heap_access_oob,
5825            unaligned_atomic,
5826            |this, addr| {
5827                let mut temps = vec![];
5828                let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
5829                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5830                })?;
5831                let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
5832                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5833                })?;
5834                let dst =
5835                    this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
5836                let reread = this.get_label();
5837
5838                this.emit_label(reread)?;
5839                this.assembler
5840                    .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?;
5841                this.emit_binop_add64(dst, loc, Location::GPR(tmp1))?;
5842                this.assembler.emit_stlxr(
5843                    Size::S64,
5844                    Location::GPR(tmp2),
5845                    Location::GPR(tmp1),
5846                    Location::GPR(addr),
5847                )?;
5848                this.assembler
5849                    .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
5850                this.assembler.emit_dmb()?;
5851
5852                if dst != ret {
5853                    this.move_location(Size::S64, ret, dst)?;
5854                }
5855                for r in temps {
5856                    this.release_gpr(r);
5857                }
5858                this.release_gpr(tmp1);
5859                this.release_gpr(tmp2);
5860                Ok(())
5861            },
5862        )
5863    }
5864    // i64 atomic Add with u8
5865    fn i64_atomic_add_8u(
5866        &mut self,
5867        loc: Location,
5868        target: Location,
5869        memarg: &MemArg,
5870        ret: Location,
5871        need_check: bool,
5872        imported_memories: bool,
5873        offset: i32,
5874        heap_access_oob: Label,
5875        unaligned_atomic: Label,
5876    ) -> Result<(), CompileError> {
5877        self.memory_op(
5878            target,
5879            memarg,
5880            true,
5881            1,
5882            need_check,
5883            imported_memories,
5884            offset,
5885            heap_access_oob,
5886            unaligned_atomic,
5887            |this, addr| {
5888                let mut temps = vec![];
5889                let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
5890                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5891                })?;
5892                let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
5893                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5894                })?;
5895                let dst =
5896                    this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
5897                let reread = this.get_label();
5898
5899                this.emit_label(reread)?;
5900                this.assembler
5901                    .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?;
5902                this.emit_binop_add64(dst, loc, Location::GPR(tmp1))?;
5903                this.assembler.emit_stlxrb(
5904                    Size::S64,
5905                    Location::GPR(tmp2),
5906                    Location::GPR(tmp1),
5907                    Location::GPR(addr),
5908                )?;
5909                this.assembler
5910                    .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
5911                this.assembler.emit_dmb()?;
5912
5913                if dst != ret {
5914                    this.move_location(Size::S64, ret, dst)?;
5915                }
5916                for r in temps {
5917                    this.release_gpr(r);
5918                }
5919                this.release_gpr(tmp1);
5920                this.release_gpr(tmp2);
5921                Ok(())
5922            },
5923        )
5924    }
5925    // i64 atomic Add with u16
5926    fn i64_atomic_add_16u(
5927        &mut self,
5928        loc: Location,
5929        target: Location,
5930        memarg: &MemArg,
5931        ret: Location,
5932        need_check: bool,
5933        imported_memories: bool,
5934        offset: i32,
5935        heap_access_oob: Label,
5936        unaligned_atomic: Label,
5937    ) -> Result<(), CompileError> {
5938        self.memory_op(
5939            target,
5940            memarg,
5941            true,
5942            2,
5943            need_check,
5944            imported_memories,
5945            offset,
5946            heap_access_oob,
5947            unaligned_atomic,
5948            |this, addr| {
5949                let mut temps = vec![];
5950                let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
5951                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5952                })?;
5953                let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
5954                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5955                })?;
5956                let dst =
5957                    this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
5958                let reread = this.get_label();
5959
5960                this.emit_label(reread)?;
5961                this.assembler
5962                    .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?;
5963                this.emit_binop_add64(dst, loc, Location::GPR(tmp1))?;
5964                this.assembler.emit_stlxrh(
5965                    Size::S64,
5966                    Location::GPR(tmp2),
5967                    Location::GPR(tmp1),
5968                    Location::GPR(addr),
5969                )?;
5970                this.assembler
5971                    .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
5972                this.assembler.emit_dmb()?;
5973
5974                if dst != ret {
5975                    this.move_location(Size::S64, ret, dst)?;
5976                }
5977                for r in temps {
5978                    this.release_gpr(r);
5979                }
5980                this.release_gpr(tmp1);
5981                this.release_gpr(tmp2);
5982                Ok(())
5983            },
5984        )
5985    }
5986    // i64 atomic Add with u32
5987    fn i64_atomic_add_32u(
5988        &mut self,
5989        loc: Location,
5990        target: Location,
5991        memarg: &MemArg,
5992        ret: Location,
5993        need_check: bool,
5994        imported_memories: bool,
5995        offset: i32,
5996        heap_access_oob: Label,
5997        unaligned_atomic: Label,
5998    ) -> Result<(), CompileError> {
5999        self.memory_op(
6000            target,
6001            memarg,
6002            true,
6003            4,
6004            need_check,
6005            imported_memories,
6006            offset,
6007            heap_access_oob,
6008            unaligned_atomic,
6009            |this, addr| {
6010                let mut temps = vec![];
6011                let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6012                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6013                })?;
6014                let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6015                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6016                })?;
6017                let dst =
6018                    this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6019                let reread = this.get_label();
6020
6021                this.emit_label(reread)?;
6022                this.assembler
6023                    .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
6024                this.emit_binop_add64(dst, loc, Location::GPR(tmp1))?;
6025                this.assembler.emit_stlxr(
6026                    Size::S32,
6027                    Location::GPR(tmp2),
6028                    Location::GPR(tmp1),
6029                    Location::GPR(addr),
6030                )?;
6031                this.assembler
6032                    .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6033                this.assembler.emit_dmb()?;
6034
6035                if dst != ret {
6036                    this.move_location(Size::S64, ret, dst)?;
6037                }
6038                for r in temps {
6039                    this.release_gpr(r);
6040                }
6041                this.release_gpr(tmp1);
6042                this.release_gpr(tmp2);
6043                Ok(())
6044            },
6045        )
6046    }
6047    // i64 atomic Sub with i64
6048    fn i64_atomic_sub(
6049        &mut self,
6050        loc: Location,
6051        target: Location,
6052        memarg: &MemArg,
6053        ret: Location,
6054        need_check: bool,
6055        imported_memories: bool,
6056        offset: i32,
6057        heap_access_oob: Label,
6058        unaligned_atomic: Label,
6059    ) -> Result<(), CompileError> {
6060        self.memory_op(
6061            target,
6062            memarg,
6063            true,
6064            8,
6065            need_check,
6066            imported_memories,
6067            offset,
6068            heap_access_oob,
6069            unaligned_atomic,
6070            |this, addr| {
6071                let mut temps = vec![];
6072                let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6073                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6074                })?;
6075                let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6076                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6077                })?;
6078                let dst =
6079                    this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6080                let reread = this.get_label();
6081
6082                this.emit_label(reread)?;
6083                this.assembler
6084                    .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?;
6085                this.emit_binop_sub64(dst, loc, Location::GPR(tmp1))?;
6086                this.assembler.emit_stlxr(
6087                    Size::S64,
6088                    Location::GPR(tmp2),
6089                    Location::GPR(tmp1),
6090                    Location::GPR(addr),
6091                )?;
6092                this.assembler
6093                    .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6094                this.assembler.emit_dmb()?;
6095
6096                if dst != ret {
6097                    this.move_location(Size::S64, ret, dst)?;
6098                }
6099                for r in temps {
6100                    this.release_gpr(r);
6101                }
6102                this.release_gpr(tmp1);
6103                this.release_gpr(tmp2);
6104                Ok(())
6105            },
6106        )
6107    }
6108    // i64 atomic Sub with u8
6109    fn i64_atomic_sub_8u(
6110        &mut self,
6111        loc: Location,
6112        target: Location,
6113        memarg: &MemArg,
6114        ret: Location,
6115        need_check: bool,
6116        imported_memories: bool,
6117        offset: i32,
6118        heap_access_oob: Label,
6119        unaligned_atomic: Label,
6120    ) -> Result<(), CompileError> {
6121        self.memory_op(
6122            target,
6123            memarg,
6124            true,
6125            1,
6126            need_check,
6127            imported_memories,
6128            offset,
6129            heap_access_oob,
6130            unaligned_atomic,
6131            |this, addr| {
6132                let mut temps = vec![];
6133                let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6134                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6135                })?;
6136                let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6137                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6138                })?;
6139                let dst =
6140                    this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6141                let reread = this.get_label();
6142
6143                this.emit_label(reread)?;
6144                this.assembler
6145                    .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?;
6146                this.emit_binop_sub64(dst, loc, Location::GPR(tmp1))?;
6147                this.assembler.emit_stlxrb(
6148                    Size::S64,
6149                    Location::GPR(tmp2),
6150                    Location::GPR(tmp1),
6151                    Location::GPR(addr),
6152                )?;
6153                this.assembler
6154                    .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6155                this.assembler.emit_dmb()?;
6156
6157                if dst != ret {
6158                    this.move_location(Size::S64, ret, dst)?;
6159                }
6160                for r in temps {
6161                    this.release_gpr(r);
6162                }
6163                this.release_gpr(tmp1);
6164                this.release_gpr(tmp2);
6165                Ok(())
6166            },
6167        )
6168    }
6169    // i64 atomic Sub with u16
6170    fn i64_atomic_sub_16u(
6171        &mut self,
6172        loc: Location,
6173        target: Location,
6174        memarg: &MemArg,
6175        ret: Location,
6176        need_check: bool,
6177        imported_memories: bool,
6178        offset: i32,
6179        heap_access_oob: Label,
6180        unaligned_atomic: Label,
6181    ) -> Result<(), CompileError> {
6182        self.memory_op(
6183            target,
6184            memarg,
6185            true,
6186            2,
6187            need_check,
6188            imported_memories,
6189            offset,
6190            heap_access_oob,
6191            unaligned_atomic,
6192            |this, addr| {
6193                let mut temps = vec![];
6194                let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6195                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6196                })?;
6197                let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6198                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6199                })?;
6200                let dst =
6201                    this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6202                let reread = this.get_label();
6203
6204                this.emit_label(reread)?;
6205                this.assembler
6206                    .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?;
6207                this.emit_binop_sub64(dst, loc, Location::GPR(tmp1))?;
6208                this.assembler.emit_stlxrh(
6209                    Size::S64,
6210                    Location::GPR(tmp2),
6211                    Location::GPR(tmp1),
6212                    Location::GPR(addr),
6213                )?;
6214                this.assembler
6215                    .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6216                this.assembler.emit_dmb()?;
6217
6218                if dst != ret {
6219                    this.move_location(Size::S64, ret, dst)?;
6220                }
6221                for r in temps {
6222                    this.release_gpr(r);
6223                }
6224                this.release_gpr(tmp1);
6225                this.release_gpr(tmp2);
6226                Ok(())
6227            },
6228        )
6229    }
6230    // i64 atomic Sub with u32
6231    fn i64_atomic_sub_32u(
6232        &mut self,
6233        loc: Location,
6234        target: Location,
6235        memarg: &MemArg,
6236        ret: Location,
6237        need_check: bool,
6238        imported_memories: bool,
6239        offset: i32,
6240        heap_access_oob: Label,
6241        unaligned_atomic: Label,
6242    ) -> Result<(), CompileError> {
6243        self.memory_op(
6244            target,
6245            memarg,
6246            true,
6247            4,
6248            need_check,
6249            imported_memories,
6250            offset,
6251            heap_access_oob,
6252            unaligned_atomic,
6253            |this, addr| {
6254                let mut temps = vec![];
6255                let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6256                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6257                })?;
6258                let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6259                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6260                })?;
6261                let dst =
6262                    this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6263                let reread = this.get_label();
6264
6265                this.emit_label(reread)?;
6266                this.assembler
6267                    .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
6268                this.emit_binop_sub64(dst, loc, Location::GPR(tmp1))?;
6269                this.assembler.emit_stlxr(
6270                    Size::S32,
6271                    Location::GPR(tmp2),
6272                    Location::GPR(tmp1),
6273                    Location::GPR(addr),
6274                )?;
6275                this.assembler
6276                    .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6277                this.assembler.emit_dmb()?;
6278
6279                if dst != ret {
6280                    this.move_location(Size::S64, ret, dst)?;
6281                }
6282                for r in temps {
6283                    this.release_gpr(r);
6284                }
6285                this.release_gpr(tmp1);
6286                this.release_gpr(tmp2);
6287                Ok(())
6288            },
6289        )
6290    }
6291    // i64 atomic And with i64
6292    fn i64_atomic_and(
6293        &mut self,
6294        loc: Location,
6295        target: Location,
6296        memarg: &MemArg,
6297        ret: Location,
6298        need_check: bool,
6299        imported_memories: bool,
6300        offset: i32,
6301        heap_access_oob: Label,
6302        unaligned_atomic: Label,
6303    ) -> Result<(), CompileError> {
6304        self.memory_op(
6305            target,
6306            memarg,
6307            true,
6308            8,
6309            need_check,
6310            imported_memories,
6311            offset,
6312            heap_access_oob,
6313            unaligned_atomic,
6314            |this, addr| {
6315                let mut temps = vec![];
6316                let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6317                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6318                })?;
6319                let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6320                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6321                })?;
6322                let dst =
6323                    this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6324                let reread = this.get_label();
6325
6326                this.emit_label(reread)?;
6327                this.assembler
6328                    .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?;
6329                this.emit_binop_and64(dst, loc, Location::GPR(tmp1))?;
6330                this.assembler.emit_stlxr(
6331                    Size::S64,
6332                    Location::GPR(tmp2),
6333                    Location::GPR(tmp1),
6334                    Location::GPR(addr),
6335                )?;
6336                this.assembler
6337                    .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6338                this.assembler.emit_dmb()?;
6339
6340                if dst != ret {
6341                    this.move_location(Size::S64, ret, dst)?;
6342                }
6343                for r in temps {
6344                    this.release_gpr(r);
6345                }
6346                this.release_gpr(tmp1);
6347                this.release_gpr(tmp2);
6348                Ok(())
6349            },
6350        )
6351    }
6352    // i64 atomic And with u8
6353    fn i64_atomic_and_8u(
6354        &mut self,
6355        loc: Location,
6356        target: Location,
6357        memarg: &MemArg,
6358        ret: Location,
6359        need_check: bool,
6360        imported_memories: bool,
6361        offset: i32,
6362        heap_access_oob: Label,
6363        unaligned_atomic: Label,
6364    ) -> Result<(), CompileError> {
6365        self.memory_op(
6366            target,
6367            memarg,
6368            true,
6369            1,
6370            need_check,
6371            imported_memories,
6372            offset,
6373            heap_access_oob,
6374            unaligned_atomic,
6375            |this, addr| {
6376                let mut temps = vec![];
6377                let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6378                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6379                })?;
6380                let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6381                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6382                })?;
6383                let dst =
6384                    this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6385                let reread = this.get_label();
6386
6387                this.emit_label(reread)?;
6388                this.assembler
6389                    .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?;
6390                this.emit_binop_and64(dst, loc, Location::GPR(tmp1))?;
6391                this.assembler.emit_stlxrb(
6392                    Size::S64,
6393                    Location::GPR(tmp2),
6394                    Location::GPR(tmp1),
6395                    Location::GPR(addr),
6396                )?;
6397                this.assembler
6398                    .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6399                this.assembler.emit_dmb()?;
6400
6401                if dst != ret {
6402                    this.move_location(Size::S64, ret, dst)?;
6403                }
6404                for r in temps {
6405                    this.release_gpr(r);
6406                }
6407                this.release_gpr(tmp1);
6408                this.release_gpr(tmp2);
6409                Ok(())
6410            },
6411        )
6412    }
6413    // i64 atomic And with u16
6414    fn i64_atomic_and_16u(
6415        &mut self,
6416        loc: Location,
6417        target: Location,
6418        memarg: &MemArg,
6419        ret: Location,
6420        need_check: bool,
6421        imported_memories: bool,
6422        offset: i32,
6423        heap_access_oob: Label,
6424        unaligned_atomic: Label,
6425    ) -> Result<(), CompileError> {
6426        self.memory_op(
6427            target,
6428            memarg,
6429            true,
6430            2,
6431            need_check,
6432            imported_memories,
6433            offset,
6434            heap_access_oob,
6435            unaligned_atomic,
6436            |this, addr| {
6437                let mut temps = vec![];
6438                let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6439                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6440                })?;
6441                let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6442                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6443                })?;
6444                let dst =
6445                    this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6446                let reread = this.get_label();
6447
6448                this.emit_label(reread)?;
6449                this.assembler
6450                    .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?;
6451                this.emit_binop_and64(dst, loc, Location::GPR(tmp1))?;
6452                this.assembler.emit_stlxrh(
6453                    Size::S64,
6454                    Location::GPR(tmp2),
6455                    Location::GPR(tmp1),
6456                    Location::GPR(addr),
6457                )?;
6458                this.assembler
6459                    .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6460                this.assembler.emit_dmb()?;
6461
6462                if dst != ret {
6463                    this.move_location(Size::S64, ret, dst)?;
6464                }
6465                for r in temps {
6466                    this.release_gpr(r);
6467                }
6468                this.release_gpr(tmp1);
6469                this.release_gpr(tmp2);
6470                Ok(())
6471            },
6472        )
6473    }
6474    // i64 atomic And with u32
6475    fn i64_atomic_and_32u(
6476        &mut self,
6477        loc: Location,
6478        target: Location,
6479        memarg: &MemArg,
6480        ret: Location,
6481        need_check: bool,
6482        imported_memories: bool,
6483        offset: i32,
6484        heap_access_oob: Label,
6485        unaligned_atomic: Label,
6486    ) -> Result<(), CompileError> {
6487        self.memory_op(
6488            target,
6489            memarg,
6490            true,
6491            4,
6492            need_check,
6493            imported_memories,
6494            offset,
6495            heap_access_oob,
6496            unaligned_atomic,
6497            |this, addr| {
6498                let mut temps = vec![];
6499                let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6500                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6501                })?;
6502                let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6503                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6504                })?;
6505                let dst =
6506                    this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6507                let reread = this.get_label();
6508
6509                this.emit_label(reread)?;
6510                this.assembler
6511                    .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
6512                this.emit_binop_and64(dst, loc, Location::GPR(tmp1))?;
6513                this.assembler.emit_stlxr(
6514                    Size::S32,
6515                    Location::GPR(tmp2),
6516                    Location::GPR(tmp1),
6517                    Location::GPR(addr),
6518                )?;
6519                this.assembler
6520                    .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6521                this.assembler.emit_dmb()?;
6522
6523                if dst != ret {
6524                    this.move_location(Size::S64, ret, dst)?;
6525                }
6526                for r in temps {
6527                    this.release_gpr(r);
6528                }
6529                this.release_gpr(tmp1);
6530                this.release_gpr(tmp2);
6531                Ok(())
6532            },
6533        )
6534    }
6535    // i64 atomic Or with i64
6536    fn i64_atomic_or(
6537        &mut self,
6538        loc: Location,
6539        target: Location,
6540        memarg: &MemArg,
6541        ret: Location,
6542        need_check: bool,
6543        imported_memories: bool,
6544        offset: i32,
6545        heap_access_oob: Label,
6546        unaligned_atomic: Label,
6547    ) -> Result<(), CompileError> {
6548        self.memory_op(
6549            target,
6550            memarg,
6551            true,
6552            8,
6553            need_check,
6554            imported_memories,
6555            offset,
6556            heap_access_oob,
6557            unaligned_atomic,
6558            |this, addr| {
6559                let mut temps = vec![];
6560                let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6561                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6562                })?;
6563                let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6564                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6565                })?;
6566                let dst =
6567                    this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6568                let reread = this.get_label();
6569
6570                this.emit_label(reread)?;
6571                this.assembler
6572                    .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?;
6573                this.emit_binop_or64(dst, loc, Location::GPR(tmp1))?;
6574                this.assembler.emit_stlxr(
6575                    Size::S64,
6576                    Location::GPR(tmp2),
6577                    Location::GPR(tmp1),
6578                    Location::GPR(addr),
6579                )?;
6580                this.assembler
6581                    .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6582                this.assembler.emit_dmb()?;
6583
6584                if dst != ret {
6585                    this.move_location(Size::S64, ret, dst)?;
6586                }
6587                for r in temps {
6588                    this.release_gpr(r);
6589                }
6590                this.release_gpr(tmp1);
6591                this.release_gpr(tmp2);
6592                Ok(())
6593            },
6594        )
6595    }
6596    // i64 atomic Or with u8
6597    fn i64_atomic_or_8u(
6598        &mut self,
6599        loc: Location,
6600        target: Location,
6601        memarg: &MemArg,
6602        ret: Location,
6603        need_check: bool,
6604        imported_memories: bool,
6605        offset: i32,
6606        heap_access_oob: Label,
6607        unaligned_atomic: Label,
6608    ) -> Result<(), CompileError> {
6609        self.memory_op(
6610            target,
6611            memarg,
6612            true,
6613            1,
6614            need_check,
6615            imported_memories,
6616            offset,
6617            heap_access_oob,
6618            unaligned_atomic,
6619            |this, addr| {
6620                let mut temps = vec![];
6621                let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6622                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6623                })?;
6624                let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6625                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6626                })?;
6627                let dst =
6628                    this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6629                let reread = this.get_label();
6630
6631                this.emit_label(reread)?;
6632                this.assembler
6633                    .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?;
6634                this.emit_binop_or64(dst, loc, Location::GPR(tmp1))?;
6635                this.assembler.emit_stlxrb(
6636                    Size::S64,
6637                    Location::GPR(tmp2),
6638                    Location::GPR(tmp1),
6639                    Location::GPR(addr),
6640                )?;
6641                this.assembler
6642                    .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6643                this.assembler.emit_dmb()?;
6644
6645                if dst != ret {
6646                    this.move_location(Size::S64, ret, dst)?;
6647                }
6648                for r in temps {
6649                    this.release_gpr(r);
6650                }
6651                this.release_gpr(tmp1);
6652                this.release_gpr(tmp2);
6653                Ok(())
6654            },
6655        )
6656    }
6657    // i64 atomic Or with u16
6658    fn i64_atomic_or_16u(
6659        &mut self,
6660        loc: Location,
6661        target: Location,
6662        memarg: &MemArg,
6663        ret: Location,
6664        need_check: bool,
6665        imported_memories: bool,
6666        offset: i32,
6667        heap_access_oob: Label,
6668        unaligned_atomic: Label,
6669    ) -> Result<(), CompileError> {
6670        self.memory_op(
6671            target,
6672            memarg,
6673            true,
6674            2,
6675            need_check,
6676            imported_memories,
6677            offset,
6678            heap_access_oob,
6679            unaligned_atomic,
6680            |this, addr| {
6681                let mut temps = vec![];
6682                let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6683                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6684                })?;
6685                let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6686                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6687                })?;
6688                let dst =
6689                    this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6690                let reread = this.get_label();
6691
6692                this.emit_label(reread)?;
6693                this.assembler
6694                    .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?;
6695                this.emit_binop_or64(dst, loc, Location::GPR(tmp1))?;
6696                this.assembler.emit_stlxrh(
6697                    Size::S64,
6698                    Location::GPR(tmp2),
6699                    Location::GPR(tmp1),
6700                    Location::GPR(addr),
6701                )?;
6702                this.assembler
6703                    .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6704                this.assembler.emit_dmb()?;
6705
6706                if dst != ret {
6707                    this.move_location(Size::S64, ret, dst)?;
6708                }
6709                for r in temps {
6710                    this.release_gpr(r);
6711                }
6712                this.release_gpr(tmp1);
6713                this.release_gpr(tmp2);
6714                Ok(())
6715            },
6716        )
6717    }
6718    // i64 atomic Or with u32
6719    fn i64_atomic_or_32u(
6720        &mut self,
6721        loc: Location,
6722        target: Location,
6723        memarg: &MemArg,
6724        ret: Location,
6725        need_check: bool,
6726        imported_memories: bool,
6727        offset: i32,
6728        heap_access_oob: Label,
6729        unaligned_atomic: Label,
6730    ) -> Result<(), CompileError> {
6731        self.memory_op(
6732            target,
6733            memarg,
6734            true,
6735            4,
6736            need_check,
6737            imported_memories,
6738            offset,
6739            heap_access_oob,
6740            unaligned_atomic,
6741            |this, addr| {
6742                let mut temps = vec![];
6743                let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6744                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6745                })?;
6746                let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6747                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6748                })?;
6749                let dst =
6750                    this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6751                let reread = this.get_label();
6752
6753                this.emit_label(reread)?;
6754                this.assembler
6755                    .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
6756                this.emit_binop_or64(dst, loc, Location::GPR(tmp1))?;
6757                this.assembler.emit_stlxr(
6758                    Size::S32,
6759                    Location::GPR(tmp2),
6760                    Location::GPR(tmp1),
6761                    Location::GPR(addr),
6762                )?;
6763                this.assembler
6764                    .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6765                this.assembler.emit_dmb()?;
6766
6767                if dst != ret {
6768                    this.move_location(Size::S64, ret, dst)?;
6769                }
6770                for r in temps {
6771                    this.release_gpr(r);
6772                }
6773                this.release_gpr(tmp1);
6774                this.release_gpr(tmp2);
6775                Ok(())
6776            },
6777        )
6778    }
6779    // i64 atomic Xor with i64
6780    fn i64_atomic_xor(
6781        &mut self,
6782        loc: Location,
6783        target: Location,
6784        memarg: &MemArg,
6785        ret: Location,
6786        need_check: bool,
6787        imported_memories: bool,
6788        offset: i32,
6789        heap_access_oob: Label,
6790        unaligned_atomic: Label,
6791    ) -> Result<(), CompileError> {
6792        self.memory_op(
6793            target,
6794            memarg,
6795            true,
6796            8,
6797            need_check,
6798            imported_memories,
6799            offset,
6800            heap_access_oob,
6801            unaligned_atomic,
6802            |this, addr| {
6803                let mut temps = vec![];
6804                let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6805                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6806                })?;
6807                let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6808                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6809                })?;
6810                let dst =
6811                    this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6812                let reread = this.get_label();
6813
6814                this.emit_label(reread)?;
6815                this.assembler
6816                    .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?;
6817                this.emit_binop_xor64(dst, loc, Location::GPR(tmp1))?;
6818                this.assembler.emit_stlxr(
6819                    Size::S64,
6820                    Location::GPR(tmp2),
6821                    Location::GPR(tmp1),
6822                    Location::GPR(addr),
6823                )?;
6824                this.assembler
6825                    .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6826                this.assembler.emit_dmb()?;
6827
6828                if dst != ret {
6829                    this.move_location(Size::S64, ret, dst)?;
6830                }
6831                for r in temps {
6832                    this.release_gpr(r);
6833                }
6834                this.release_gpr(tmp1);
6835                this.release_gpr(tmp2);
6836                Ok(())
6837            },
6838        )
6839    }
6840    // i64 atomic Xor with u8
6841    fn i64_atomic_xor_8u(
6842        &mut self,
6843        loc: Location,
6844        target: Location,
6845        memarg: &MemArg,
6846        ret: Location,
6847        need_check: bool,
6848        imported_memories: bool,
6849        offset: i32,
6850        heap_access_oob: Label,
6851        unaligned_atomic: Label,
6852    ) -> Result<(), CompileError> {
6853        self.memory_op(
6854            target,
6855            memarg,
6856            true,
6857            1,
6858            need_check,
6859            imported_memories,
6860            offset,
6861            heap_access_oob,
6862            unaligned_atomic,
6863            |this, addr| {
6864                let mut temps = vec![];
6865                let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6866                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6867                })?;
6868                let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6869                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6870                })?;
6871                let dst =
6872                    this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6873                let reread = this.get_label();
6874
6875                this.emit_label(reread)?;
6876                this.assembler
6877                    .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?;
6878                this.emit_binop_xor64(dst, loc, Location::GPR(tmp1))?;
6879                this.assembler.emit_stlxrb(
6880                    Size::S64,
6881                    Location::GPR(tmp2),
6882                    Location::GPR(tmp1),
6883                    Location::GPR(addr),
6884                )?;
6885                this.assembler
6886                    .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6887                this.assembler.emit_dmb()?;
6888
6889                if dst != ret {
6890                    this.move_location(Size::S64, ret, dst)?;
6891                }
6892                for r in temps {
6893                    this.release_gpr(r);
6894                }
6895                this.release_gpr(tmp1);
6896                this.release_gpr(tmp2);
6897                Ok(())
6898            },
6899        )
6900    }
6901    // i64 atomic Xor with u16
6902    fn i64_atomic_xor_16u(
6903        &mut self,
6904        loc: Location,
6905        target: Location,
6906        memarg: &MemArg,
6907        ret: Location,
6908        need_check: bool,
6909        imported_memories: bool,
6910        offset: i32,
6911        heap_access_oob: Label,
6912        unaligned_atomic: Label,
6913    ) -> Result<(), CompileError> {
6914        self.memory_op(
6915            target,
6916            memarg,
6917            true,
6918            2,
6919            need_check,
6920            imported_memories,
6921            offset,
6922            heap_access_oob,
6923            unaligned_atomic,
6924            |this, addr| {
6925                let mut temps = vec![];
6926                let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6927                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6928                })?;
6929                let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6930                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6931                })?;
6932                let dst =
6933                    this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6934                let reread = this.get_label();
6935
6936                this.emit_label(reread)?;
6937                this.assembler
6938                    .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?;
6939                this.emit_binop_xor64(dst, loc, Location::GPR(tmp1))?;
6940                this.assembler.emit_stlxrh(
6941                    Size::S64,
6942                    Location::GPR(tmp2),
6943                    Location::GPR(tmp1),
6944                    Location::GPR(addr),
6945                )?;
6946                this.assembler
6947                    .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6948                this.assembler.emit_dmb()?;
6949
6950                if dst != ret {
6951                    this.move_location(Size::S64, ret, dst)?;
6952                }
6953                for r in temps {
6954                    this.release_gpr(r);
6955                }
6956                this.release_gpr(tmp1);
6957                this.release_gpr(tmp2);
6958                Ok(())
6959            },
6960        )
6961    }
6962    // i64 atomic Xor with u32
6963    fn i64_atomic_xor_32u(
6964        &mut self,
6965        loc: Location,
6966        target: Location,
6967        memarg: &MemArg,
6968        ret: Location,
6969        need_check: bool,
6970        imported_memories: bool,
6971        offset: i32,
6972        heap_access_oob: Label,
6973        unaligned_atomic: Label,
6974    ) -> Result<(), CompileError> {
6975        self.memory_op(
6976            target,
6977            memarg,
6978            true,
6979            4,
6980            need_check,
6981            imported_memories,
6982            offset,
6983            heap_access_oob,
6984            unaligned_atomic,
6985            |this, addr| {
6986                let mut temps = vec![];
6987                let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6988                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6989                })?;
6990                let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6991                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6992                })?;
6993                let dst =
6994                    this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6995                let reread = this.get_label();
6996
6997                this.emit_label(reread)?;
6998                this.assembler
6999                    .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
7000                this.emit_binop_xor64(dst, loc, Location::GPR(tmp1))?;
7001                this.assembler.emit_stlxr(
7002                    Size::S32,
7003                    Location::GPR(tmp2),
7004                    Location::GPR(tmp1),
7005                    Location::GPR(addr),
7006                )?;
7007                this.assembler
7008                    .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
7009                this.assembler.emit_dmb()?;
7010
7011                if dst != ret {
7012                    this.move_location(Size::S64, ret, dst)?;
7013                }
7014                for r in temps {
7015                    this.release_gpr(r);
7016                }
7017                this.release_gpr(tmp1);
7018                this.release_gpr(tmp2);
7019                Ok(())
7020            },
7021        )
7022    }
7023    // i64 atomic Exchange with i64
7024    fn i64_atomic_xchg(
7025        &mut self,
7026        loc: Location,
7027        target: Location,
7028        memarg: &MemArg,
7029        ret: Location,
7030        need_check: bool,
7031        imported_memories: bool,
7032        offset: i32,
7033        heap_access_oob: Label,
7034        unaligned_atomic: Label,
7035    ) -> Result<(), CompileError> {
7036        self.memory_op(
7037            target,
7038            memarg,
7039            true,
7040            8,
7041            need_check,
7042            imported_memories,
7043            offset,
7044            heap_access_oob,
7045            unaligned_atomic,
7046            |this, addr| {
7047                let mut temps = vec![];
7048                let tmp = this.acquire_temp_gpr().ok_or_else(|| {
7049                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7050                })?;
7051                let dst =
7052                    this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7053                let org =
7054                    this.location_to_reg(Size::S64, loc, &mut temps, ImmType::None, false, None)?;
7055                let reread = this.get_label();
7056
7057                this.emit_label(reread)?;
7058                this.assembler
7059                    .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?;
7060                this.assembler.emit_stlxr(
7061                    Size::S64,
7062                    Location::GPR(tmp),
7063                    org,
7064                    Location::GPR(addr),
7065                )?;
7066                this.assembler
7067                    .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7068                this.assembler.emit_dmb()?;
7069
7070                if dst != ret {
7071                    this.move_location(Size::S64, ret, dst)?;
7072                }
7073                for r in temps {
7074                    this.release_gpr(r);
7075                }
7076                this.release_gpr(tmp);
7077                Ok(())
7078            },
7079        )
7080    }
7081    // i64 atomic Exchange with u8
7082    fn i64_atomic_xchg_8u(
7083        &mut self,
7084        loc: Location,
7085        target: Location,
7086        memarg: &MemArg,
7087        ret: Location,
7088        need_check: bool,
7089        imported_memories: bool,
7090        offset: i32,
7091        heap_access_oob: Label,
7092        unaligned_atomic: Label,
7093    ) -> Result<(), CompileError> {
7094        self.memory_op(
7095            target,
7096            memarg,
7097            true,
7098            1,
7099            need_check,
7100            imported_memories,
7101            offset,
7102            heap_access_oob,
7103            unaligned_atomic,
7104            |this, addr| {
7105                let mut temps = vec![];
7106                let tmp = this.acquire_temp_gpr().ok_or_else(|| {
7107                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7108                })?;
7109                let dst =
7110                    this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7111                let org =
7112                    this.location_to_reg(Size::S64, loc, &mut temps, ImmType::None, false, None)?;
7113                let reread = this.get_label();
7114
7115                this.emit_label(reread)?;
7116                this.assembler
7117                    .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?;
7118                this.assembler.emit_stlxrb(
7119                    Size::S64,
7120                    Location::GPR(tmp),
7121                    org,
7122                    Location::GPR(addr),
7123                )?;
7124                this.assembler
7125                    .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7126                this.assembler.emit_dmb()?;
7127
7128                if dst != ret {
7129                    this.move_location(Size::S64, ret, dst)?;
7130                }
7131                for r in temps {
7132                    this.release_gpr(r);
7133                }
7134                this.release_gpr(tmp);
7135                Ok(())
7136            },
7137        )
7138    }
7139    // i64 atomic Exchange with u16
7140    fn i64_atomic_xchg_16u(
7141        &mut self,
7142        loc: Location,
7143        target: Location,
7144        memarg: &MemArg,
7145        ret: Location,
7146        need_check: bool,
7147        imported_memories: bool,
7148        offset: i32,
7149        heap_access_oob: Label,
7150        unaligned_atomic: Label,
7151    ) -> Result<(), CompileError> {
7152        self.memory_op(
7153            target,
7154            memarg,
7155            true,
7156            2,
7157            need_check,
7158            imported_memories,
7159            offset,
7160            heap_access_oob,
7161            unaligned_atomic,
7162            |this, addr| {
7163                let mut temps = vec![];
7164                let tmp = this.acquire_temp_gpr().ok_or_else(|| {
7165                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7166                })?;
7167                let dst =
7168                    this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7169                let org =
7170                    this.location_to_reg(Size::S64, loc, &mut temps, ImmType::None, false, None)?;
7171                let reread = this.get_label();
7172
7173                this.emit_label(reread)?;
7174                this.assembler
7175                    .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?;
7176                this.assembler.emit_stlxrh(
7177                    Size::S64,
7178                    Location::GPR(tmp),
7179                    org,
7180                    Location::GPR(addr),
7181                )?;
7182                this.assembler
7183                    .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7184                this.assembler.emit_dmb()?;
7185
7186                if dst != ret {
7187                    this.move_location(Size::S64, ret, dst)?;
7188                }
7189                for r in temps {
7190                    this.release_gpr(r);
7191                }
7192                this.release_gpr(tmp);
7193                Ok(())
7194            },
7195        )
7196    }
7197    // i64 atomic Exchange with u32
7198    fn i64_atomic_xchg_32u(
7199        &mut self,
7200        loc: Location,
7201        target: Location,
7202        memarg: &MemArg,
7203        ret: Location,
7204        need_check: bool,
7205        imported_memories: bool,
7206        offset: i32,
7207        heap_access_oob: Label,
7208        unaligned_atomic: Label,
7209    ) -> Result<(), CompileError> {
7210        self.memory_op(
7211            target,
7212            memarg,
7213            true,
7214            4,
7215            need_check,
7216            imported_memories,
7217            offset,
7218            heap_access_oob,
7219            unaligned_atomic,
7220            |this, addr| {
7221                let mut temps = vec![];
7222                let tmp = this.acquire_temp_gpr().ok_or_else(|| {
7223                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7224                })?;
7225                let dst =
7226                    this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7227                let org =
7228                    this.location_to_reg(Size::S64, loc, &mut temps, ImmType::None, false, None)?;
7229                let reread = this.get_label();
7230
7231                this.emit_label(reread)?;
7232                this.assembler
7233                    .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
7234                this.assembler.emit_stlxr(
7235                    Size::S32,
7236                    Location::GPR(tmp),
7237                    org,
7238                    Location::GPR(addr),
7239                )?;
7240                this.assembler
7241                    .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7242                this.assembler.emit_dmb()?;
7243
7244                if dst != ret {
7245                    this.move_location(Size::S64, ret, dst)?;
7246                }
7247                for r in temps {
7248                    this.release_gpr(r);
7249                }
7250                this.release_gpr(tmp);
7251                Ok(())
7252            },
7253        )
7254    }
7255    // i64 atomic Exchange with i64
7256    fn i64_atomic_cmpxchg(
7257        &mut self,
7258        new: Location,
7259        cmp: Location,
7260        target: Location,
7261        memarg: &MemArg,
7262        ret: Location,
7263        need_check: bool,
7264        imported_memories: bool,
7265        offset: i32,
7266        heap_access_oob: Label,
7267        unaligned_atomic: Label,
7268    ) -> Result<(), CompileError> {
7269        self.memory_op(
7270            target,
7271            memarg,
7272            true,
7273            8,
7274            need_check,
7275            imported_memories,
7276            offset,
7277            heap_access_oob,
7278            unaligned_atomic,
7279            |this, addr| {
7280                let mut temps = vec![];
7281                let tmp = this.acquire_temp_gpr().ok_or_else(|| {
7282                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7283                })?;
7284                let dst =
7285                    this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7286                let org =
7287                    this.location_to_reg(Size::S64, new, &mut temps, ImmType::None, false, None)?;
7288                let reread = this.get_label();
7289                let nosame = this.get_label();
7290
7291                this.emit_label(reread)?;
7292                this.assembler
7293                    .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?;
7294                this.emit_relaxed_cmp(Size::S64, dst, cmp)?;
7295                this.assembler.emit_bcond_label(Condition::Ne, nosame)?;
7296                this.assembler.emit_stlxr(
7297                    Size::S64,
7298                    Location::GPR(tmp),
7299                    org,
7300                    Location::GPR(addr),
7301                )?;
7302                this.assembler
7303                    .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7304                this.assembler.emit_dmb()?;
7305
7306                this.emit_label(nosame)?;
7307                if dst != ret {
7308                    this.move_location(Size::S64, ret, dst)?;
7309                }
7310                for r in temps {
7311                    this.release_gpr(r);
7312                }
7313                this.release_gpr(tmp);
7314                Ok(())
7315            },
7316        )
7317    }
7318    // i64 atomic Exchange with u8
7319    fn i64_atomic_cmpxchg_8u(
7320        &mut self,
7321        new: Location,
7322        cmp: Location,
7323        target: Location,
7324        memarg: &MemArg,
7325        ret: Location,
7326        need_check: bool,
7327        imported_memories: bool,
7328        offset: i32,
7329        heap_access_oob: Label,
7330        unaligned_atomic: Label,
7331    ) -> Result<(), CompileError> {
7332        self.memory_op(
7333            target,
7334            memarg,
7335            true,
7336            1,
7337            need_check,
7338            imported_memories,
7339            offset,
7340            heap_access_oob,
7341            unaligned_atomic,
7342            |this, addr| {
7343                let mut temps = vec![];
7344                let tmp = this.acquire_temp_gpr().ok_or_else(|| {
7345                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7346                })?;
7347                let dst =
7348                    this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7349                let org =
7350                    this.location_to_reg(Size::S64, new, &mut temps, ImmType::None, false, None)?;
7351                let reread = this.get_label();
7352                let nosame = this.get_label();
7353
7354                this.emit_label(reread)?;
7355                this.assembler
7356                    .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?;
7357                this.emit_relaxed_cmp(Size::S64, dst, cmp)?;
7358                this.assembler.emit_bcond_label(Condition::Ne, nosame)?;
7359                this.assembler.emit_stlxrb(
7360                    Size::S64,
7361                    Location::GPR(tmp),
7362                    org,
7363                    Location::GPR(addr),
7364                )?;
7365                this.assembler
7366                    .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7367                this.assembler.emit_dmb()?;
7368
7369                this.emit_label(nosame)?;
7370                if dst != ret {
7371                    this.move_location(Size::S64, ret, dst)?;
7372                }
7373                for r in temps {
7374                    this.release_gpr(r);
7375                }
7376                this.release_gpr(tmp);
7377                Ok(())
7378            },
7379        )
7380    }
7381    // i64 atomic Exchange with u16
7382    fn i64_atomic_cmpxchg_16u(
7383        &mut self,
7384        new: Location,
7385        cmp: Location,
7386        target: Location,
7387        memarg: &MemArg,
7388        ret: Location,
7389        need_check: bool,
7390        imported_memories: bool,
7391        offset: i32,
7392        heap_access_oob: Label,
7393        unaligned_atomic: Label,
7394    ) -> Result<(), CompileError> {
7395        self.memory_op(
7396            target,
7397            memarg,
7398            true,
7399            2,
7400            need_check,
7401            imported_memories,
7402            offset,
7403            heap_access_oob,
7404            unaligned_atomic,
7405            |this, addr| {
7406                let mut temps = vec![];
7407                let tmp = this.acquire_temp_gpr().ok_or_else(|| {
7408                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7409                })?;
7410                let dst =
7411                    this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7412                let org =
7413                    this.location_to_reg(Size::S64, new, &mut temps, ImmType::None, false, None)?;
7414                let reread = this.get_label();
7415                let nosame = this.get_label();
7416
7417                this.emit_label(reread)?;
7418                this.assembler
7419                    .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?;
7420                this.emit_relaxed_cmp(Size::S64, dst, cmp)?;
7421                this.assembler.emit_bcond_label(Condition::Ne, nosame)?;
7422                this.assembler.emit_stlxrh(
7423                    Size::S64,
7424                    Location::GPR(tmp),
7425                    org,
7426                    Location::GPR(addr),
7427                )?;
7428                this.assembler
7429                    .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7430                this.assembler.emit_dmb()?;
7431
7432                this.emit_label(nosame)?;
7433                if dst != ret {
7434                    this.move_location(Size::S64, ret, dst)?;
7435                }
7436                for r in temps {
7437                    this.release_gpr(r);
7438                }
7439                this.release_gpr(tmp);
7440                Ok(())
7441            },
7442        )
7443    }
7444    // i64 atomic Exchange with u32
7445    fn i64_atomic_cmpxchg_32u(
7446        &mut self,
7447        new: Location,
7448        cmp: Location,
7449        target: Location,
7450        memarg: &MemArg,
7451        ret: Location,
7452        need_check: bool,
7453        imported_memories: bool,
7454        offset: i32,
7455        heap_access_oob: Label,
7456        unaligned_atomic: Label,
7457    ) -> Result<(), CompileError> {
7458        self.memory_op(
7459            target,
7460            memarg,
7461            true,
7462            4,
7463            need_check,
7464            imported_memories,
7465            offset,
7466            heap_access_oob,
7467            unaligned_atomic,
7468            |this, addr| {
7469                let mut temps = vec![];
7470                let tmp = this.acquire_temp_gpr().ok_or_else(|| {
7471                    CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7472                })?;
7473                let dst =
7474                    this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7475                let org =
7476                    this.location_to_reg(Size::S64, new, &mut temps, ImmType::None, false, None)?;
7477                let reread = this.get_label();
7478                let nosame = this.get_label();
7479
7480                this.emit_label(reread)?;
7481                this.assembler
7482                    .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
7483                this.emit_relaxed_cmp(Size::S64, dst, cmp)?;
7484                this.assembler.emit_bcond_label(Condition::Ne, nosame)?;
7485                this.assembler.emit_stlxr(
7486                    Size::S32,
7487                    Location::GPR(tmp),
7488                    org,
7489                    Location::GPR(addr),
7490                )?;
7491                this.assembler
7492                    .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7493                this.assembler.emit_dmb()?;
7494
7495                this.emit_label(nosame)?;
7496                if dst != ret {
7497                    this.move_location(Size::S64, ret, dst)?;
7498                }
7499                for r in temps {
7500                    this.release_gpr(r);
7501                }
7502                this.release_gpr(tmp);
7503                Ok(())
7504            },
7505        )
7506    }
7507
7508    fn f32_load(
7509        &mut self,
7510        addr: Location,
7511        memarg: &MemArg,
7512        ret: Location,
7513        need_check: bool,
7514        imported_memories: bool,
7515        offset: i32,
7516        heap_access_oob: Label,
7517        unaligned_atomic: Label,
7518    ) -> Result<(), CompileError> {
7519        self.memory_op(
7520            addr,
7521            memarg,
7522            false,
7523            4,
7524            need_check,
7525            imported_memories,
7526            offset,
7527            heap_access_oob,
7528            unaligned_atomic,
7529            |this, addr| this.emit_relaxed_ldr32(Size::S32, ret, Location::Memory(addr, 0)),
7530        )
7531    }
7532    fn f32_save(
7533        &mut self,
7534        target_value: Location,
7535        memarg: &MemArg,
7536        target_addr: Location,
7537        canonicalize: bool,
7538        need_check: bool,
7539        imported_memories: bool,
7540        offset: i32,
7541        heap_access_oob: Label,
7542        unaligned_atomic: Label,
7543    ) -> Result<(), CompileError> {
7544        let canonicalize = canonicalize && self.arch_supports_canonicalize_nan();
7545        self.memory_op(
7546            target_addr,
7547            memarg,
7548            false,
7549            4,
7550            need_check,
7551            imported_memories,
7552            offset,
7553            heap_access_oob,
7554            unaligned_atomic,
7555            |this, addr| {
7556                if !canonicalize {
7557                    this.emit_relaxed_str32(target_value, Location::Memory(addr, 0))
7558                } else {
7559                    this.canonicalize_nan(Size::S32, target_value, Location::Memory(addr, 0))
7560                }
7561            },
7562        )
7563    }
7564    fn f64_load(
7565        &mut self,
7566        addr: Location,
7567        memarg: &MemArg,
7568        ret: Location,
7569        need_check: bool,
7570        imported_memories: bool,
7571        offset: i32,
7572        heap_access_oob: Label,
7573        unaligned_atomic: Label,
7574    ) -> Result<(), CompileError> {
7575        self.memory_op(
7576            addr,
7577            memarg,
7578            false,
7579            8,
7580            need_check,
7581            imported_memories,
7582            offset,
7583            heap_access_oob,
7584            unaligned_atomic,
7585            |this, addr| this.emit_relaxed_ldr64(Size::S64, ret, Location::Memory(addr, 0)),
7586        )
7587    }
7588    fn f64_save(
7589        &mut self,
7590        target_value: Location,
7591        memarg: &MemArg,
7592        target_addr: Location,
7593        canonicalize: bool,
7594        need_check: bool,
7595        imported_memories: bool,
7596        offset: i32,
7597        heap_access_oob: Label,
7598        unaligned_atomic: Label,
7599    ) -> Result<(), CompileError> {
7600        let canonicalize = canonicalize && self.arch_supports_canonicalize_nan();
7601        self.memory_op(
7602            target_addr,
7603            memarg,
7604            false,
7605            8,
7606            need_check,
7607            imported_memories,
7608            offset,
7609            heap_access_oob,
7610            unaligned_atomic,
7611            |this, addr| {
7612                if !canonicalize {
7613                    this.emit_relaxed_str64(target_value, Location::Memory(addr, 0))
7614                } else {
7615                    this.canonicalize_nan(Size::S64, target_value, Location::Memory(addr, 0))
7616                }
7617            },
7618        )
7619    }
7620
7621    fn convert_f64_i64(
7622        &mut self,
7623        loc: Location,
7624        signed: bool,
7625        ret: Location,
7626    ) -> Result<(), CompileError> {
7627        let mut gprs = vec![];
7628        let mut neons = vec![];
7629        let src = self.location_to_reg(Size::S64, loc, &mut gprs, ImmType::NoneXzr, true, None)?;
7630        let dest = self.location_to_neon(Size::S64, ret, &mut neons, ImmType::None, false)?;
7631        if signed {
7632            self.assembler.emit_scvtf(Size::S64, src, Size::S64, dest)?;
7633        } else {
7634            self.assembler.emit_ucvtf(Size::S64, src, Size::S64, dest)?;
7635        }
7636        if ret != dest {
7637            self.move_location(Size::S64, dest, ret)?;
7638        }
7639        for r in gprs {
7640            self.release_gpr(r);
7641        }
7642        for r in neons {
7643            self.release_simd(r);
7644        }
7645        Ok(())
7646    }
7647    fn convert_f64_i32(
7648        &mut self,
7649        loc: Location,
7650        signed: bool,
7651        ret: Location,
7652    ) -> Result<(), CompileError> {
7653        let mut gprs = vec![];
7654        let mut neons = vec![];
7655        let src = self.location_to_reg(Size::S32, loc, &mut gprs, ImmType::NoneXzr, true, None)?;
7656        let dest = self.location_to_neon(Size::S64, ret, &mut neons, ImmType::None, false)?;
7657        if signed {
7658            self.assembler.emit_scvtf(Size::S32, src, Size::S64, dest)?;
7659        } else {
7660            self.assembler.emit_ucvtf(Size::S32, src, Size::S64, dest)?;
7661        }
7662        if ret != dest {
7663            self.move_location(Size::S64, dest, ret)?;
7664        }
7665        for r in gprs {
7666            self.release_gpr(r);
7667        }
7668        for r in neons {
7669            self.release_simd(r);
7670        }
7671        Ok(())
7672    }
7673    fn convert_f32_i64(
7674        &mut self,
7675        loc: Location,
7676        signed: bool,
7677        ret: Location,
7678    ) -> Result<(), CompileError> {
7679        let mut gprs = vec![];
7680        let mut neons = vec![];
7681        let src = self.location_to_reg(Size::S64, loc, &mut gprs, ImmType::NoneXzr, true, None)?;
7682        let dest = self.location_to_neon(Size::S32, ret, &mut neons, ImmType::None, false)?;
7683        if signed {
7684            self.assembler.emit_scvtf(Size::S64, src, Size::S32, dest)?;
7685        } else {
7686            self.assembler.emit_ucvtf(Size::S64, src, Size::S32, dest)?;
7687        }
7688        if ret != dest {
7689            self.move_location(Size::S32, dest, ret)?;
7690        }
7691        for r in gprs {
7692            self.release_gpr(r);
7693        }
7694        for r in neons {
7695            self.release_simd(r);
7696        }
7697        Ok(())
7698    }
7699    fn convert_f32_i32(
7700        &mut self,
7701        loc: Location,
7702        signed: bool,
7703        ret: Location,
7704    ) -> Result<(), CompileError> {
7705        let mut gprs = vec![];
7706        let mut neons = vec![];
7707        let src = self.location_to_reg(Size::S32, loc, &mut gprs, ImmType::NoneXzr, true, None)?;
7708        let dest = self.location_to_neon(Size::S32, ret, &mut neons, ImmType::None, false)?;
7709        if signed {
7710            self.assembler.emit_scvtf(Size::S32, src, Size::S32, dest)?;
7711        } else {
7712            self.assembler.emit_ucvtf(Size::S32, src, Size::S32, dest)?;
7713        }
7714        if ret != dest {
7715            self.move_location(Size::S32, dest, ret)?;
7716        }
7717        for r in gprs {
7718            self.release_gpr(r);
7719        }
7720        for r in neons {
7721            self.release_simd(r);
7722        }
7723        Ok(())
7724    }
7725    fn convert_i64_f64(
7726        &mut self,
7727        loc: Location,
7728        ret: Location,
7729        signed: bool,
7730        sat: bool,
7731    ) -> Result<(), CompileError> {
7732        let mut gprs = vec![];
7733        let mut neons = vec![];
7734        let src = self.location_to_neon(Size::S64, loc, &mut neons, ImmType::None, true)?;
7735        let dest = self.location_to_reg(Size::S64, ret, &mut gprs, ImmType::None, false, None)?;
7736        let old_fpcr = if !sat {
7737            self.reset_exception_fpsr()?;
7738            self.set_trap_enabled(&mut gprs)?
7739        } else {
7740            GPR::XzrSp
7741        };
7742        if signed {
7743            self.assembler
7744                .emit_fcvtzs(Size::S64, src, Size::S64, dest)?;
7745        } else {
7746            self.assembler
7747                .emit_fcvtzu(Size::S64, src, Size::S64, dest)?;
7748        }
7749        if !sat {
7750            self.trap_float_convertion_errors(old_fpcr, Size::S64, src, &mut gprs)?;
7751        }
7752        if ret != dest {
7753            self.move_location(Size::S64, dest, ret)?;
7754        }
7755        for r in gprs {
7756            self.release_gpr(r);
7757        }
7758        for r in neons {
7759            self.release_simd(r);
7760        }
7761        Ok(())
7762    }
7763    fn convert_i32_f64(
7764        &mut self,
7765        loc: Location,
7766        ret: Location,
7767        signed: bool,
7768        sat: bool,
7769    ) -> Result<(), CompileError> {
7770        let mut gprs = vec![];
7771        let mut neons = vec![];
7772        let src = self.location_to_neon(Size::S64, loc, &mut neons, ImmType::None, true)?;
7773        let dest = self.location_to_reg(Size::S32, ret, &mut gprs, ImmType::None, false, None)?;
7774        let old_fpcr = if !sat {
7775            self.reset_exception_fpsr()?;
7776            self.set_trap_enabled(&mut gprs)?
7777        } else {
7778            GPR::XzrSp
7779        };
7780        if signed {
7781            self.assembler
7782                .emit_fcvtzs(Size::S64, src, Size::S32, dest)?;
7783        } else {
7784            self.assembler
7785                .emit_fcvtzu(Size::S64, src, Size::S32, dest)?;
7786        }
7787        if !sat {
7788            self.trap_float_convertion_errors(old_fpcr, Size::S64, src, &mut gprs)?;
7789        }
7790        if ret != dest {
7791            self.move_location(Size::S32, dest, ret)?;
7792        }
7793        for r in gprs {
7794            self.release_gpr(r);
7795        }
7796        for r in neons {
7797            self.release_simd(r);
7798        }
7799        Ok(())
7800    }
7801    fn convert_i64_f32(
7802        &mut self,
7803        loc: Location,
7804        ret: Location,
7805        signed: bool,
7806        sat: bool,
7807    ) -> Result<(), CompileError> {
7808        let mut gprs = vec![];
7809        let mut neons = vec![];
7810        let src = self.location_to_neon(Size::S32, loc, &mut neons, ImmType::None, true)?;
7811        let dest = self.location_to_reg(Size::S64, ret, &mut gprs, ImmType::None, false, None)?;
7812        let old_fpcr = if !sat {
7813            self.reset_exception_fpsr()?;
7814            self.set_trap_enabled(&mut gprs)?
7815        } else {
7816            GPR::XzrSp
7817        };
7818        if signed {
7819            self.assembler
7820                .emit_fcvtzs(Size::S32, src, Size::S64, dest)?;
7821        } else {
7822            self.assembler
7823                .emit_fcvtzu(Size::S32, src, Size::S64, dest)?;
7824        }
7825        if !sat {
7826            self.trap_float_convertion_errors(old_fpcr, Size::S32, src, &mut gprs)?;
7827        }
7828        if ret != dest {
7829            self.move_location(Size::S64, dest, ret)?;
7830        }
7831        for r in gprs {
7832            self.release_gpr(r);
7833        }
7834        for r in neons {
7835            self.release_simd(r);
7836        }
7837        Ok(())
7838    }
7839    fn convert_i32_f32(
7840        &mut self,
7841        loc: Location,
7842        ret: Location,
7843        signed: bool,
7844        sat: bool,
7845    ) -> Result<(), CompileError> {
7846        let mut gprs = vec![];
7847        let mut neons = vec![];
7848        let src = self.location_to_neon(Size::S32, loc, &mut neons, ImmType::None, true)?;
7849        let dest = self.location_to_reg(Size::S32, ret, &mut gprs, ImmType::None, false, None)?;
7850        let old_fpcr = if !sat {
7851            self.reset_exception_fpsr()?;
7852            self.set_trap_enabled(&mut gprs)?
7853        } else {
7854            GPR::XzrSp
7855        };
7856        if signed {
7857            self.assembler
7858                .emit_fcvtzs(Size::S32, src, Size::S32, dest)?;
7859        } else {
7860            self.assembler
7861                .emit_fcvtzu(Size::S32, src, Size::S32, dest)?;
7862        }
7863        if !sat {
7864            self.trap_float_convertion_errors(old_fpcr, Size::S32, src, &mut gprs)?;
7865        }
7866        if ret != dest {
7867            self.move_location(Size::S32, dest, ret)?;
7868        }
7869        for r in gprs {
7870            self.release_gpr(r);
7871        }
7872        for r in neons {
7873            self.release_simd(r);
7874        }
7875        Ok(())
7876    }
7877    fn convert_f64_f32(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7878        self.emit_relaxed_binop_neon(Assembler::emit_fcvt, Size::S32, loc, ret, true)
7879    }
7880    fn convert_f32_f64(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7881        self.emit_relaxed_binop_neon(Assembler::emit_fcvt, Size::S64, loc, ret, true)
7882    }
7883    fn f64_neg(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7884        self.emit_relaxed_binop_neon(Assembler::emit_fneg, Size::S64, loc, ret, true)
7885    }
7886    fn f64_abs(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7887        let tmp = self.acquire_temp_gpr().ok_or_else(|| {
7888            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7889        })?;
7890
7891        self.move_location(Size::S64, loc, Location::GPR(tmp))?;
7892        self.assembler.emit_and(
7893            Size::S64,
7894            Location::GPR(tmp),
7895            Location::Imm64(0x7fffffffffffffffu64),
7896            Location::GPR(tmp),
7897        )?;
7898        self.move_location(Size::S64, Location::GPR(tmp), ret)?;
7899
7900        self.release_gpr(tmp);
7901        Ok(())
7902    }
7903    fn emit_i64_copysign(&mut self, tmp1: GPR, tmp2: GPR) -> Result<(), CompileError> {
7904        self.assembler.emit_and(
7905            Size::S64,
7906            Location::GPR(tmp1),
7907            Location::Imm64(0x7fffffffffffffffu64),
7908            Location::GPR(tmp1),
7909        )?;
7910
7911        self.assembler.emit_and(
7912            Size::S64,
7913            Location::GPR(tmp2),
7914            Location::Imm64(0x8000000000000000u64),
7915            Location::GPR(tmp2),
7916        )?;
7917
7918        self.assembler.emit_or(
7919            Size::S64,
7920            Location::GPR(tmp1),
7921            Location::GPR(tmp2),
7922            Location::GPR(tmp1),
7923        )
7924    }
7925    fn f64_sqrt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7926        self.emit_relaxed_binop_neon(Assembler::emit_fsqrt, Size::S64, loc, ret, true)
7927    }
7928    fn f64_trunc(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7929        self.emit_relaxed_binop_neon(Assembler::emit_frintz, Size::S64, loc, ret, true)
7930    }
7931    fn f64_ceil(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7932        self.emit_relaxed_binop_neon(Assembler::emit_frintp, Size::S64, loc, ret, true)
7933    }
7934    fn f64_floor(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7935        self.emit_relaxed_binop_neon(Assembler::emit_frintm, Size::S64, loc, ret, true)
7936    }
7937    fn f64_nearest(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7938        self.emit_relaxed_binop_neon(Assembler::emit_frintn, Size::S64, loc, ret, true)
7939    }
7940    fn f64_cmp_ge(
7941        &mut self,
7942        loc_a: Location,
7943        loc_b: Location,
7944        ret: Location,
7945    ) -> Result<(), CompileError> {
7946        let mut temps = vec![];
7947        let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7948        self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S64, loc_b, loc_a, false)?;
7949        self.assembler.emit_cset(Size::S32, dest, Condition::Ls)?;
7950        if ret != dest {
7951            self.move_location(Size::S32, dest, ret)?;
7952        }
7953        for r in temps {
7954            self.release_gpr(r);
7955        }
7956        Ok(())
7957    }
7958    fn f64_cmp_gt(
7959        &mut self,
7960        loc_a: Location,
7961        loc_b: Location,
7962        ret: Location,
7963    ) -> Result<(), CompileError> {
7964        let mut temps = vec![];
7965        let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7966        self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S64, loc_b, loc_a, false)?;
7967        self.assembler.emit_cset(Size::S32, dest, Condition::Cc)?;
7968        if ret != dest {
7969            self.move_location(Size::S32, dest, ret)?;
7970        }
7971        for r in temps {
7972            self.release_gpr(r);
7973        }
7974        Ok(())
7975    }
7976    fn f64_cmp_le(
7977        &mut self,
7978        loc_a: Location,
7979        loc_b: Location,
7980        ret: Location,
7981    ) -> Result<(), CompileError> {
7982        let mut temps = vec![];
7983        let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7984        self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S64, loc_a, loc_b, false)?;
7985        self.assembler.emit_cset(Size::S32, dest, Condition::Ls)?;
7986        if ret != dest {
7987            self.move_location(Size::S32, dest, ret)?;
7988        }
7989        for r in temps {
7990            self.release_gpr(r);
7991        }
7992        Ok(())
7993    }
7994    fn f64_cmp_lt(
7995        &mut self,
7996        loc_a: Location,
7997        loc_b: Location,
7998        ret: Location,
7999    ) -> Result<(), CompileError> {
8000        let mut temps = vec![];
8001        let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
8002        self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S64, loc_a, loc_b, false)?;
8003        self.assembler.emit_cset(Size::S32, dest, Condition::Cc)?;
8004        if ret != dest {
8005            self.move_location(Size::S32, dest, ret)?;
8006        }
8007        for r in temps {
8008            self.release_gpr(r);
8009        }
8010        Ok(())
8011    }
8012    fn f64_cmp_ne(
8013        &mut self,
8014        loc_a: Location,
8015        loc_b: Location,
8016        ret: Location,
8017    ) -> Result<(), CompileError> {
8018        let mut temps = vec![];
8019        let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
8020        self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S64, loc_a, loc_b, false)?;
8021        self.assembler.emit_cset(Size::S32, dest, Condition::Ne)?;
8022        if ret != dest {
8023            self.move_location(Size::S32, dest, ret)?;
8024        }
8025        for r in temps {
8026            self.release_gpr(r);
8027        }
8028        Ok(())
8029    }
8030    fn f64_cmp_eq(
8031        &mut self,
8032        loc_a: Location,
8033        loc_b: Location,
8034        ret: Location,
8035    ) -> Result<(), CompileError> {
8036        let mut temps = vec![];
8037        let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
8038        self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S64, loc_a, loc_b, false)?;
8039        self.assembler.emit_cset(Size::S32, dest, Condition::Eq)?;
8040        if ret != dest {
8041            self.move_location(Size::S32, dest, ret)?;
8042        }
8043        for r in temps {
8044            self.release_gpr(r);
8045        }
8046        Ok(())
8047    }
8048    fn f64_min(
8049        &mut self,
8050        loc_a: Location,
8051        loc_b: Location,
8052        ret: Location,
8053    ) -> Result<(), CompileError> {
8054        let mut temps = vec![];
8055        let old_fpcr = self.set_default_nan(&mut temps)?;
8056        self.emit_relaxed_binop3_neon(
8057            Assembler::emit_fmin,
8058            Size::S64,
8059            loc_a,
8060            loc_b,
8061            ret,
8062            ImmType::None,
8063        )?;
8064        self.restore_fpcr(old_fpcr)?;
8065        for r in temps {
8066            self.release_gpr(r);
8067        }
8068        Ok(())
8069    }
8070    fn f64_max(
8071        &mut self,
8072        loc_a: Location,
8073        loc_b: Location,
8074        ret: Location,
8075    ) -> Result<(), CompileError> {
8076        let mut temps = vec![];
8077        let old_fpcr = self.set_default_nan(&mut temps)?;
8078        self.emit_relaxed_binop3_neon(
8079            Assembler::emit_fmax,
8080            Size::S64,
8081            loc_a,
8082            loc_b,
8083            ret,
8084            ImmType::None,
8085        )?;
8086        self.restore_fpcr(old_fpcr)?;
8087        for r in temps {
8088            self.release_gpr(r);
8089        }
8090        Ok(())
8091    }
8092    fn f64_add(
8093        &mut self,
8094        loc_a: Location,
8095        loc_b: Location,
8096        ret: Location,
8097    ) -> Result<(), CompileError> {
8098        self.emit_relaxed_binop3_neon(
8099            Assembler::emit_fadd,
8100            Size::S64,
8101            loc_a,
8102            loc_b,
8103            ret,
8104            ImmType::None,
8105        )
8106    }
8107    fn f64_sub(
8108        &mut self,
8109        loc_a: Location,
8110        loc_b: Location,
8111        ret: Location,
8112    ) -> Result<(), CompileError> {
8113        self.emit_relaxed_binop3_neon(
8114            Assembler::emit_fsub,
8115            Size::S64,
8116            loc_a,
8117            loc_b,
8118            ret,
8119            ImmType::None,
8120        )
8121    }
8122    fn f64_mul(
8123        &mut self,
8124        loc_a: Location,
8125        loc_b: Location,
8126        ret: Location,
8127    ) -> Result<(), CompileError> {
8128        self.emit_relaxed_binop3_neon(
8129            Assembler::emit_fmul,
8130            Size::S64,
8131            loc_a,
8132            loc_b,
8133            ret,
8134            ImmType::None,
8135        )
8136    }
8137    fn f64_div(
8138        &mut self,
8139        loc_a: Location,
8140        loc_b: Location,
8141        ret: Location,
8142    ) -> Result<(), CompileError> {
8143        self.emit_relaxed_binop3_neon(
8144            Assembler::emit_fdiv,
8145            Size::S64,
8146            loc_a,
8147            loc_b,
8148            ret,
8149            ImmType::None,
8150        )
8151    }
8152    fn f32_neg(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
8153        self.emit_relaxed_binop_neon(Assembler::emit_fneg, Size::S32, loc, ret, true)
8154    }
8155    fn f32_abs(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
8156        let tmp = self.acquire_temp_gpr().ok_or_else(|| {
8157            CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
8158        })?;
8159        self.move_location(Size::S32, loc, Location::GPR(tmp))?;
8160        self.assembler.emit_and(
8161            Size::S32,
8162            Location::GPR(tmp),
8163            Location::Imm32(0x7fffffffu32),
8164            Location::GPR(tmp),
8165        )?;
8166        self.move_location(Size::S32, Location::GPR(tmp), ret)?;
8167        self.release_gpr(tmp);
8168        Ok(())
8169    }
8170    fn emit_i32_copysign(&mut self, tmp1: GPR, tmp2: GPR) -> Result<(), CompileError> {
8171        self.assembler.emit_and(
8172            Size::S32,
8173            Location::GPR(tmp1),
8174            Location::Imm32(0x7fffffffu32),
8175            Location::GPR(tmp1),
8176        )?;
8177        self.assembler.emit_and(
8178            Size::S32,
8179            Location::GPR(tmp2),
8180            Location::Imm32(0x80000000u32),
8181            Location::GPR(tmp2),
8182        )?;
8183        self.assembler.emit_or(
8184            Size::S32,
8185            Location::GPR(tmp1),
8186            Location::GPR(tmp2),
8187            Location::GPR(tmp1),
8188        )
8189    }
8190    fn f32_sqrt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
8191        self.emit_relaxed_binop_neon(Assembler::emit_fsqrt, Size::S32, loc, ret, true)
8192    }
8193    fn f32_trunc(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
8194        self.emit_relaxed_binop_neon(Assembler::emit_frintz, Size::S32, loc, ret, true)
8195    }
8196    fn f32_ceil(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
8197        self.emit_relaxed_binop_neon(Assembler::emit_frintp, Size::S32, loc, ret, true)
8198    }
8199    fn f32_floor(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
8200        self.emit_relaxed_binop_neon(Assembler::emit_frintm, Size::S32, loc, ret, true)
8201    }
8202    fn f32_nearest(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
8203        self.emit_relaxed_binop_neon(Assembler::emit_frintn, Size::S32, loc, ret, true)
8204    }
8205    fn f32_cmp_ge(
8206        &mut self,
8207        loc_a: Location,
8208        loc_b: Location,
8209        ret: Location,
8210    ) -> Result<(), CompileError> {
8211        let mut temps = vec![];
8212        let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
8213        self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S32, loc_b, loc_a, false)?;
8214        self.assembler.emit_cset(Size::S32, dest, Condition::Ls)?;
8215        if ret != dest {
8216            self.move_location(Size::S32, dest, ret)?;
8217        }
8218        for r in temps {
8219            self.release_gpr(r);
8220        }
8221        Ok(())
8222    }
8223    fn f32_cmp_gt(
8224        &mut self,
8225        loc_a: Location,
8226        loc_b: Location,
8227        ret: Location,
8228    ) -> Result<(), CompileError> {
8229        let mut temps = vec![];
8230        let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
8231        self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S32, loc_b, loc_a, false)?;
8232        self.assembler.emit_cset(Size::S32, dest, Condition::Cc)?;
8233        if ret != dest {
8234            self.move_location(Size::S32, dest, ret)?;
8235        }
8236        for r in temps {
8237            self.release_gpr(r);
8238        }
8239        Ok(())
8240    }
8241    fn f32_cmp_le(
8242        &mut self,
8243        loc_a: Location,
8244        loc_b: Location,
8245        ret: Location,
8246    ) -> Result<(), CompileError> {
8247        let mut temps = vec![];
8248        let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
8249        self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S32, loc_a, loc_b, false)?;
8250        self.assembler.emit_cset(Size::S32, dest, Condition::Ls)?;
8251        if ret != dest {
8252            self.move_location(Size::S32, dest, ret)?;
8253        }
8254        for r in temps {
8255            self.release_gpr(r);
8256        }
8257        Ok(())
8258    }
8259    fn f32_cmp_lt(
8260        &mut self,
8261        loc_a: Location,
8262        loc_b: Location,
8263        ret: Location,
8264    ) -> Result<(), CompileError> {
8265        let mut temps = vec![];
8266        let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
8267        self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S32, loc_a, loc_b, false)?;
8268        self.assembler.emit_cset(Size::S32, dest, Condition::Cc)?;
8269        if ret != dest {
8270            self.move_location(Size::S32, dest, ret)?;
8271        }
8272        for r in temps {
8273            self.release_gpr(r);
8274        }
8275        Ok(())
8276    }
8277    fn f32_cmp_ne(
8278        &mut self,
8279        loc_a: Location,
8280        loc_b: Location,
8281        ret: Location,
8282    ) -> Result<(), CompileError> {
8283        let mut temps = vec![];
8284        let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
8285        self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S32, loc_a, loc_b, false)?;
8286        self.assembler.emit_cset(Size::S32, dest, Condition::Ne)?;
8287        if ret != dest {
8288            self.move_location(Size::S32, dest, ret)?;
8289        }
8290        for r in temps {
8291            self.release_gpr(r);
8292        }
8293        Ok(())
8294    }
8295    fn f32_cmp_eq(
8296        &mut self,
8297        loc_a: Location,
8298        loc_b: Location,
8299        ret: Location,
8300    ) -> Result<(), CompileError> {
8301        let mut temps = vec![];
8302        let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
8303        self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S32, loc_a, loc_b, false)?;
8304        self.assembler.emit_cset(Size::S32, dest, Condition::Eq)?;
8305        if ret != dest {
8306            self.move_location(Size::S32, dest, ret)?;
8307        }
8308        for r in temps {
8309            self.release_gpr(r);
8310        }
8311        Ok(())
8312    }
8313    fn f32_min(
8314        &mut self,
8315        loc_a: Location,
8316        loc_b: Location,
8317        ret: Location,
8318    ) -> Result<(), CompileError> {
8319        let mut temps = vec![];
8320        let old_fpcr = self.set_default_nan(&mut temps)?;
8321        self.emit_relaxed_binop3_neon(
8322            Assembler::emit_fmin,
8323            Size::S32,
8324            loc_a,
8325            loc_b,
8326            ret,
8327            ImmType::None,
8328        )?;
8329        self.restore_fpcr(old_fpcr)?;
8330        for r in temps {
8331            self.release_gpr(r);
8332        }
8333        Ok(())
8334    }
8335    fn f32_max(
8336        &mut self,
8337        loc_a: Location,
8338        loc_b: Location,
8339        ret: Location,
8340    ) -> Result<(), CompileError> {
8341        let mut temps = vec![];
8342        let old_fpcr = self.set_default_nan(&mut temps)?;
8343        self.emit_relaxed_binop3_neon(
8344            Assembler::emit_fmax,
8345            Size::S32,
8346            loc_a,
8347            loc_b,
8348            ret,
8349            ImmType::None,
8350        )?;
8351        self.restore_fpcr(old_fpcr)?;
8352        for r in temps {
8353            self.release_gpr(r);
8354        }
8355        Ok(())
8356    }
8357    fn f32_add(
8358        &mut self,
8359        loc_a: Location,
8360        loc_b: Location,
8361        ret: Location,
8362    ) -> Result<(), CompileError> {
8363        self.emit_relaxed_binop3_neon(
8364            Assembler::emit_fadd,
8365            Size::S32,
8366            loc_a,
8367            loc_b,
8368            ret,
8369            ImmType::None,
8370        )
8371    }
8372    fn f32_sub(
8373        &mut self,
8374        loc_a: Location,
8375        loc_b: Location,
8376        ret: Location,
8377    ) -> Result<(), CompileError> {
8378        self.emit_relaxed_binop3_neon(
8379            Assembler::emit_fsub,
8380            Size::S32,
8381            loc_a,
8382            loc_b,
8383            ret,
8384            ImmType::None,
8385        )
8386    }
8387    fn f32_mul(
8388        &mut self,
8389        loc_a: Location,
8390        loc_b: Location,
8391        ret: Location,
8392    ) -> Result<(), CompileError> {
8393        self.emit_relaxed_binop3_neon(
8394            Assembler::emit_fmul,
8395            Size::S32,
8396            loc_a,
8397            loc_b,
8398            ret,
8399            ImmType::None,
8400        )
8401    }
8402    fn f32_div(
8403        &mut self,
8404        loc_a: Location,
8405        loc_b: Location,
8406        ret: Location,
8407    ) -> Result<(), CompileError> {
8408        self.emit_relaxed_binop3_neon(
8409            Assembler::emit_fdiv,
8410            Size::S32,
8411            loc_a,
8412            loc_b,
8413            ret,
8414            ImmType::None,
8415        )
8416    }
8417
8418    fn gen_std_trampoline(
8419        &self,
8420        sig: &FunctionType,
8421        calling_convention: CallingConvention,
8422    ) -> Result<FunctionBody, CompileError> {
8423        gen_std_trampoline_arm64(sig, calling_convention)
8424    }
8425    // Generates dynamic import function call trampoline for a function type.
8426    fn gen_std_dynamic_import_trampoline(
8427        &self,
8428        vmoffsets: &VMOffsets,
8429        sig: &FunctionType,
8430        calling_convention: CallingConvention,
8431    ) -> Result<FunctionBody, CompileError> {
8432        gen_std_dynamic_import_trampoline_arm64(vmoffsets, sig, calling_convention)
8433    }
8434    // Singlepass calls import functions through a trampoline.
8435    fn gen_import_call_trampoline(
8436        &self,
8437        vmoffsets: &VMOffsets,
8438        index: FunctionIndex,
8439        sig: &FunctionType,
8440        calling_convention: CallingConvention,
8441    ) -> Result<CustomSection, CompileError> {
8442        gen_import_call_trampoline_arm64(vmoffsets, index, sig, calling_convention)
8443    }
8444    #[cfg(feature = "unwind")]
8445    fn gen_dwarf_unwind_info(&mut self, code_len: usize) -> Option<UnwindInstructions> {
8446        let mut instructions = vec![];
8447        for &(instruction_offset, ref inst) in &self.unwind_ops {
8448            let instruction_offset = instruction_offset as u32;
8449            match *inst {
8450                UnwindOps::PushFP { up_to_sp } => {
8451                    instructions.push((
8452                        instruction_offset,
8453                        CallFrameInstruction::CfaOffset(up_to_sp as i32),
8454                    ));
8455                    instructions.push((
8456                        instruction_offset,
8457                        CallFrameInstruction::Offset(AArch64::X29, -(up_to_sp as i32)),
8458                    ));
8459                }
8460                UnwindOps::Push2Regs {
8461                    reg1,
8462                    reg2,
8463                    up_to_sp,
8464                } => {
8465                    instructions.push((
8466                        instruction_offset,
8467                        CallFrameInstruction::CfaOffset(up_to_sp as i32),
8468                    ));
8469                    instructions.push((
8470                        instruction_offset,
8471                        CallFrameInstruction::Offset(reg2.dwarf_index(), -(up_to_sp as i32) + 8),
8472                    ));
8473                    instructions.push((
8474                        instruction_offset,
8475                        CallFrameInstruction::Offset(reg1.dwarf_index(), -(up_to_sp as i32)),
8476                    ));
8477                }
8478                UnwindOps::DefineNewFrame => {
8479                    instructions.push((
8480                        instruction_offset,
8481                        CallFrameInstruction::CfaRegister(AArch64::X29),
8482                    ));
8483                }
8484                UnwindOps::SaveRegister { reg, bp_neg_offset } => instructions.push((
8485                    instruction_offset,
8486                    CallFrameInstruction::Offset(reg.dwarf_index(), -bp_neg_offset),
8487                )),
8488            }
8489        }
8490        Some(UnwindInstructions {
8491            instructions,
8492            len: code_len as u32,
8493        })
8494    }
8495    #[cfg(not(feature = "unwind"))]
8496    fn gen_dwarf_unwind_info(&mut self, _code_len: usize) -> Option<UnwindInstructions> {
8497        None
8498    }
8499
8500    fn gen_windows_unwind_info(&mut self, _code_len: usize) -> Option<Vec<u8>> {
8501        None
8502    }
8503}
8504
8505#[cfg(test)]
8506mod test {
8507    use super::*;
8508
8509    fn test_move_location(machine: &mut MachineARM64, size: Size) -> Result<(), CompileError> {
8510        machine.move_location(size, Location::GPR(GPR::X1), Location::GPR(GPR::X2))?;
8511        machine.move_location(size, Location::GPR(GPR::X1), Location::Memory(GPR::X2, 10))?;
8512        machine.move_location(size, Location::GPR(GPR::X1), Location::Memory(GPR::X2, -10))?;
8513        machine.move_location(
8514            size,
8515            Location::GPR(GPR::X1),
8516            Location::Memory(GPR::X2, 1024),
8517        )?;
8518        machine.move_location(
8519            size,
8520            Location::GPR(GPR::X1),
8521            Location::Memory(GPR::X2, -1024),
8522        )?;
8523        machine.move_location(size, Location::Memory(GPR::X2, 10), Location::GPR(GPR::X1))?;
8524        machine.move_location(size, Location::Memory(GPR::X2, -10), Location::GPR(GPR::X1))?;
8525        machine.move_location(
8526            size,
8527            Location::Memory(GPR::X2, 1024),
8528            Location::GPR(GPR::X1),
8529        )?;
8530        machine.move_location(
8531            size,
8532            Location::Memory(GPR::X2, -1024),
8533            Location::GPR(GPR::X1),
8534        )?;
8535        machine.move_location(size, Location::GPR(GPR::X1), Location::SIMD(NEON::V0))?;
8536        machine.move_location(size, Location::SIMD(NEON::V0), Location::GPR(GPR::X1))?;
8537        machine.move_location(
8538            size,
8539            Location::SIMD(NEON::V0),
8540            Location::Memory(GPR::X2, 10),
8541        )?;
8542        machine.move_location(
8543            size,
8544            Location::SIMD(NEON::V0),
8545            Location::Memory(GPR::X2, -10),
8546        )?;
8547        machine.move_location(
8548            size,
8549            Location::SIMD(NEON::V0),
8550            Location::Memory(GPR::X2, 1024),
8551        )?;
8552        machine.move_location(
8553            size,
8554            Location::SIMD(NEON::V0),
8555            Location::Memory(GPR::X2, -1024),
8556        )?;
8557        machine.move_location(
8558            size,
8559            Location::Memory(GPR::X2, 10),
8560            Location::SIMD(NEON::V0),
8561        )?;
8562        machine.move_location(
8563            size,
8564            Location::Memory(GPR::X2, -10),
8565            Location::SIMD(NEON::V0),
8566        )?;
8567        machine.move_location(
8568            size,
8569            Location::Memory(GPR::X2, 1024),
8570            Location::SIMD(NEON::V0),
8571        )?;
8572        machine.move_location(
8573            size,
8574            Location::Memory(GPR::X2, -1024),
8575            Location::SIMD(NEON::V0),
8576        )?;
8577
8578        Ok(())
8579    }
8580
8581    fn test_move_location_extended(
8582        machine: &mut MachineARM64,
8583        signed: bool,
8584        sized: Size,
8585    ) -> Result<(), CompileError> {
8586        machine.move_location_extend(
8587            sized,
8588            signed,
8589            Location::GPR(GPR::X0),
8590            Size::S64,
8591            Location::GPR(GPR::X1),
8592        )?;
8593        machine.move_location_extend(
8594            sized,
8595            signed,
8596            Location::GPR(GPR::X0),
8597            Size::S64,
8598            Location::Memory(GPR::X1, 10),
8599        )?;
8600        machine.move_location_extend(
8601            sized,
8602            signed,
8603            Location::GPR(GPR::X0),
8604            Size::S64,
8605            Location::Memory(GPR::X1, 16),
8606        )?;
8607        machine.move_location_extend(
8608            sized,
8609            signed,
8610            Location::GPR(GPR::X0),
8611            Size::S64,
8612            Location::Memory(GPR::X1, -16),
8613        )?;
8614        machine.move_location_extend(
8615            sized,
8616            signed,
8617            Location::GPR(GPR::X0),
8618            Size::S64,
8619            Location::Memory(GPR::X1, 1024),
8620        )?;
8621        machine.move_location_extend(
8622            sized,
8623            signed,
8624            Location::GPR(GPR::X0),
8625            Size::S64,
8626            Location::Memory(GPR::X1, -1024),
8627        )?;
8628        machine.move_location_extend(
8629            sized,
8630            signed,
8631            Location::Memory(GPR::X0, 10),
8632            Size::S64,
8633            Location::GPR(GPR::X1),
8634        )?;
8635
8636        Ok(())
8637    }
8638
8639    fn test_binop_op(
8640        machine: &mut MachineARM64,
8641        op: fn(&mut MachineARM64, Location, Location, Location) -> Result<(), CompileError>,
8642    ) -> Result<(), CompileError> {
8643        op(
8644            machine,
8645            Location::GPR(GPR::X2),
8646            Location::GPR(GPR::X2),
8647            Location::GPR(GPR::X0),
8648        )?;
8649        op(
8650            machine,
8651            Location::GPR(GPR::X2),
8652            Location::Imm32(10),
8653            Location::GPR(GPR::X0),
8654        )?;
8655        op(
8656            machine,
8657            Location::GPR(GPR::X0),
8658            Location::GPR(GPR::X0),
8659            Location::GPR(GPR::X0),
8660        )?;
8661        op(
8662            machine,
8663            Location::Imm32(10),
8664            Location::GPR(GPR::X2),
8665            Location::GPR(GPR::X0),
8666        )?;
8667        op(
8668            machine,
8669            Location::GPR(GPR::X0),
8670            Location::GPR(GPR::X2),
8671            Location::Memory(GPR::X0, 10),
8672        )?;
8673        op(
8674            machine,
8675            Location::GPR(GPR::X0),
8676            Location::Memory(GPR::X2, 16),
8677            Location::Memory(GPR::X0, 10),
8678        )?;
8679        op(
8680            machine,
8681            Location::Memory(GPR::X0, 0),
8682            Location::Memory(GPR::X2, 16),
8683            Location::Memory(GPR::X0, 10),
8684        )?;
8685
8686        Ok(())
8687    }
8688
8689    fn test_float_binop_op(
8690        machine: &mut MachineARM64,
8691        op: fn(&mut MachineARM64, Location, Location, Location) -> Result<(), CompileError>,
8692    ) -> Result<(), CompileError> {
8693        op(
8694            machine,
8695            Location::SIMD(NEON::V3),
8696            Location::SIMD(NEON::V2),
8697            Location::SIMD(NEON::V0),
8698        )?;
8699        op(
8700            machine,
8701            Location::SIMD(NEON::V0),
8702            Location::SIMD(NEON::V2),
8703            Location::SIMD(NEON::V0),
8704        )?;
8705        op(
8706            machine,
8707            Location::SIMD(NEON::V0),
8708            Location::SIMD(NEON::V0),
8709            Location::SIMD(NEON::V0),
8710        )?;
8711        op(
8712            machine,
8713            Location::Memory(GPR::X0, 0),
8714            Location::SIMD(NEON::V2),
8715            Location::SIMD(NEON::V0),
8716        )?;
8717        op(
8718            machine,
8719            Location::Memory(GPR::X0, 0),
8720            Location::Memory(GPR::X1, 10),
8721            Location::SIMD(NEON::V0),
8722        )?;
8723        op(
8724            machine,
8725            Location::Memory(GPR::X0, 0),
8726            Location::Memory(GPR::X1, 16),
8727            Location::Memory(GPR::X2, 32),
8728        )?;
8729        op(
8730            machine,
8731            Location::SIMD(NEON::V0),
8732            Location::Memory(GPR::X1, 16),
8733            Location::Memory(GPR::X2, 32),
8734        )?;
8735        op(
8736            machine,
8737            Location::SIMD(NEON::V0),
8738            Location::SIMD(NEON::V1),
8739            Location::Memory(GPR::X2, 32),
8740        )?;
8741
8742        Ok(())
8743    }
8744
8745    fn test_float_cmp_op(
8746        machine: &mut MachineARM64,
8747        op: fn(&mut MachineARM64, Location, Location, Location) -> Result<(), CompileError>,
8748    ) -> Result<(), CompileError> {
8749        op(
8750            machine,
8751            Location::SIMD(NEON::V3),
8752            Location::SIMD(NEON::V2),
8753            Location::GPR(GPR::X0),
8754        )?;
8755        op(
8756            machine,
8757            Location::SIMD(NEON::V0),
8758            Location::SIMD(NEON::V0),
8759            Location::GPR(GPR::X0),
8760        )?;
8761        op(
8762            machine,
8763            Location::Memory(GPR::X1, 0),
8764            Location::SIMD(NEON::V2),
8765            Location::GPR(GPR::X0),
8766        )?;
8767        op(
8768            machine,
8769            Location::Memory(GPR::X1, 0),
8770            Location::Memory(GPR::X2, 10),
8771            Location::GPR(GPR::X0),
8772        )?;
8773        op(
8774            machine,
8775            Location::Memory(GPR::X1, 0),
8776            Location::Memory(GPR::X2, 16),
8777            Location::Memory(GPR::X0, 32),
8778        )?;
8779        op(
8780            machine,
8781            Location::SIMD(NEON::V0),
8782            Location::Memory(GPR::X2, 16),
8783            Location::Memory(GPR::X0, 32),
8784        )?;
8785        op(
8786            machine,
8787            Location::SIMD(NEON::V0),
8788            Location::SIMD(NEON::V1),
8789            Location::Memory(GPR::X0, 32),
8790        )?;
8791
8792        Ok(())
8793    }
8794
8795    #[test]
8796    fn tests_arm64() -> Result<(), CompileError> {
8797        let mut machine = MachineARM64::new(None);
8798
8799        test_move_location(&mut machine, Size::S32)?;
8800        test_move_location(&mut machine, Size::S64)?;
8801        test_move_location_extended(&mut machine, false, Size::S8)?;
8802        test_move_location_extended(&mut machine, false, Size::S16)?;
8803        test_move_location_extended(&mut machine, false, Size::S32)?;
8804        test_move_location_extended(&mut machine, true, Size::S8)?;
8805        test_move_location_extended(&mut machine, true, Size::S16)?;
8806        test_move_location_extended(&mut machine, true, Size::S32)?;
8807        test_binop_op(&mut machine, MachineARM64::emit_binop_add32)?;
8808        test_binop_op(&mut machine, MachineARM64::emit_binop_add64)?;
8809        test_binop_op(&mut machine, MachineARM64::emit_binop_sub32)?;
8810        test_binop_op(&mut machine, MachineARM64::emit_binop_sub64)?;
8811        test_binop_op(&mut machine, MachineARM64::emit_binop_and32)?;
8812        test_binop_op(&mut machine, MachineARM64::emit_binop_and64)?;
8813        test_binop_op(&mut machine, MachineARM64::emit_binop_xor32)?;
8814        test_binop_op(&mut machine, MachineARM64::emit_binop_xor64)?;
8815        test_binop_op(&mut machine, MachineARM64::emit_binop_or32)?;
8816        test_binop_op(&mut machine, MachineARM64::emit_binop_or64)?;
8817        test_binop_op(&mut machine, MachineARM64::emit_binop_mul32)?;
8818        test_binop_op(&mut machine, MachineARM64::emit_binop_mul64)?;
8819        test_float_binop_op(&mut machine, MachineARM64::f32_add)?;
8820        test_float_binop_op(&mut machine, MachineARM64::f32_sub)?;
8821        test_float_binop_op(&mut machine, MachineARM64::f32_mul)?;
8822        test_float_binop_op(&mut machine, MachineARM64::f32_div)?;
8823        test_float_cmp_op(&mut machine, MachineARM64::f32_cmp_eq)?;
8824        test_float_cmp_op(&mut machine, MachineARM64::f32_cmp_lt)?;
8825        test_float_cmp_op(&mut machine, MachineARM64::f32_cmp_le)?;
8826
8827        Ok(())
8828    }
8829}