wasmer_compiler_singlepass/
emitter_arm64.rs

1pub use crate::{
2    arm64_decl::{ARM64Register, ArgumentRegisterAllocator, GPR, NEON},
3    location::{Multiplier, Reg},
4    machine::{Label, Offset},
5};
6use crate::{codegen_error, common_decl::Size, location::Location as AbstractLocation};
7use dynasm::dynasm;
8pub use dynasmrt::aarch64::{encode_logical_immediate_32bit, encode_logical_immediate_64bit};
9use dynasmrt::{
10    AssemblyOffset, DynamicLabel, DynasmApi, DynasmLabelApi, VecAssembler,
11    aarch64::Aarch64Relocation,
12};
13use wasmer_compiler::types::{
14    function::FunctionBody,
15    section::{CustomSection, CustomSectionProtection, SectionBody},
16};
17use wasmer_types::{
18    CompileError, FunctionIndex, FunctionType, Type, VMOffsets, target::CallingConvention,
19};
20
21type Assembler = VecAssembler<Aarch64Relocation>;
22
23/// Force `dynasm!` to use the correct arch (aarch64) when cross-compiling.
24/// `dynasm!` proc-macro tries to auto-detect it by default by looking at the
25/// `target_arch`, but it sees the `target_arch` of the proc-macro itself, which
26/// is always equal to host, even when cross-compiling.
27macro_rules! dynasm {
28    ($a:expr_2021 ; $($tt:tt)*) => {
29        dynasm::dynasm!(
30            $a
31            ; .arch aarch64
32            ; $($tt)*
33        )
34    };
35}
36
37pub type Location = AbstractLocation<GPR, NEON>;
38
39#[derive(Copy, Clone, Debug, Eq, PartialEq)]
40#[allow(dead_code)]
41#[repr(u8)]
42pub enum Condition {
43    // meaning for cmp or sub
44    /// Equal
45    Eq = 0,
46    /// Not equal
47    Ne = 1,
48    /// Unsigned higher or same (or carry set)
49    Cs = 2,
50    /// Unsigned lower (or carry clear)
51    Cc = 3,
52    /// Negative. The mnemonic stands for "minus"
53    Mi = 4,
54    /// Positive or zero. The mnemonic stands for "plus"
55    Pl = 5,
56    /// Signed overflow. The mnemonic stands for "V set"
57    Vs = 6,
58    /// No signed overflow. The mnemonic stands for "V clear"
59    Vc = 7,
60    /// Unsigned higher
61    Hi = 8,
62    /// Unsigned lower or same
63    Ls = 9,
64    /// Signed greater than or equal
65    Ge = 10,
66    /// Signed less than
67    Lt = 11,
68    /// Signed greater than
69    Gt = 12,
70    /// Signed less than or equal
71    Le = 13,
72    /// Always executed
73    Al = 14,
74}
75
76#[derive(Copy, Clone, Debug, Eq, PartialEq)]
77#[allow(dead_code, clippy::upper_case_acronyms)]
78pub enum NeonOrMemory {
79    NEON(NEON),
80    Memory(GPR, i32),
81}
82
83#[derive(Copy, Clone, Debug)]
84#[allow(dead_code, clippy::upper_case_acronyms)]
85pub enum GPROrMemory {
86    GPR(GPR),
87    Memory(GPR, i32),
88}
89
90#[allow(unused)]
91pub trait EmitterARM64 {
92    fn get_label(&mut self) -> Label;
93    fn get_offset(&self) -> Offset;
94    fn get_jmp_instr_size(&self) -> u8;
95
96    fn finalize_function(&mut self);
97
98    fn emit_str(&mut self, sz: Size, reg: Location, addr: Location) -> Result<(), CompileError>;
99    fn emit_ldr(&mut self, sz: Size, reg: Location, addr: Location) -> Result<(), CompileError>;
100    fn emit_stur(
101        &mut self,
102        sz: Size,
103        reg: Location,
104        addr: GPR,
105        offset: i32,
106    ) -> Result<(), CompileError>;
107    fn emit_ldur(
108        &mut self,
109        sz: Size,
110        reg: Location,
111        addr: GPR,
112        offset: i32,
113    ) -> Result<(), CompileError>;
114    fn emit_strdb(
115        &mut self,
116        sz: Size,
117        reg: Location,
118        addr: GPR,
119        offset: u32,
120    ) -> Result<(), CompileError>;
121    fn emit_stria(
122        &mut self,
123        sz: Size,
124        reg: Location,
125        addr: GPR,
126        offset: u32,
127    ) -> Result<(), CompileError>;
128    fn emit_ldria(
129        &mut self,
130        sz: Size,
131        reg: Location,
132        addr: GPR,
133        offset: u32,
134    ) -> Result<(), CompileError>;
135    fn emit_stpdb(
136        &mut self,
137        sz: Size,
138        reg1: Location,
139        reg2: Location,
140        addr: GPR,
141        offset: u32,
142    ) -> Result<(), CompileError>;
143    fn emit_ldpia(
144        &mut self,
145        sz: Size,
146        reg1: Location,
147        reg2: Location,
148        addr: GPR,
149        offset: u32,
150    ) -> Result<(), CompileError>;
151
152    fn emit_ldrb(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
153    fn emit_ldrh(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
154    fn emit_ldrsb(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
155    fn emit_ldrsh(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
156    fn emit_ldrsw(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
157    fn emit_strb(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
158    fn emit_strh(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
159
160    fn emit_ldaxr(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
161    fn emit_ldaxrb(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
162    fn emit_ldaxrh(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
163    fn emit_stlxr(
164        &mut self,
165        sz: Size,
166        status: Location,
167        reg: Location,
168        dst: Location,
169    ) -> Result<(), CompileError>;
170    fn emit_stlxrb(
171        &mut self,
172        sz: Size,
173        status: Location,
174        reg: Location,
175        dst: Location,
176    ) -> Result<(), CompileError>;
177    fn emit_stlxrh(
178        &mut self,
179        sz: Size,
180        status: Location,
181        reg: Location,
182        dst: Location,
183    ) -> Result<(), CompileError>;
184
185    fn emit_mov(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
186
187    fn emit_movn(&mut self, sz: Size, reg: Location, val: u32) -> Result<(), CompileError>;
188    fn emit_movz(&mut self, reg: Location, val: u32) -> Result<(), CompileError>;
189    fn emit_movk(&mut self, reg: Location, val: u32, shift: u32) -> Result<(), CompileError>;
190
191    fn emit_mov_imm(&mut self, dst: Location, val: u64) -> Result<(), CompileError>;
192
193    fn emit_add(
194        &mut self,
195        sz: Size,
196        src1: Location,
197        src2: Location,
198        dst: Location,
199    ) -> Result<(), CompileError>;
200    fn emit_sub(
201        &mut self,
202        sz: Size,
203        src1: Location,
204        src2: Location,
205        dst: Location,
206    ) -> Result<(), CompileError>;
207    fn emit_mul(
208        &mut self,
209        sz: Size,
210        src1: Location,
211        src2: Location,
212        dst: Location,
213    ) -> Result<(), CompileError>;
214    fn emit_adds(
215        &mut self,
216        sz: Size,
217        src1: Location,
218        src2: Location,
219        dst: Location,
220    ) -> Result<(), CompileError>;
221    fn emit_subs(
222        &mut self,
223        sz: Size,
224        src1: Location,
225        src2: Location,
226        dst: Location,
227    ) -> Result<(), CompileError>;
228
229    fn emit_add_lsl(
230        &mut self,
231        sz: Size,
232        src1: Location,
233        src2: Location,
234        lsl: u32,
235        dst: Location,
236    ) -> Result<(), CompileError>;
237
238    fn emit_cmp(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
239    fn emit_tst(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
240
241    fn emit_lsl(
242        &mut self,
243        sz: Size,
244        src1: Location,
245        src2: Location,
246        dst: Location,
247    ) -> Result<(), CompileError>;
248    fn emit_lsr(
249        &mut self,
250        sz: Size,
251        src1: Location,
252        src2: Location,
253        dst: Location,
254    ) -> Result<(), CompileError>;
255    fn emit_asr(
256        &mut self,
257        sz: Size,
258        src1: Location,
259        src2: Location,
260        dst: Location,
261    ) -> Result<(), CompileError>;
262    fn emit_ror(
263        &mut self,
264        sz: Size,
265        src1: Location,
266        src2: Location,
267        dst: Location,
268    ) -> Result<(), CompileError>;
269
270    fn emit_or(
271        &mut self,
272        sz: Size,
273        src1: Location,
274        src2: Location,
275        dst: Location,
276    ) -> Result<(), CompileError>;
277    fn emit_and(
278        &mut self,
279        sz: Size,
280        src1: Location,
281        src2: Location,
282        dst: Location,
283    ) -> Result<(), CompileError>;
284    fn emit_eor(
285        &mut self,
286        sz: Size,
287        src1: Location,
288        src2: Location,
289        dst: Location,
290    ) -> Result<(), CompileError>;
291
292    fn emit_bfc(
293        &mut self,
294        se: Size,
295        lsb: u32,
296        width: u32,
297        dst: Location,
298    ) -> Result<(), CompileError>;
299    fn emit_bfi(
300        &mut self,
301        se: Size,
302        src: Location,
303        lsb: u32,
304        width: u32,
305        dst: Location,
306    ) -> Result<(), CompileError>;
307
308    fn emit_udiv(
309        &mut self,
310        sz: Size,
311        src1: Location,
312        src2: Location,
313        dst: Location,
314    ) -> Result<(), CompileError>;
315    fn emit_sdiv(
316        &mut self,
317        sz: Size,
318        src1: Location,
319        src2: Location,
320        dst: Location,
321    ) -> Result<(), CompileError>;
322    /// msub : c - a*b -> dst
323    fn emit_msub(
324        &mut self,
325        sz: Size,
326        a: Location,
327        b: Location,
328        c: Location,
329        dst: Location,
330    ) -> Result<(), CompileError>;
331
332    fn emit_sxtb(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
333    fn emit_sxth(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
334    fn emit_sxtw(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
335    fn emit_uxtb(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
336    fn emit_uxth(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
337
338    fn emit_cset(&mut self, sz: Size, dst: Location, cond: Condition) -> Result<(), CompileError>;
339    fn emit_csetm(&mut self, sz: Size, dst: Location, cond: Condition) -> Result<(), CompileError>;
340    fn emit_cinc(
341        &mut self,
342        sz: Size,
343        src: Location,
344        dst: Location,
345        cond: Condition,
346    ) -> Result<(), CompileError>;
347    fn emit_clz(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
348    fn emit_rbit(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
349
350    fn emit_label(&mut self, label: Label) -> Result<(), CompileError>;
351    fn emit_load_label(&mut self, reg: GPR, label: Label) -> Result<(), CompileError>;
352    fn emit_b_label(&mut self, label: Label) -> Result<(), CompileError>;
353    fn emit_cbz_label(&mut self, sz: Size, reg: Location, label: Label)
354    -> Result<(), CompileError>;
355    fn emit_cbnz_label(
356        &mut self,
357        sz: Size,
358        reg: Location,
359        label: Label,
360    ) -> Result<(), CompileError>;
361    fn emit_cbz_label_far(
362        &mut self,
363        sz: Size,
364        reg: Location,
365        label: Label,
366    ) -> Result<(), CompileError>;
367    fn emit_tbz_label(
368        &mut self,
369        sz: Size,
370        reg: Location,
371        n: u32,
372        label: Label,
373    ) -> Result<(), CompileError>;
374    fn emit_tbnz_label(
375        &mut self,
376        sz: Size,
377        reg: Location,
378        n: u32,
379        label: Label,
380    ) -> Result<(), CompileError>;
381    fn emit_bcond_label(&mut self, condition: Condition, label: Label) -> Result<(), CompileError>;
382    fn emit_bcond_label_far(
383        &mut self,
384        condition: Condition,
385        label: Label,
386    ) -> Result<(), CompileError>;
387    fn emit_b_register(&mut self, reg: GPR) -> Result<(), CompileError>;
388    fn emit_call_label(&mut self, label: Label) -> Result<(), CompileError>;
389    fn emit_call_register(&mut self, reg: GPR) -> Result<(), CompileError>;
390    fn emit_ret(&mut self) -> Result<(), CompileError>;
391
392    fn emit_udf(&mut self, payload: u16) -> Result<(), CompileError>;
393    fn emit_dmb(&mut self) -> Result<(), CompileError>;
394    fn emit_brk(&mut self) -> Result<(), CompileError>;
395
396    fn emit_fcmp(&mut self, sz: Size, src1: Location, src2: Location) -> Result<(), CompileError>;
397    fn emit_fneg(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
398    fn emit_fsqrt(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
399
400    fn emit_fadd(
401        &mut self,
402        sz: Size,
403        src1: Location,
404        src2: Location,
405        dst: Location,
406    ) -> Result<(), CompileError>;
407    fn emit_fsub(
408        &mut self,
409        sz: Size,
410        src1: Location,
411        src2: Location,
412        dst: Location,
413    ) -> Result<(), CompileError>;
414    fn emit_fmul(
415        &mut self,
416        sz: Size,
417        src1: Location,
418        src2: Location,
419        dst: Location,
420    ) -> Result<(), CompileError>;
421    fn emit_fdiv(
422        &mut self,
423        sz: Size,
424        src1: Location,
425        src2: Location,
426        dst: Location,
427    ) -> Result<(), CompileError>;
428
429    fn emit_fmin(
430        &mut self,
431        sz: Size,
432        src1: Location,
433        src2: Location,
434        dst: Location,
435    ) -> Result<(), CompileError>;
436    fn emit_fmax(
437        &mut self,
438        sz: Size,
439        src1: Location,
440        src2: Location,
441        dst: Location,
442    ) -> Result<(), CompileError>;
443
444    fn emit_frintz(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
445    fn emit_frintn(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
446    fn emit_frintm(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
447    fn emit_frintp(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
448
449    fn emit_scvtf(
450        &mut self,
451        sz_in: Size,
452        src: Location,
453        sz_out: Size,
454        dst: Location,
455    ) -> Result<(), CompileError>;
456    fn emit_ucvtf(
457        &mut self,
458        sz_in: Size,
459        src: Location,
460        sz_out: Size,
461        dst: Location,
462    ) -> Result<(), CompileError>;
463    fn emit_fcvt(&mut self, sz_in: Size, src: Location, dst: Location) -> Result<(), CompileError>;
464    fn emit_fcvtzs(
465        &mut self,
466        sz_in: Size,
467        src: Location,
468        sz_out: Size,
469        dst: Location,
470    ) -> Result<(), CompileError>;
471    fn emit_fcvtzu(
472        &mut self,
473        sz_in: Size,
474        src: Location,
475        sz_out: Size,
476        dst: Location,
477    ) -> Result<(), CompileError>;
478
479    fn emit_fmov(
480        &mut self,
481        src_size: Size,
482        src: Location,
483        dst_size: Size,
484        dst: Location,
485    ) -> Result<(), CompileError>;
486    fn emit_cnt(&mut self, src: NEON, dst: NEON) -> Result<(), CompileError>;
487    fn emit_addv(&mut self, src: NEON, dst: NEON) -> Result<(), CompileError>;
488    fn emit_read_fpcr(&mut self, reg: GPR) -> Result<(), CompileError>;
489    fn emit_write_fpcr(&mut self, reg: GPR) -> Result<(), CompileError>;
490    fn emit_read_fpsr(&mut self, reg: GPR) -> Result<(), CompileError>;
491    fn emit_write_fpsr(&mut self, reg: GPR) -> Result<(), CompileError>;
492
493    fn arch_supports_canonicalize_nan(&self) -> bool {
494        true
495    }
496
497    fn arch_requires_indirect_call_trampoline(&self) -> bool {
498        false
499    }
500
501    fn arch_emit_indirect_call_with_trampoline(
502        &mut self,
503        _loc: Location,
504    ) -> Result<(), CompileError> {
505        codegen_error!("singlepass arch_emit_indirect_call_with_trampoline unimplemented")
506    }
507}
508
509impl EmitterARM64 for Assembler {
510    fn get_label(&mut self) -> DynamicLabel {
511        self.new_dynamic_label()
512    }
513
514    fn get_offset(&self) -> AssemblyOffset {
515        self.offset()
516    }
517
518    fn get_jmp_instr_size(&self) -> u8 {
519        4 // relative jump, not full 32bits capable
520    }
521
522    fn finalize_function(&mut self) {
523        dynasm!(
524            self
525            ; const_neg_one_32:
526            ; .i32 -1
527            ; const_zero_32:
528            ; .i32 0
529            ; const_pos_one_32:
530            ; .i32 1
531        );
532    }
533
534    fn emit_str(&mut self, sz: Size, reg: Location, addr: Location) -> Result<(), CompileError> {
535        match (sz, reg, addr) {
536            (Size::S64, Location::GPR(reg), Location::Memory(addr, disp)) => {
537                let disp = disp as u32;
538                assert!((disp & 0x7) == 0 && (disp < 0x8000));
539                dynasm!(self ; str X(reg), [X(addr), disp]);
540            }
541            (Size::S32, Location::GPR(reg), Location::Memory(addr, disp)) => {
542                let disp = disp as u32;
543                assert!((disp & 0x3) == 0 && (disp < 0x4000));
544                dynasm!(self ; str W(reg), [X(addr), disp]);
545            }
546            (Size::S16, Location::GPR(reg), Location::Memory(addr, disp)) => {
547                let disp = disp as u32;
548                assert!((disp & 0x1) == 0 && (disp < 0x2000));
549                dynasm!(self ; strh W(reg), [X(addr), disp]);
550            }
551            (Size::S8, Location::GPR(reg), Location::Memory(addr, disp)) => {
552                let disp = disp as u32;
553                assert!(disp < 0x1000);
554                dynasm!(self ; strb W(reg), [X(addr), disp]);
555            }
556            (Size::S64, Location::SIMD(reg), Location::Memory(addr, disp)) => {
557                let disp = disp as u32;
558                assert!((disp & 0x7) == 0 && (disp < 0x8000));
559                dynasm!(self ; str D(reg), [X(addr), disp]);
560            }
561            (Size::S32, Location::SIMD(reg), Location::Memory(addr, disp)) => {
562                let disp = disp as u32;
563                assert!((disp & 0x3) == 0 && (disp < 0x4000));
564                dynasm!(self ; str S(reg), [X(addr), disp]);
565            }
566            (Size::S64, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
567                assert!(offs == 0);
568                let mult = mult as u32;
569                match mult {
570                    0 => dynasm!(self ; str X(reg), [X(addr)]),
571                    1 => dynasm!(self ; str X(reg), [X(addr), X(r2)]),
572                    _ => dynasm!(self ; str X(reg), [X(addr), X(r2), LSL mult]),
573                };
574            }
575            (Size::S32, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
576                assert!(offs == 0);
577                let mult = mult as u32;
578                match mult {
579                    0 => dynasm!(self ; str W(reg), [X(addr)]),
580                    1 => dynasm!(self ; str W(reg), [X(addr), X(r2)]),
581                    _ => dynasm!(self ; str W(reg), [X(addr), X(r2), LSL mult]),
582                };
583            }
584            _ => codegen_error!("singlepass can't emit STR {:?}, {:?}, {:?}", sz, reg, addr),
585        }
586        Ok(())
587    }
588    fn emit_ldr(&mut self, sz: Size, reg: Location, addr: Location) -> Result<(), CompileError> {
589        match (sz, reg, addr) {
590            (Size::S64, Location::GPR(reg), Location::Memory(addr, disp)) => {
591                assert!((disp & 0x7) == 0 && (disp < 0x8000));
592                let disp = disp as u32;
593                dynasm!(self ; ldr X(reg), [X(addr), disp]);
594            }
595            (Size::S32, Location::GPR(reg), Location::Memory(addr, disp)) => {
596                assert!((disp & 0x3) == 0 && (disp < 0x4000));
597                let disp = disp as u32;
598                dynasm!(self ; ldr W(reg), [X(addr), disp]);
599            }
600            (Size::S16, Location::GPR(reg), Location::Memory(addr, disp)) => {
601                assert!((disp & 0x1 == 0) && (disp < 0x2000));
602                let disp = disp as u32;
603                dynasm!(self ; ldrh W(reg), [X(addr), disp]);
604            }
605            (Size::S8, Location::GPR(reg), Location::Memory(addr, disp)) => {
606                assert!(disp < 0x1000);
607                let disp = disp as u32;
608                dynasm!(self ; ldrb W(reg), [X(addr), disp]);
609            }
610            (Size::S64, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
611                assert!(offs == 0);
612                let mult = mult as u32;
613                match mult {
614                    0 => dynasm!(self ; ldr X(reg), [X(addr)]),
615                    1 => dynasm!(self ; ldr X(reg), [X(addr), X(r2)]),
616                    _ => dynasm!(self ; ldr X(reg), [X(addr), X(r2), LSL mult]),
617                };
618            }
619            (Size::S32, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
620                assert!(offs == 0);
621                let mult = mult as u32;
622                match mult {
623                    0 => dynasm!(self ; ldr W(reg), [X(addr)]),
624                    1 => dynasm!(self ; ldr W(reg), [X(addr), X(r2)]),
625                    _ => dynasm!(self ; ldr W(reg), [X(addr), X(r2), LSL mult]),
626                };
627            }
628            (Size::S64, Location::SIMD(reg), Location::Memory(addr, disp)) => {
629                let disp = disp as u32;
630                assert!((disp & 0x7) == 0 && (disp < 0x8000));
631                dynasm!(self ; ldr D(reg), [X(addr), disp]);
632            }
633            (Size::S32, Location::SIMD(reg), Location::Memory(addr, disp)) => {
634                let disp = disp as u32;
635                assert!((disp & 0x3) == 0 && (disp < 0x4000));
636                dynasm!(self ; ldr S(reg), [X(addr), disp]);
637            }
638            (Size::S64, Location::SIMD(reg), Location::Memory2(addr, r2, mult, offs)) => {
639                assert!(offs == 0);
640                let mult = mult as u32;
641                match mult {
642                    0 => dynasm!(self ; ldr D(reg), [X(addr)]),
643                    1 => dynasm!(self ; ldr D(reg), [X(addr), X(r2)]),
644                    _ => dynasm!(self ; ldr D(reg), [X(addr), X(r2), LSL mult]),
645                };
646            }
647            (Size::S32, Location::SIMD(reg), Location::Memory2(addr, r2, mult, offs)) => {
648                assert!(offs == 0);
649                let mult = mult as u32;
650                match mult {
651                    0 => dynasm!(self ; ldr S(reg), [X(addr)]),
652                    1 => dynasm!(self ; ldr S(reg), [X(addr), X(r2)]),
653                    _ => dynasm!(self ; ldr S(reg), [X(addr), X(r2), LSL mult]),
654                };
655            }
656            _ => codegen_error!("singlepass can't emit LDR {:?}, {:?}, {:?}", sz, reg, addr),
657        }
658        Ok(())
659    }
660    fn emit_stur(
661        &mut self,
662        sz: Size,
663        reg: Location,
664        addr: GPR,
665        offset: i32,
666    ) -> Result<(), CompileError> {
667        assert!((-255..=255).contains(&offset));
668        match (sz, reg) {
669            (Size::S64, Location::GPR(reg)) => {
670                dynasm!(self ; stur X(reg), [X(addr), offset]);
671            }
672            (Size::S32, Location::GPR(reg)) => {
673                dynasm!(self ; stur W(reg), [X(addr), offset]);
674            }
675            (Size::S64, Location::SIMD(reg)) => {
676                dynasm!(self ; stur D(reg), [X(addr), offset]);
677            }
678            (Size::S32, Location::SIMD(reg)) => {
679                dynasm!(self ; stur S(reg), [X(addr), offset]);
680            }
681            _ => codegen_error!(
682                "singlepass can't emit STUR {:?}, {:?}, {:?}, {:?}",
683                sz,
684                reg,
685                addr,
686                offset
687            ),
688        }
689        Ok(())
690    }
691    fn emit_ldur(
692        &mut self,
693        sz: Size,
694        reg: Location,
695        addr: GPR,
696        offset: i32,
697    ) -> Result<(), CompileError> {
698        assert!((-255..=255).contains(&offset));
699        match (sz, reg) {
700            (Size::S64, Location::GPR(reg)) => {
701                dynasm!(self ; ldur X(reg), [X(addr), offset]);
702            }
703            (Size::S32, Location::GPR(reg)) => {
704                dynasm!(self ; ldur W(reg), [X(addr), offset]);
705            }
706            (Size::S64, Location::SIMD(reg)) => {
707                dynasm!(self ; ldur D(reg), [X(addr), offset]);
708            }
709            (Size::S32, Location::SIMD(reg)) => {
710                dynasm!(self ; ldur S(reg), [X(addr), offset]);
711            }
712            _ => codegen_error!(
713                "singlepass can't emit LDUR {:?}, {:?}, {:?}, {:?}",
714                sz,
715                reg,
716                addr,
717                offset
718            ),
719        }
720        Ok(())
721    }
722
723    fn emit_strdb(
724        &mut self,
725        sz: Size,
726        reg: Location,
727        addr: GPR,
728        offset: u32,
729    ) -> Result<(), CompileError> {
730        assert!(offset <= 255);
731        match (sz, reg) {
732            (Size::S64, Location::GPR(reg)) => {
733                dynasm!(self ; str X(reg), [X(addr), -(offset as i32)]!);
734            }
735            (Size::S64, Location::SIMD(reg)) => {
736                dynasm!(self ; str D(reg), [X(addr), -(offset as i32)]!);
737            }
738            _ => codegen_error!("singlepass can't emit STRDB"),
739        }
740        Ok(())
741    }
742    fn emit_stria(
743        &mut self,
744        sz: Size,
745        reg: Location,
746        addr: GPR,
747        offset: u32,
748    ) -> Result<(), CompileError> {
749        assert!(offset <= 255);
750        match (sz, reg) {
751            (Size::S64, Location::GPR(reg)) => {
752                dynasm!(self ; str X(reg), [X(addr)], offset as i32);
753            }
754            (Size::S64, Location::SIMD(reg)) => {
755                dynasm!(self ; str D(reg), [X(addr)], offset as i32);
756            }
757            _ => codegen_error!("singlepass can't emit STRIA"),
758        }
759        Ok(())
760    }
761    fn emit_ldria(
762        &mut self,
763        sz: Size,
764        reg: Location,
765        addr: GPR,
766        offset: u32,
767    ) -> Result<(), CompileError> {
768        assert!(offset <= 255);
769        match (sz, reg) {
770            (Size::S64, Location::GPR(reg)) => {
771                dynasm!(self ; ldr X(reg), [X(addr)], offset as _);
772            }
773            (Size::S64, Location::SIMD(reg)) => {
774                dynasm!(self ; ldr D(reg), [X(addr)], offset as _);
775            }
776            _ => codegen_error!("singlepass can't emit LDRIA"),
777        }
778        Ok(())
779    }
780
781    fn emit_stpdb(
782        &mut self,
783        sz: Size,
784        reg1: Location,
785        reg2: Location,
786        addr: GPR,
787        offset: u32,
788    ) -> Result<(), CompileError> {
789        assert!(offset <= 255);
790        match (sz, reg1, reg2) {
791            (Size::S64, Location::GPR(reg1), Location::GPR(reg2)) => {
792                dynasm!(self ; stp X(reg1), X(reg2), [X(addr), -(offset as i32)]!);
793            }
794            _ => codegen_error!("singlepass can't emit STPDB"),
795        }
796        Ok(())
797    }
798    fn emit_ldpia(
799        &mut self,
800        sz: Size,
801        reg1: Location,
802        reg2: Location,
803        addr: GPR,
804        offset: u32,
805    ) -> Result<(), CompileError> {
806        assert!(offset <= 255);
807        match (sz, reg1, reg2) {
808            (Size::S64, Location::GPR(reg1), Location::GPR(reg2)) => {
809                dynasm!(self ; ldp X(reg1), X(reg2), [X(addr)], offset as _);
810            }
811            _ => codegen_error!("singlepass can't emit LDPIA"),
812        }
813        Ok(())
814    }
815
816    fn emit_ldrb(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
817        match (reg, dst) {
818            (Location::GPR(reg), Location::Memory(addr, offset)) => {
819                let offset = offset as u32;
820                assert!(offset < 0x1000);
821                dynasm!(self ; ldrb W(reg), [X(addr), offset]);
822            }
823            (Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
824                assert!(offs == 0);
825                let mult = mult as u32;
826                match mult {
827                    0 => dynasm!(self ; ldrb W(reg), [X(addr)]),
828                    1 => dynasm!(self ; ldrb W(reg), [X(addr), X(r2)]),
829                    _ => dynasm!(self ; ldrb W(reg), [X(addr), X(r2), LSL mult]),
830                };
831            }
832            _ => codegen_error!("singlepass can't emit LDRB {:?}, {:?}", reg, dst),
833        }
834        Ok(())
835    }
836    fn emit_ldrh(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
837        match (reg, dst) {
838            (Location::GPR(reg), Location::Memory(addr, offset)) => {
839                let offset = offset as u32;
840                assert!((offset & 1 == 0) && (offset < 0x2000));
841                dynasm!(self ; ldrh W(reg), [X(addr), offset]);
842            }
843            (Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
844                assert!(offs == 0);
845                let mult = mult as u32;
846                match mult {
847                    0 => dynasm!(self ; ldrh W(reg), [X(addr)]),
848                    1 => dynasm!(self ; ldrh W(reg), [X(addr), X(r2)]),
849                    _ => dynasm!(self ; ldrh W(reg), [X(addr), X(r2), LSL mult]),
850                };
851            }
852            _ => codegen_error!("singlepass can't emit LDRH {:?}, {:?}", reg, dst),
853        }
854        Ok(())
855    }
856    fn emit_ldrsb(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
857        match (sz, reg, dst) {
858            (Size::S64, Location::GPR(reg), Location::Memory(addr, offset)) => {
859                let offset = offset as u32;
860                assert!(offset < 0x1000);
861                dynasm!(self ; ldrsb X(reg), [X(addr), offset]);
862            }
863            (Size::S32, Location::GPR(reg), Location::Memory(addr, offset)) => {
864                let offset = offset as u32;
865                assert!(offset < 0x1000);
866                dynasm!(self ; ldrsb W(reg), [X(addr), offset]);
867            }
868            (Size::S64, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
869                assert!(offs == 0);
870                let mult = mult as u32;
871                match mult {
872                    0 => dynasm!(self ; ldrsb X(reg), [X(addr)]),
873                    1 => dynasm!(self ; ldrsb X(reg), [X(addr), X(r2)]),
874                    _ => dynasm!(self ; ldrsb X(reg), [X(addr), X(r2), LSL mult]),
875                };
876            }
877            (Size::S32, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
878                assert!(offs == 0);
879                let mult = mult as u32;
880                match mult {
881                    0 => dynasm!(self ; ldrsb W(reg), [X(addr)]),
882                    1 => dynasm!(self ; ldrsb W(reg), [X(addr), X(r2)]),
883                    _ => dynasm!(self ; ldrsb W(reg), [X(addr), X(r2), LSL mult]),
884                };
885            }
886            _ => codegen_error!("singlepass can't emit LDRSB {:?}, {:?}, {:?}", sz, reg, dst),
887        }
888        Ok(())
889    }
890    fn emit_ldrsh(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
891        match (sz, reg, dst) {
892            (Size::S64, Location::GPR(reg), Location::Memory(addr, offset)) => {
893                let offset = offset as u32;
894                assert!((offset & 1 == 0) && (offset < 0x2000));
895                dynasm!(self ; ldrsh X(reg), [X(addr), offset]);
896            }
897            (Size::S32, Location::GPR(reg), Location::Memory(addr, offset)) => {
898                let offset = offset as u32;
899                assert!((offset & 1 == 0) && (offset < 0x2000));
900                dynasm!(self ; ldrsh W(reg), [X(addr), offset]);
901            }
902            (Size::S64, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
903                assert!(offs == 0);
904                let mult = mult as u32;
905                match mult {
906                    0 => dynasm!(self ; ldrsh X(reg), [X(addr)]),
907                    1 => dynasm!(self ; ldrsh X(reg), [X(addr), X(r2)]),
908                    _ => dynasm!(self ; ldrsh X(reg), [X(addr), X(r2), LSL mult]),
909                };
910            }
911            (Size::S32, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
912                assert!(offs == 0);
913                let mult = mult as u32;
914                match mult {
915                    0 => dynasm!(self ; ldrsh W(reg), [X(addr)]),
916                    1 => dynasm!(self ; ldrsh W(reg), [X(addr), X(r2)]),
917                    _ => dynasm!(self ; ldrsh W(reg), [X(addr), X(r2), LSL mult]),
918                };
919            }
920            _ => codegen_error!("singlepass can't emit LDRSH {:?}, {:?}, {:?}", sz, reg, dst),
921        }
922        Ok(())
923    }
924    fn emit_ldrsw(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
925        match (sz, reg, dst) {
926            (Size::S64, Location::GPR(reg), Location::Memory(addr, offset)) => {
927                let offset = offset as u32;
928                assert!((offset & 3 == 0) && (offset < 0x4000));
929                dynasm!(self ; ldrsw X(reg), [X(addr), offset]);
930            }
931            (Size::S64, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
932                assert!(offs == 0);
933                let mult = mult as u32;
934                match mult {
935                    0 => dynasm!(self ; ldrsw X(reg), [X(addr)]),
936                    1 => dynasm!(self ; ldrsw X(reg), [X(addr), X(r2)]),
937                    _ => dynasm!(self ; ldrsw X(reg), [X(addr), X(r2), LSL mult]),
938                };
939            }
940            _ => codegen_error!("singlepass can't emit LDRSW {:?}, {:?}, {:?}", sz, reg, dst),
941        }
942        Ok(())
943    }
944    fn emit_strb(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
945        match (reg, dst) {
946            (Location::GPR(reg), Location::Memory(addr, offset)) => {
947                let offset = offset as u32;
948                assert!(offset < 0x1000);
949                dynasm!(self ; strb W(reg), [X(addr), offset]);
950            }
951            (Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
952                assert!(offs == 0);
953                let mult = mult as u32;
954                match mult {
955                    0 => dynasm!(self ; strb W(reg), [X(addr)]),
956                    1 => dynasm!(self ; strb W(reg), [X(addr), X(r2)]),
957                    _ => dynasm!(self ; strb W(reg), [X(addr), X(r2), LSL mult]),
958                };
959            }
960            _ => codegen_error!("singlepass can't emit STRB {:?}, {:?}", reg, dst),
961        }
962        Ok(())
963    }
964    fn emit_strh(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
965        match (reg, dst) {
966            (Location::GPR(reg), Location::Memory(addr, offset)) => {
967                let offset = offset as u32;
968                assert!((offset & 1 == 0) && (offset < 0x2000));
969                dynasm!(self ; strh W(reg), [X(addr), offset]);
970            }
971            (Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
972                assert!(offs == 0);
973                let mult = mult as u32;
974                match mult {
975                    0 => dynasm!(self ; strh W(reg), [X(addr)]),
976                    1 => dynasm!(self ; strh W(reg), [X(addr), X(r2)]),
977                    _ => dynasm!(self ; strh W(reg), [X(addr), X(r2), LSL mult]),
978                };
979            }
980            _ => codegen_error!("singlepass can't emit STRH {:?}, {:?}", reg, dst),
981        }
982        Ok(())
983    }
984
985    fn emit_ldaxr(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
986        match (sz, reg, dst) {
987            (Size::S32, Location::GPR(reg), Location::GPR(dst)) => {
988                dynasm!(self ; ldaxr W(reg), [X(dst)]);
989            }
990            (Size::S64, Location::GPR(reg), Location::GPR(dst)) => {
991                dynasm!(self ; ldaxr X(reg), [X(dst)]);
992            }
993            _ => codegen_error!("singlepass can't emit LDAXR {:?}, {:?}", reg, dst),
994        }
995        Ok(())
996    }
997    fn emit_ldaxrb(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
998        match (reg, dst) {
999            (Location::GPR(reg), Location::GPR(dst)) => {
1000                dynasm!(self ; ldaxrb W(reg), [X(dst)]);
1001            }
1002            _ => codegen_error!("singlepass can't emit LDAXRB {:?}, {:?}", reg, dst),
1003        }
1004        Ok(())
1005    }
1006    fn emit_ldaxrh(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
1007        match (reg, dst) {
1008            (Location::GPR(reg), Location::GPR(dst)) => {
1009                dynasm!(self ; ldaxrh W(reg), [X(dst)]);
1010            }
1011            _ => codegen_error!("singlepass can't emit LDAXRH {:?}, {:?}", reg, dst),
1012        }
1013        Ok(())
1014    }
1015    fn emit_stlxr(
1016        &mut self,
1017        sz: Size,
1018        status: Location,
1019        reg: Location,
1020        dst: Location,
1021    ) -> Result<(), CompileError> {
1022        match (sz, status, reg, dst) {
1023            (Size::S32, Location::GPR(status), Location::GPR(reg), Location::GPR(dst)) => {
1024                dynasm!(self ; stlxr W(status), W(reg), [X(dst)]);
1025            }
1026            (Size::S64, Location::GPR(status), Location::GPR(reg), Location::GPR(dst)) => {
1027                dynasm!(self ; stlxr W(status), X(reg), [X(dst)]);
1028            }
1029            _ => codegen_error!("singlepass can't emit STLXR {:?}, {:?}", reg, dst),
1030        }
1031        Ok(())
1032    }
1033    fn emit_stlxrb(
1034        &mut self,
1035        _sz: Size,
1036        status: Location,
1037        reg: Location,
1038        dst: Location,
1039    ) -> Result<(), CompileError> {
1040        match (status, reg, dst) {
1041            (Location::GPR(status), Location::GPR(reg), Location::GPR(dst)) => {
1042                dynasm!(self ; stlxrb W(status), W(reg), [X(dst)]);
1043            }
1044            _ => codegen_error!("singlepass can't emit STLXRB {:?}, {:?}", reg, dst),
1045        }
1046        Ok(())
1047    }
1048    fn emit_stlxrh(
1049        &mut self,
1050        _sz: Size,
1051        status: Location,
1052        reg: Location,
1053        dst: Location,
1054    ) -> Result<(), CompileError> {
1055        match (status, reg, dst) {
1056            (Location::GPR(status), Location::GPR(reg), Location::GPR(dst)) => {
1057                dynasm!(self ; stlxrh W(status), W(reg), [X(dst)]);
1058            }
1059            _ => codegen_error!("singlepass can't emit STLXRH {:?}, {:?}", reg, dst),
1060        }
1061        Ok(())
1062    }
1063
1064    fn emit_mov(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1065        match (sz, src, dst) {
1066            (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1067                dynasm!(self ; mov X(dst), X(src));
1068            }
1069            (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1070                dynasm!(self ; mov W(dst), W(src));
1071            }
1072            (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
1073                dynasm!(self ; mov V(dst).D[0], V(src).D[0]);
1074            }
1075            (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
1076                dynasm!(self ; mov V(dst).S[0], V(src).S[0]);
1077            }
1078            (Size::S64, Location::GPR(src), Location::SIMD(dst)) => {
1079                dynasm!(self ; mov V(dst).D[0], X(src));
1080            }
1081            (Size::S32, Location::GPR(src), Location::SIMD(dst)) => {
1082                dynasm!(self ; mov V(dst).S[0], W(src));
1083            }
1084            (Size::S64, Location::SIMD(src), Location::GPR(dst)) => {
1085                dynasm!(self ; mov X(dst), V(src).D[0]);
1086            }
1087            (Size::S32, Location::SIMD(src), Location::GPR(dst)) => {
1088                dynasm!(self ; mov W(dst), V(src).S[0]);
1089            }
1090            (Size::S32, Location::Imm32(val), Location::GPR(dst)) => {
1091                if val < 0x1000 || encode_logical_immediate_32bit(val as _).is_some() {
1092                    dynasm!(self ; mov W(dst), val);
1093                } else {
1094                    dynasm!(self
1095                        ; movz W(dst), val
1096                        ; movk W(dst), val, lsl #16);
1097                }
1098            }
1099            (Size::S64, Location::Imm32(val), Location::GPR(dst)) => {
1100                if val < 0x1000 {
1101                    dynasm!(self ; mov W(dst), val);
1102                } else if encode_logical_immediate_64bit(val as _).is_some() {
1103                    dynasm!(self ; mov X(dst), val as _);
1104                } else {
1105                    dynasm!(self
1106                        ; movz X(dst), val
1107                        ; movk X(dst), val, lsl #16);
1108                }
1109            }
1110            (Size::S64, Location::Imm64(val), Location::GPR(dst)) => {
1111                if val < 0x1000 || encode_logical_immediate_64bit(val as _).is_some() {
1112                    dynasm!(self ; mov X(dst), val as _);
1113                } else {
1114                    dynasm!(self
1115                        ; movz X(dst), val as _
1116                        ; movk X(dst), val as _, lsl #16
1117                        ; movk X(dst), val as _, lsl #32
1118                        ; movk X(dst), val as _, lsl #48);
1119                }
1120            }
1121            _ => codegen_error!("singlepass can't emit MOV {:?}, {:?}, {:?}", sz, src, dst),
1122        }
1123        Ok(())
1124    }
1125
1126    fn emit_movn(&mut self, sz: Size, reg: Location, val: u32) -> Result<(), CompileError> {
1127        match (sz, reg) {
1128            (Size::S32, Location::GPR(reg)) => {
1129                dynasm!(self ; movn W(reg), val);
1130            }
1131            (Size::S64, Location::GPR(reg)) => {
1132                dynasm!(self ; movn X(reg), val);
1133            }
1134            _ => codegen_error!("singlepass can't emit MOVN"),
1135        }
1136        Ok(())
1137    }
1138    fn emit_movz(&mut self, reg: Location, val: u32) -> Result<(), CompileError> {
1139        match reg {
1140            Location::GPR(reg) => {
1141                dynasm!(self ; movz W(reg), val);
1142            }
1143            _ => codegen_error!("singlepass can't emit MOVZ"),
1144        }
1145        Ok(())
1146    }
1147    fn emit_movk(&mut self, reg: Location, val: u32, shift: u32) -> Result<(), CompileError> {
1148        match reg {
1149            Location::GPR(reg) => {
1150                dynasm!(self ; movk X(reg), val, LSL shift);
1151            }
1152            _ => codegen_error!("singlepass can't emit MOVK"),
1153        }
1154        Ok(())
1155    }
1156
1157    fn emit_mov_imm(&mut self, dst: Location, val: u64) -> Result<(), CompileError> {
1158        match dst {
1159            Location::GPR(dst) => {
1160                let offset = val.trailing_zeros() & 48;
1161                let masked = 0xffff & (val >> offset);
1162                if (masked << offset) == val {
1163                    dynasm!(self ; movz X(dst), masked as u32, LSL offset);
1164                } else if val >> 16 == 0xffff_ffff_ffff {
1165                    let val: u16 = !((val & 0xffff) as u16);
1166                    dynasm!(self ; movn X(dst), val as u32);
1167                } else if val >> 16 == 0xffff {
1168                    let val: u16 = !((val & 0xffff) as u16);
1169                    dynasm!(self ; movn W(dst), val as u32);
1170                } else {
1171                    dynasm!(self ; movz W(dst), (val&0xffff) as u32);
1172                    let val = val >> 16;
1173                    if val != 0 {
1174                        dynasm!(self ; movk X(dst), (val&0xffff) as u32, LSL 16);
1175                        let val = val >> 16;
1176                        if val != 0 {
1177                            dynasm!(self ; movk X(dst), (val&0xffff) as u32, LSL 32);
1178                            let val = val >> 16;
1179                            if val != 0 {
1180                                dynasm!(self ; movk X(dst), (val&0xffff) as u32, LSL 48);
1181                            }
1182                        }
1183                    }
1184                }
1185            }
1186            _ => codegen_error!("singlepass can't emit MOVW {:?}", dst),
1187        }
1188        Ok(())
1189    }
1190
1191    fn emit_add(
1192        &mut self,
1193        sz: Size,
1194        src1: Location,
1195        src2: Location,
1196        dst: Location,
1197    ) -> Result<(), CompileError> {
1198        match (sz, src1, src2, dst) {
1199            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1200                dynasm!(self ; add XSP(dst), XSP(src1), X(src2), UXTX);
1201            }
1202            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1203                dynasm!(self ; add WSP(dst), WSP(src1), W(src2), UXTX);
1204            }
1205            (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1206            | (Size::S64, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1207                dynasm!(self ; add XSP(dst), XSP(src1), imm as _);
1208            }
1209            (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
1210            | (Size::S64, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
1211                if imm >= 0x1000 {
1212                    unreachable!();
1213                }
1214                dynasm!(self ; add XSP(dst), XSP(src1), imm);
1215            }
1216            (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst))
1217            | (Size::S64, Location::Imm64(imm), Location::GPR(src1), Location::GPR(dst)) => {
1218                if imm >= 0x1000 {
1219                    unreachable!();
1220                }
1221                let imm = imm as u32;
1222                dynasm!(self ; add XSP(dst), XSP(src1), imm);
1223            }
1224            (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1225            | (Size::S32, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1226                dynasm!(self ; add WSP(dst), WSP(src1), imm as u32);
1227            }
1228            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
1229            | (Size::S32, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
1230                if imm >= 0x1000 {
1231                    unreachable!();
1232                }
1233                dynasm!(self ; add WSP(dst), WSP(src1), imm);
1234            }
1235            _ => codegen_error!(
1236                "singlepass can't emit ADD {:?} {:?} {:?} {:?}",
1237                sz,
1238                src1,
1239                src2,
1240                dst
1241            ),
1242        }
1243        Ok(())
1244    }
1245    fn emit_sub(
1246        &mut self,
1247        sz: Size,
1248        src1: Location,
1249        src2: Location,
1250        dst: Location,
1251    ) -> Result<(), CompileError> {
1252        match (sz, src1, src2, dst) {
1253            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1254                dynasm!(self; sub XSP(dst), XSP(src1), X(src2), UXTX 0);
1255            }
1256            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1257                dynasm!(self ; sub WSP(dst), WSP(src1), W(src2), UXTX 0);
1258            }
1259            (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1260                dynasm!(self ; sub XSP(dst), XSP(src1), imm as u32);
1261            }
1262            (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1263                dynasm!(self ; sub WSP(dst), WSP(src1), imm as u32);
1264            }
1265            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1266                if imm >= 0x1000 {
1267                    unreachable!();
1268                }
1269                dynasm!(self ; sub WSP(dst), WSP(src1), imm);
1270            }
1271            (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1272                if imm >= 0x1000 {
1273                    unreachable!();
1274                }
1275                dynasm!(self ; sub XSP(dst), XSP(src1), imm);
1276            }
1277            (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
1278                if imm >= 0x1000 {
1279                    unreachable!();
1280                }
1281                dynasm!(self ; sub XSP(dst), XSP(src1), imm as u32);
1282            }
1283            _ => codegen_error!(
1284                "singlepass can't emit SUB {:?} {:?} {:?} {:?}",
1285                sz,
1286                src1,
1287                src2,
1288                dst
1289            ),
1290        }
1291        Ok(())
1292    }
1293    fn emit_mul(
1294        &mut self,
1295        sz: Size,
1296        src1: Location,
1297        src2: Location,
1298        dst: Location,
1299    ) -> Result<(), CompileError> {
1300        match (sz, src1, src2, dst) {
1301            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1302                dynasm!(self ; mul X(dst), X(src1), X(src2));
1303            }
1304            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1305                dynasm!(self ; mul W(dst), W(src1), W(src2));
1306            }
1307            _ => codegen_error!(
1308                "singlepass can't emit MUL {:?} {:?} {:?} {:?}",
1309                sz,
1310                src1,
1311                src2,
1312                dst
1313            ),
1314        }
1315        Ok(())
1316    }
1317    fn emit_adds(
1318        &mut self,
1319        sz: Size,
1320        src1: Location,
1321        src2: Location,
1322        dst: Location,
1323    ) -> Result<(), CompileError> {
1324        match (sz, src1, src2, dst) {
1325            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1326                dynasm!(self ; adds X(dst), X(src1), X(src2));
1327            }
1328            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1329                dynasm!(self ; adds W(dst), W(src1), W(src2));
1330            }
1331            (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1332            | (Size::S64, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1333                dynasm!(self ; adds X(dst), XSP(src1), imm as u32);
1334            }
1335            (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
1336            | (Size::S64, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
1337                if imm >= 0x1000 {
1338                    codegen_error!("singlepass ADD.S with imm too large {}", imm);
1339                }
1340                dynasm!(self ; adds X(dst), XSP(src1), imm);
1341            }
1342            (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1343            | (Size::S32, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1344                dynasm!(self ; adds W(dst), WSP(src1), imm as u32);
1345            }
1346            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
1347            | (Size::S32, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
1348                if imm >= 0x1000 {
1349                    codegen_error!("singlepass ADD.S with imm too large {}", imm);
1350                }
1351                dynasm!(self ; adds W(dst), WSP(src1), imm);
1352            }
1353            _ => codegen_error!(
1354                "singlepass can't emit ADD.S {:?} {:?} {:?} {:?}",
1355                sz,
1356                src1,
1357                src2,
1358                dst
1359            ),
1360        }
1361        Ok(())
1362    }
1363    fn emit_subs(
1364        &mut self,
1365        sz: Size,
1366        src1: Location,
1367        src2: Location,
1368        dst: Location,
1369    ) -> Result<(), CompileError> {
1370        match (sz, src1, src2, dst) {
1371            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1372                dynasm!(self ; subs X(dst), X(src1), X(src2));
1373            }
1374            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1375                dynasm!(self ; subs W(dst), W(src1), W(src2));
1376            }
1377            (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1378                dynasm!(self ; subs X(dst), XSP(src1), imm as u32);
1379            }
1380            (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1381                dynasm!(self ; subs W(dst), WSP(src1), imm as u32);
1382            }
1383            _ => codegen_error!(
1384                "singlepass can't emit SUB.S {:?} {:?} {:?} {:?}",
1385                sz,
1386                src1,
1387                src2,
1388                dst
1389            ),
1390        }
1391        Ok(())
1392    }
1393    fn emit_add_lsl(
1394        &mut self,
1395        sz: Size,
1396        src1: Location,
1397        src2: Location,
1398        lsl: u32,
1399        dst: Location,
1400    ) -> Result<(), CompileError> {
1401        match (sz, src1, src2, dst) {
1402            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1403                dynasm!(self ; add X(dst), X(src1), X(src2), LSL lsl);
1404            }
1405            _ => codegen_error!(
1406                "singlepass can't emit LSL {:?} {:?} {:?} {:?} LSL {:?}",
1407                sz,
1408                src1,
1409                src2,
1410                dst,
1411                lsl
1412            ),
1413        }
1414        Ok(())
1415    }
1416
1417    fn emit_cmp(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1418        match (sz, src, dst) {
1419            (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1420                dynasm!(self ; cmp X(dst), X(src));
1421            }
1422            (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1423                dynasm!(self ; cmp W(dst), W(src));
1424            }
1425            (Size::S64, Location::Imm8(imm), Location::GPR(dst)) => {
1426                dynasm!(self ; cmp XSP(dst), imm as u32);
1427            }
1428            (Size::S64, Location::Imm32(imm), Location::GPR(dst)) => {
1429                if imm >= 0x1000 {
1430                    codegen_error!("singlepass CMP with imm too large {}", imm);
1431                }
1432                dynasm!(self ; cmp XSP(dst), imm as u32);
1433            }
1434            (Size::S64, Location::Imm64(imm), Location::GPR(dst)) => {
1435                if imm >= 0x1000 {
1436                    codegen_error!("singlepass CMP with imm too large {}", imm);
1437                }
1438                dynasm!(self ; cmp XSP(dst), imm as u32);
1439            }
1440            (Size::S32, Location::Imm8(imm), Location::GPR(dst)) => {
1441                dynasm!(self ; cmp WSP(dst), imm as u32);
1442            }
1443            (Size::S32, Location::Imm32(imm), Location::GPR(dst)) => {
1444                if imm >= 0x1000 {
1445                    codegen_error!("singlepass CMP with imm too large {}", imm);
1446                }
1447                dynasm!(self ; cmp WSP(dst), imm as u32);
1448            }
1449            _ => codegen_error!("singlepass can't emit CMP {:?} {:?} {:?}", sz, src, dst),
1450        }
1451        Ok(())
1452    }
1453
1454    fn emit_tst(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1455        match (sz, src, dst) {
1456            (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1457                dynasm!(self ; tst X(dst), X(src));
1458            }
1459            (Size::S64, Location::Imm32(imm), Location::GPR(dst)) => {
1460                if encode_logical_immediate_64bit(imm as u64).is_none() {
1461                    codegen_error!("singlepass TST with incompatible imm {}", imm);
1462                }
1463                dynasm!(self ; tst X(dst), imm as u64);
1464            }
1465            (Size::S64, Location::Imm64(imm), Location::GPR(dst)) => {
1466                if encode_logical_immediate_64bit(imm as u64).is_none() {
1467                    codegen_error!("singlepass TST with incompatible imm {}", imm);
1468                }
1469                dynasm!(self ; tst X(dst), imm as u64);
1470            }
1471            (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1472                dynasm!(self ; tst W(dst), W(src));
1473            }
1474            (Size::S32, Location::Imm32(imm), Location::GPR(dst)) => {
1475                if encode_logical_immediate_64bit(imm as u64).is_none() {
1476                    codegen_error!("singlepass TST with incompatible imm {}", imm);
1477                }
1478                dynasm!(self ; tst W(dst), imm);
1479            }
1480            _ => codegen_error!("singlepass can't emit TST"),
1481        }
1482        Ok(())
1483    }
1484
1485    fn emit_lsl(
1486        &mut self,
1487        sz: Size,
1488        src1: Location,
1489        src2: Location,
1490        dst: Location,
1491    ) -> Result<(), CompileError> {
1492        match (sz, src1, src2, dst) {
1493            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1494                dynasm!(self ; lsl X(dst), X(src1), X(src2));
1495            }
1496            (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1497                if imm > 63 {
1498                    codegen_error!("singlepass LSL with incompatible imm {}", imm);
1499                }
1500                let imm = imm as u32;
1501
1502                dynasm!(self ; lsl X(dst), X(src1), imm);
1503            }
1504            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1505                dynasm!(self ; lsl W(dst), W(src1), W(src2));
1506            }
1507            (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1508            | (Size::S64, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1509                if imm > 63 {
1510                    codegen_error!("singlepass LSL with incompatible imm {}", imm);
1511                }
1512                dynasm!(self ; lsl X(dst), X(src1), imm as u32);
1513            }
1514            (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst))
1515            | (Size::S64, Location::Imm64(imm), Location::GPR(src1), Location::GPR(dst)) => {
1516                if imm > 63 {
1517                    codegen_error!("singlepass LSL with incompatible imm {}", imm);
1518                }
1519                dynasm!(self ; lsl X(dst), X(src1), imm as u32);
1520            }
1521            (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1522            | (Size::S32, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1523                if imm > 31 {
1524                    codegen_error!("singlepass LSL with incompatible imm {}", imm);
1525                }
1526                dynasm!(self ; lsl W(dst), W(src1), imm as u32);
1527            }
1528            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
1529            | (Size::S32, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
1530                if imm > 31 {
1531                    codegen_error!("singlepass LSL with incompatible imm {}", imm);
1532                }
1533                dynasm!(self ; lsl W(dst), W(src1), imm as u32);
1534            }
1535            _ => codegen_error!(
1536                "singlepass can't emit LSL {:?} {:?} {:?} {:?}",
1537                sz,
1538                src1,
1539                src2,
1540                dst
1541            ),
1542        }
1543        Ok(())
1544    }
1545    fn emit_asr(
1546        &mut self,
1547        sz: Size,
1548        src1: Location,
1549        src2: Location,
1550        dst: Location,
1551    ) -> Result<(), CompileError> {
1552        match (sz, src1, src2, dst) {
1553            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1554                dynasm!(self ; asr X(dst), X(src1), X(src2));
1555            }
1556            (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1557                let imm = imm as u32;
1558
1559                if imm == 0 || imm > 63 {
1560                    codegen_error!("singlepass ASR with incompatible imm {}", imm);
1561                }
1562                dynasm!(self ; asr X(dst), X(src1), imm);
1563            }
1564            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1565                dynasm!(self ; asr W(dst), W(src1), W(src2));
1566            }
1567            (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1568            | (Size::S64, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1569                if imm == 0 || imm > 63 {
1570                    codegen_error!("singlepass ASR with incompatible imm {}", imm);
1571                }
1572                dynasm!(self ; asr X(dst), X(src1), imm as u32);
1573            }
1574            (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst))
1575            | (Size::S64, Location::Imm64(imm), Location::GPR(src1), Location::GPR(dst)) => {
1576                if imm == 0 || imm > 63 {
1577                    codegen_error!("singlepass ASR with incompatible imm {}", imm);
1578                }
1579                dynasm!(self ; asr X(dst), X(src1), imm as u32);
1580            }
1581            (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1582            | (Size::S32, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1583                if imm == 0 || imm > 31 {
1584                    codegen_error!("singlepass ASR with incompatible imm {}", imm);
1585                }
1586                dynasm!(self ; asr W(dst), W(src1), imm as u32);
1587            }
1588            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
1589            | (Size::S32, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
1590                if imm == 0 || imm > 31 {
1591                    codegen_error!("singlepass ASR with incompatible imm {}", imm);
1592                }
1593                dynasm!(self ; asr W(dst), W(src1), imm as u32);
1594            }
1595            _ => codegen_error!(
1596                "singlepass can't emit ASR {:?} {:?} {:?} {:?}",
1597                sz,
1598                src1,
1599                src2,
1600                dst
1601            ),
1602        }
1603        Ok(())
1604    }
1605    fn emit_lsr(
1606        &mut self,
1607        sz: Size,
1608        src1: Location,
1609        src2: Location,
1610        dst: Location,
1611    ) -> Result<(), CompileError> {
1612        match (sz, src1, src2, dst) {
1613            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1614                dynasm!(self ; lsr X(dst), X(src1), X(src2));
1615            }
1616            (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1617                let imm = imm as u32;
1618
1619                if imm == 0 || imm > 63 {
1620                    codegen_error!("singlepass LSR with incompatible imm {}", imm);
1621                }
1622                dynasm!(self ; lsr X(dst), X(src1), imm);
1623            }
1624            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1625                dynasm!(self ; lsr W(dst), W(src1), W(src2));
1626            }
1627            (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1628            | (Size::S64, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1629                if imm == 0 || imm > 63 {
1630                    codegen_error!("singlepass LSR with incompatible imm {}", imm);
1631                }
1632                dynasm!(self ; lsr X(dst), X(src1), imm as u32);
1633            }
1634            (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst))
1635            | (Size::S64, Location::Imm64(imm), Location::GPR(src1), Location::GPR(dst)) => {
1636                if imm == 0 || imm > 63 {
1637                    codegen_error!("singlepass LSR with incompatible imm {}", imm);
1638                }
1639                dynasm!(self ; lsr X(dst), X(src1), imm as u32);
1640            }
1641            (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1642            | (Size::S32, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1643                if imm == 0 || imm > 31 {
1644                    codegen_error!("singlepass LSR with incompatible imm {}", imm);
1645                }
1646                dynasm!(self ; lsr W(dst), W(src1), imm as u32);
1647            }
1648            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
1649            | (Size::S32, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
1650                if imm == 0 || imm > 31 {
1651                    codegen_error!("singlepass LSR with incompatible imm {}", imm);
1652                }
1653                dynasm!(self ; lsr W(dst), W(src1), imm as u32);
1654            }
1655            _ => codegen_error!(
1656                "singlepass can't emit LSR {:?} {:?} {:?} {:?}",
1657                sz,
1658                src1,
1659                src2,
1660                dst
1661            ),
1662        }
1663        Ok(())
1664    }
1665    fn emit_ror(
1666        &mut self,
1667        sz: Size,
1668        src1: Location,
1669        src2: Location,
1670        dst: Location,
1671    ) -> Result<(), CompileError> {
1672        match (sz, src1, src2, dst) {
1673            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1674                dynasm!(self ; ror X(dst), X(src1), X(src2));
1675            }
1676            (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
1677            | (Size::S64, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
1678                let imm = imm as u32;
1679
1680                if imm == 0 || imm > 63 {
1681                    codegen_error!("singlepass ROR with incompatible imm {}", imm);
1682                }
1683                dynasm!(self ; ror X(dst), X(src1), imm);
1684            }
1685            (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst))
1686            | (Size::S64, Location::Imm64(imm), Location::GPR(src1), Location::GPR(dst)) => {
1687                let imm = imm as u32;
1688
1689                if imm == 0 || imm > 63 {
1690                    codegen_error!("singlepass ROR with incompatible imm {}", imm);
1691                }
1692                dynasm!(self ; ror X(dst), X(src1), imm);
1693            }
1694            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
1695            | (Size::S32, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
1696                if imm == 0 || imm > 31 {
1697                    codegen_error!("singlepass ROR with incompatible imm {}", imm);
1698                }
1699                dynasm!(self ; ror W(dst), W(src1), imm as u32);
1700            }
1701            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1702                dynasm!(self ; ror W(dst), W(src1), W(src2));
1703            }
1704            (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1705            | (Size::S64, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1706                if imm == 0 || imm > 63 {
1707                    codegen_error!("singlepass ROR with incompatible imm {}", imm);
1708                }
1709                dynasm!(self ; ror X(dst), X(src1), imm as u32);
1710            }
1711            (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1712            | (Size::S32, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1713                if imm == 0 || imm > 31 {
1714                    codegen_error!("singlepass ROR with incompatible imm {}", imm);
1715                }
1716                dynasm!(self ; ror W(dst), W(src1), imm as u32);
1717            }
1718            _ => codegen_error!(
1719                "singlepass can't emit ROR {:?} {:?} {:?} {:?}",
1720                sz,
1721                src1,
1722                src2,
1723                dst
1724            ),
1725        }
1726        Ok(())
1727    }
1728
1729    fn emit_or(
1730        &mut self,
1731        sz: Size,
1732        src1: Location,
1733        src2: Location,
1734        dst: Location,
1735    ) -> Result<(), CompileError> {
1736        match (sz, src1, src2, dst) {
1737            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1738                dynasm!(self ; orr X(dst), X(src1), X(src2));
1739            }
1740            (Size::S64, Location::GPR(src1), Location::Imm64(src2), Location::GPR(dst)) => {
1741                let src2 = src2 as u64;
1742
1743                if encode_logical_immediate_64bit(src2 as u64).is_none() {
1744                    codegen_error!("singlepass OR with incompatible imm {}", src2);
1745                }
1746                dynasm!(self ; orr XSP(dst), X(src1), src2);
1747            }
1748            (Size::S32, Location::GPR(src1), Location::Imm32(src2), Location::GPR(dst)) => {
1749                let src2 = src2 as u32;
1750
1751                if encode_logical_immediate_32bit(src2).is_none() {
1752                    codegen_error!("singlepass OR with incompatible imm {}", src2);
1753                }
1754                dynasm!(self ; orr WSP(dst), W(src1), src2);
1755            }
1756            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1757                dynasm!(self ; orr W(dst), W(src1), W(src2));
1758            }
1759            _ => codegen_error!(
1760                "singlepass can't emit OR {:?} {:?} {:?} {:?}",
1761                sz,
1762                src1,
1763                src2,
1764                dst
1765            ),
1766        }
1767        Ok(())
1768    }
1769    fn emit_and(
1770        &mut self,
1771        sz: Size,
1772        src1: Location,
1773        src2: Location,
1774        dst: Location,
1775    ) -> Result<(), CompileError> {
1776        match (sz, src1, src2, dst) {
1777            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1778                dynasm!(self ; and X(dst), X(src1), X(src2));
1779            }
1780            (Size::S64, Location::GPR(src1), Location::Imm64(src2), Location::GPR(dst)) => {
1781                let src2 = src2 as u64;
1782
1783                if encode_logical_immediate_64bit(src2 as u64).is_none() {
1784                    codegen_error!("singlepass AND with incompatible imm {}", src2);
1785                }
1786                dynasm!(self ; and XSP(dst), X(src1), src2);
1787            }
1788            (Size::S32, Location::GPR(src1), Location::Imm32(src2), Location::GPR(dst)) => {
1789                let src2 = src2 as u32;
1790
1791                if encode_logical_immediate_32bit(src2).is_none() {
1792                    codegen_error!("singlepass AND with incompatible imm {}", src2);
1793                }
1794                dynasm!(self ; and WSP(dst), W(src1), src2);
1795            }
1796            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1797                dynasm!(self ; and W(dst), W(src1), W(src2));
1798            }
1799            _ => codegen_error!(
1800                "singlepass can't emit AND {:?} {:?} {:?} {:?}",
1801                sz,
1802                src1,
1803                src2,
1804                dst
1805            ),
1806        }
1807        Ok(())
1808    }
1809    fn emit_eor(
1810        &mut self,
1811        sz: Size,
1812        src1: Location,
1813        src2: Location,
1814        dst: Location,
1815    ) -> Result<(), CompileError> {
1816        match (sz, src1, src2, dst) {
1817            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1818                dynasm!(self ; eor X(dst), X(src1), X(src2));
1819            }
1820            (Size::S64, Location::GPR(src1), Location::Imm64(src2), Location::GPR(dst)) => {
1821                let src2 = src2 as u64;
1822
1823                if encode_logical_immediate_64bit(src2 as u64).is_none() {
1824                    codegen_error!("singlepass EOR with incompatible imm {}", src2);
1825                }
1826                dynasm!(self ; eor XSP(dst), X(src1), src2);
1827            }
1828            (Size::S32, Location::GPR(src1), Location::Imm32(src2), Location::GPR(dst)) => {
1829                let src2 = src2 as u32;
1830
1831                if encode_logical_immediate_32bit(src2).is_none() {
1832                    codegen_error!("singlepass EOR with incompatible imm {}", src2);
1833                }
1834                dynasm!(self ; eor WSP(dst), W(src1), src2);
1835            }
1836            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1837                dynasm!(self ; eor W(dst), W(src1), W(src2));
1838            }
1839            _ => codegen_error!(
1840                "singlepass can't emit EOR {:?} {:?} {:?} {:?}",
1841                sz,
1842                src1,
1843                src2,
1844                dst
1845            ),
1846        }
1847        Ok(())
1848    }
1849
1850    fn emit_bfc(
1851        &mut self,
1852        sz: Size,
1853        lsb: u32,
1854        width: u32,
1855        dst: Location,
1856    ) -> Result<(), CompileError> {
1857        match (sz, dst) {
1858            (Size::S32, Location::GPR(dst)) => {
1859                dynasm!(self ; bfc W(dst), lsb, width);
1860            }
1861            (Size::S64, Location::GPR(dst)) => {
1862                dynasm!(self ; bfc X(dst), lsb, width);
1863            }
1864            _ => codegen_error!("singlepass can't emit BFC"),
1865        }
1866        Ok(())
1867    }
1868    fn emit_bfi(
1869        &mut self,
1870        sz: Size,
1871        src: Location,
1872        lsb: u32,
1873        width: u32,
1874        dst: Location,
1875    ) -> Result<(), CompileError> {
1876        match (sz, src, dst) {
1877            (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1878                dynasm!(self ; bfi W(dst), W(src), lsb, width);
1879            }
1880            (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1881                dynasm!(self ; bfi X(dst), X(src), lsb, width);
1882            }
1883            _ => codegen_error!("singlepass can't emit BFI"),
1884        }
1885        Ok(())
1886    }
1887
1888    fn emit_udiv(
1889        &mut self,
1890        sz: Size,
1891        src1: Location,
1892        src2: Location,
1893        dst: Location,
1894    ) -> Result<(), CompileError> {
1895        match (sz, src1, src2, dst) {
1896            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1897                dynasm!(self ; udiv W(dst), W(src1), W(src2));
1898            }
1899            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1900                dynasm!(self ; udiv X(dst), X(src1), X(src2));
1901            }
1902            _ => codegen_error!(
1903                "singlepass can't emit UDIV {:?} {:?} {:?} {:?}",
1904                sz,
1905                src1,
1906                src2,
1907                dst
1908            ),
1909        }
1910        Ok(())
1911    }
1912    fn emit_sdiv(
1913        &mut self,
1914        sz: Size,
1915        src1: Location,
1916        src2: Location,
1917        dst: Location,
1918    ) -> Result<(), CompileError> {
1919        match (sz, src1, src2, dst) {
1920            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1921                dynasm!(self ; sdiv W(dst), W(src1), W(src2));
1922            }
1923            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1924                dynasm!(self ; sdiv X(dst), X(src1), X(src2));
1925            }
1926            _ => codegen_error!(
1927                "singlepass can't emit UDIV {:?} {:?} {:?} {:?}",
1928                sz,
1929                src1,
1930                src2,
1931                dst
1932            ),
1933        }
1934        Ok(())
1935    }
1936
1937    /// msub : c - a*b -> dst
1938    fn emit_msub(
1939        &mut self,
1940        sz: Size,
1941        a: Location,
1942        b: Location,
1943        c: Location,
1944        dst: Location,
1945    ) -> Result<(), CompileError> {
1946        match (sz, a, b, c, dst) {
1947            (
1948                Size::S32,
1949                Location::GPR(a),
1950                Location::GPR(b),
1951                Location::GPR(c),
1952                Location::GPR(dst),
1953            ) => {
1954                dynasm!(self ; msub W(dst), W(a), W(b), W(c));
1955            }
1956            (
1957                Size::S64,
1958                Location::GPR(a),
1959                Location::GPR(b),
1960                Location::GPR(c),
1961                Location::GPR(dst),
1962            ) => {
1963                dynasm!(self ; msub X(dst), X(a), X(b), X(c));
1964            }
1965            _ => codegen_error!(
1966                "singlepass can't emit msub {:?} {:?} {:?} {:?} {:?}",
1967                sz,
1968                a,
1969                b,
1970                c,
1971                dst
1972            ),
1973        }
1974        Ok(())
1975    }
1976
1977    fn emit_sxtb(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1978        match (sz, src, dst) {
1979            (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1980                dynasm!(self ; sxtb W(dst), W(src));
1981            }
1982            (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1983                dynasm!(self ; sxtb X(dst), W(src));
1984            }
1985            _ => codegen_error!("singlepass can't emit SXTB {:?} {:?} {:?}", sz, src, dst),
1986        }
1987        Ok(())
1988    }
1989    fn emit_sxth(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1990        match (sz, src, dst) {
1991            (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1992                dynasm!(self ; sxth W(dst), W(src));
1993            }
1994            (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1995                dynasm!(self ; sxth X(dst), W(src));
1996            }
1997            _ => codegen_error!("singlepass can't emit SXTH {:?} {:?} {:?}", sz, src, dst),
1998        }
1999        Ok(())
2000    }
2001    fn emit_sxtw(&mut self, _sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2002        match (src, dst) {
2003            (Location::GPR(src), Location::GPR(dst)) => {
2004                dynasm!(self ; sxtw X(dst), W(src));
2005            }
2006            _ => codegen_error!("singlepass can't emit SXTW {:?} {:?}", src, dst),
2007        }
2008        Ok(())
2009    }
2010    fn emit_uxtb(&mut self, _sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2011        match (src, dst) {
2012            (Location::GPR(src), Location::GPR(dst)) => {
2013                dynasm!(self ; uxtb W(dst), W(src));
2014            }
2015            _ => codegen_error!("singlepass can't emit UXTB {:?} {:?}", src, dst),
2016        }
2017        Ok(())
2018    }
2019    fn emit_uxth(&mut self, _sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2020        match (src, dst) {
2021            (Location::GPR(src), Location::GPR(dst)) => {
2022                dynasm!(self ; uxth W(dst), W(src));
2023            }
2024            _ => codegen_error!("singlepass can't emit UXTH {:?} {:?}", src, dst),
2025        }
2026        Ok(())
2027    }
2028
2029    fn emit_cset(&mut self, sz: Size, dst: Location, cond: Condition) -> Result<(), CompileError> {
2030        match (sz, dst) {
2031            (Size::S32, Location::GPR(reg)) => match cond {
2032                Condition::Eq => dynasm!(self ; cset W(reg), eq),
2033                Condition::Ne => dynasm!(self ; cset W(reg), ne),
2034                Condition::Cs => dynasm!(self ; cset W(reg), cs),
2035                Condition::Cc => dynasm!(self ; cset W(reg), cc),
2036                Condition::Mi => dynasm!(self ; cset W(reg), mi),
2037                Condition::Pl => dynasm!(self ; cset W(reg), pl),
2038                Condition::Vs => dynasm!(self ; cset W(reg), vs),
2039                Condition::Vc => dynasm!(self ; cset W(reg), vc),
2040                Condition::Hi => dynasm!(self ; cset W(reg), hi),
2041                Condition::Ls => dynasm!(self ; cset W(reg), ls),
2042                Condition::Ge => dynasm!(self ; cset W(reg), ge),
2043                Condition::Lt => dynasm!(self ; cset W(reg), lt),
2044                Condition::Gt => dynasm!(self ; cset W(reg), gt),
2045                Condition::Le => dynasm!(self ; cset W(reg), le),
2046                Condition::Al => dynasm!(self ; cset W(reg), al),
2047            },
2048            (Size::S64, Location::GPR(reg)) => match cond {
2049                Condition::Eq => dynasm!(self ; cset X(reg), eq),
2050                Condition::Ne => dynasm!(self ; cset X(reg), ne),
2051                Condition::Cs => dynasm!(self ; cset X(reg), cs),
2052                Condition::Cc => dynasm!(self ; cset X(reg), cc),
2053                Condition::Mi => dynasm!(self ; cset X(reg), mi),
2054                Condition::Pl => dynasm!(self ; cset X(reg), pl),
2055                Condition::Vs => dynasm!(self ; cset X(reg), vs),
2056                Condition::Vc => dynasm!(self ; cset X(reg), vc),
2057                Condition::Hi => dynasm!(self ; cset X(reg), hi),
2058                Condition::Ls => dynasm!(self ; cset X(reg), ls),
2059                Condition::Ge => dynasm!(self ; cset X(reg), ge),
2060                Condition::Lt => dynasm!(self ; cset X(reg), lt),
2061                Condition::Gt => dynasm!(self ; cset X(reg), gt),
2062                Condition::Le => dynasm!(self ; cset X(reg), le),
2063                Condition::Al => dynasm!(self ; cset X(reg), al),
2064            },
2065            _ => codegen_error!("singlepass can't emit CSET {:?} {:?} {:?}", sz, dst, cond),
2066        }
2067        Ok(())
2068    }
2069    fn emit_csetm(&mut self, sz: Size, dst: Location, cond: Condition) -> Result<(), CompileError> {
2070        match (sz, dst) {
2071            (Size::S32, Location::GPR(reg)) => match cond {
2072                Condition::Eq => dynasm!(self ; csetm W(reg), eq),
2073                Condition::Ne => dynasm!(self ; csetm W(reg), ne),
2074                Condition::Cs => dynasm!(self ; csetm W(reg), cs),
2075                Condition::Cc => dynasm!(self ; csetm W(reg), cc),
2076                Condition::Mi => dynasm!(self ; csetm W(reg), mi),
2077                Condition::Pl => dynasm!(self ; csetm W(reg), pl),
2078                Condition::Vs => dynasm!(self ; csetm W(reg), vs),
2079                Condition::Vc => dynasm!(self ; csetm W(reg), vc),
2080                Condition::Hi => dynasm!(self ; csetm W(reg), hi),
2081                Condition::Ls => dynasm!(self ; csetm W(reg), ls),
2082                Condition::Ge => dynasm!(self ; csetm W(reg), ge),
2083                Condition::Lt => dynasm!(self ; csetm W(reg), lt),
2084                Condition::Gt => dynasm!(self ; csetm W(reg), gt),
2085                Condition::Le => dynasm!(self ; csetm W(reg), le),
2086                Condition::Al => dynasm!(self ; csetm W(reg), al),
2087            },
2088            (Size::S64, Location::GPR(reg)) => match cond {
2089                Condition::Eq => dynasm!(self ; csetm X(reg), eq),
2090                Condition::Ne => dynasm!(self ; csetm X(reg), ne),
2091                Condition::Cs => dynasm!(self ; csetm X(reg), cs),
2092                Condition::Cc => dynasm!(self ; csetm X(reg), cc),
2093                Condition::Mi => dynasm!(self ; csetm X(reg), mi),
2094                Condition::Pl => dynasm!(self ; csetm X(reg), pl),
2095                Condition::Vs => dynasm!(self ; csetm X(reg), vs),
2096                Condition::Vc => dynasm!(self ; csetm X(reg), vc),
2097                Condition::Hi => dynasm!(self ; csetm X(reg), hi),
2098                Condition::Ls => dynasm!(self ; csetm X(reg), ls),
2099                Condition::Ge => dynasm!(self ; csetm X(reg), ge),
2100                Condition::Lt => dynasm!(self ; csetm X(reg), lt),
2101                Condition::Gt => dynasm!(self ; csetm X(reg), gt),
2102                Condition::Le => dynasm!(self ; csetm X(reg), le),
2103                Condition::Al => dynasm!(self ; csetm X(reg), al),
2104            },
2105            _ => codegen_error!("singlepass can't emit CSETM {:?} {:?} {:?}", sz, dst, cond),
2106        }
2107        Ok(())
2108    }
2109    fn emit_cinc(
2110        &mut self,
2111        sz: Size,
2112        src: Location,
2113        dst: Location,
2114        cond: Condition,
2115    ) -> Result<(), CompileError> {
2116        match (sz, src, dst) {
2117            (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
2118                match cond {
2119                    Condition::Eq => dynasm!(self ; cinc W(dst), W(src), eq),
2120                    Condition::Ne => dynasm!(self ; cinc W(dst), W(src), ne),
2121                    Condition::Cs => dynasm!(self ; cinc W(dst), W(src), cs),
2122                    Condition::Cc => dynasm!(self ; cinc W(dst), W(src), cc),
2123                    Condition::Mi => dynasm!(self ; cinc W(dst), W(src), mi),
2124                    Condition::Pl => dynasm!(self ; cinc W(dst), W(src), pl),
2125                    Condition::Vs => dynasm!(self ; cinc W(dst), W(src), vs),
2126                    Condition::Vc => dynasm!(self ; cinc W(dst), W(src), vc),
2127                    Condition::Hi => dynasm!(self ; cinc W(dst), W(src), hi),
2128                    Condition::Ls => dynasm!(self ; cinc W(dst), W(src), ls),
2129                    Condition::Ge => dynasm!(self ; cinc W(dst), W(src), ge),
2130                    Condition::Lt => dynasm!(self ; cinc W(dst), W(src), lt),
2131                    Condition::Gt => dynasm!(self ; cinc W(dst), W(src), gt),
2132                    Condition::Le => dynasm!(self ; cinc W(dst), W(src), le),
2133                    Condition::Al => dynasm!(self ; cinc W(dst), W(src), al),
2134                };
2135            }
2136            (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
2137                match cond {
2138                    Condition::Eq => dynasm!(self ; cinc X(src), X(dst), eq),
2139                    Condition::Ne => dynasm!(self ; cinc X(src), X(dst), ne),
2140                    Condition::Cs => dynasm!(self ; cinc X(src), X(dst), cs),
2141                    Condition::Cc => dynasm!(self ; cinc X(src), X(dst), cc),
2142                    Condition::Mi => dynasm!(self ; cinc X(src), X(dst), mi),
2143                    Condition::Pl => dynasm!(self ; cinc X(src), X(dst), pl),
2144                    Condition::Vs => dynasm!(self ; cinc X(src), X(dst), vs),
2145                    Condition::Vc => dynasm!(self ; cinc X(src), X(dst), vc),
2146                    Condition::Hi => dynasm!(self ; cinc X(src), X(dst), hi),
2147                    Condition::Ls => dynasm!(self ; cinc X(src), X(dst), ls),
2148                    Condition::Ge => dynasm!(self ; cinc X(src), X(dst), ge),
2149                    Condition::Lt => dynasm!(self ; cinc X(src), X(dst), lt),
2150                    Condition::Gt => dynasm!(self ; cinc X(src), X(dst), gt),
2151                    Condition::Le => dynasm!(self ; cinc X(src), X(dst), le),
2152                    Condition::Al => dynasm!(self ; cinc X(src), X(dst), al),
2153                };
2154            }
2155            _ => codegen_error!("singlepass can't emit CINC"),
2156        }
2157        Ok(())
2158    }
2159
2160    fn emit_clz(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2161        match (sz, src, dst) {
2162            (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
2163                dynasm!(self ; clz X(dst), X(src));
2164            }
2165            (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
2166                dynasm!(self ; clz W(dst), W(src));
2167            }
2168            _ => codegen_error!("singlepass can't emit CLS {:?} {:?} {:?}", sz, src, dst),
2169        }
2170        Ok(())
2171    }
2172    fn emit_rbit(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2173        match (sz, src, dst) {
2174            (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
2175                dynasm!(self ; rbit X(dst), X(src));
2176            }
2177            (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
2178                dynasm!(self ; rbit W(dst), W(src));
2179            }
2180            _ => codegen_error!("singlepass can't emit CLS {:?} {:?} {:?}", sz, src, dst),
2181        }
2182        Ok(())
2183    }
2184
2185    fn emit_label(&mut self, label: Label) -> Result<(), CompileError> {
2186        dynasm!(self ; => label);
2187        Ok(())
2188    }
2189    fn emit_load_label(&mut self, reg: GPR, label: Label) -> Result<(), CompileError> {
2190        dynasm!(self ; adr X(reg), =>label);
2191        Ok(())
2192    }
2193
2194    fn emit_b_label(&mut self, label: Label) -> Result<(), CompileError> {
2195        dynasm!(self ; b =>label);
2196        Ok(())
2197    }
2198    fn emit_cbz_label(
2199        &mut self,
2200        sz: Size,
2201        reg: Location,
2202        label: Label,
2203    ) -> Result<(), CompileError> {
2204        match (sz, reg) {
2205            (Size::S32, Location::GPR(reg)) => {
2206                dynasm!(self ; cbz W(reg), =>label);
2207            }
2208            (Size::S64, Location::GPR(reg)) => {
2209                dynasm!(self ; cbz X(reg), =>label);
2210            }
2211            _ => codegen_error!("singlepass can't emit CBZ {:?} {:?} {:?}", sz, reg, label),
2212        }
2213        Ok(())
2214    }
2215    fn emit_cbnz_label(
2216        &mut self,
2217        sz: Size,
2218        reg: Location,
2219        label: Label,
2220    ) -> Result<(), CompileError> {
2221        match (sz, reg) {
2222            (Size::S32, Location::GPR(reg)) => {
2223                dynasm!(self ; cbnz W(reg), =>label);
2224            }
2225            (Size::S64, Location::GPR(reg)) => {
2226                dynasm!(self ; cbnz X(reg), =>label);
2227            }
2228            _ => codegen_error!("singlepass can't emit CBNZ {:?} {:?} {:?}", sz, reg, label),
2229        }
2230        Ok(())
2231    }
2232    fn emit_cbz_label_far(
2233        &mut self,
2234        sz: Size,
2235        reg: Location,
2236        label: Label,
2237    ) -> Result<(), CompileError> {
2238        let near_label: Label = self.get_label();
2239        let continue_label: Label = self.get_label();
2240
2241        match (sz, reg) {
2242            (Size::S32, Location::GPR(reg)) => {
2243                dynasm!(self ; cbz W(reg), => near_label);
2244                dynasm!(self ; b => continue_label);
2245            }
2246            (Size::S64, Location::GPR(reg)) => {
2247                dynasm!(self ; cbz X(reg), => near_label);
2248                dynasm!(self ; b => continue_label);
2249            }
2250            _ => codegen_error!("singlepass can't emit CBZ {:?} {:?} {:?}", sz, reg, label),
2251        }
2252        self.emit_label(near_label)?;
2253        dynasm!(self ; b => label );
2254
2255        self.emit_label(continue_label)?;
2256
2257        Ok(())
2258    }
2259    fn emit_tbz_label(
2260        &mut self,
2261        sz: Size,
2262        reg: Location,
2263        n: u32,
2264        label: Label,
2265    ) -> Result<(), CompileError> {
2266        match (sz, reg) {
2267            (Size::S32, Location::GPR(reg)) => {
2268                dynasm!(self ; tbz W(reg), n, =>label);
2269            }
2270            (Size::S64, Location::GPR(reg)) => {
2271                dynasm!(self ; tbz X(reg), n, =>label);
2272            }
2273            _ => codegen_error!(
2274                "singlepass can't emit TBZ {:?} {:?} {:?} {:?}",
2275                sz,
2276                reg,
2277                n,
2278                label
2279            ),
2280        }
2281        Ok(())
2282    }
2283    fn emit_tbnz_label(
2284        &mut self,
2285        sz: Size,
2286        reg: Location,
2287        n: u32,
2288        label: Label,
2289    ) -> Result<(), CompileError> {
2290        match (sz, reg) {
2291            (Size::S32, Location::GPR(reg)) => {
2292                dynasm!(self ; tbnz W(reg), n, =>label);
2293            }
2294            (Size::S64, Location::GPR(reg)) => {
2295                dynasm!(self ; tbnz X(reg), n, =>label);
2296            }
2297            _ => codegen_error!(
2298                "singlepass can't emit TBNZ {:?} {:?} {:?} {:?}",
2299                sz,
2300                reg,
2301                n,
2302                label
2303            ),
2304        }
2305        Ok(())
2306    }
2307    fn emit_bcond_label(&mut self, condition: Condition, label: Label) -> Result<(), CompileError> {
2308        match condition {
2309            Condition::Eq => dynasm!(self ; b.eq => label),
2310            Condition::Ne => dynasm!(self ; b.ne => label),
2311            Condition::Cs => dynasm!(self ; b.cs => label),
2312            Condition::Cc => dynasm!(self ; b.cc => label),
2313            Condition::Mi => dynasm!(self ; b.mi => label),
2314            Condition::Pl => dynasm!(self ; b.pl => label),
2315            Condition::Vs => dynasm!(self ; b.vs => label),
2316            Condition::Vc => dynasm!(self ; b.vc => label),
2317            Condition::Hi => dynasm!(self ; b.hi => label),
2318            Condition::Ls => dynasm!(self ; b.ls => label),
2319            Condition::Ge => dynasm!(self ; b.ge => label),
2320            Condition::Lt => dynasm!(self ; b.lt => label),
2321            Condition::Gt => dynasm!(self ; b.gt => label),
2322            Condition::Le => dynasm!(self ; b.le => label),
2323            Condition::Al => dynasm!(self ; b => label),
2324        }
2325        Ok(())
2326    }
2327    fn emit_bcond_label_far(
2328        &mut self,
2329        condition: Condition,
2330        label: Label,
2331    ) -> Result<(), CompileError> {
2332        let cont: Label = self.get_label();
2333        match condition {
2334            // if not condition than continue
2335            Condition::Eq => dynasm!(self ; b.ne => cont),
2336            Condition::Ne => dynasm!(self ; b.eq => cont),
2337            Condition::Cs => dynasm!(self ; b.cc => cont),
2338            Condition::Cc => dynasm!(self ; b.cs => cont),
2339            Condition::Mi => dynasm!(self ; b.pl => cont),
2340            Condition::Pl => dynasm!(self ; b.mi => cont),
2341            Condition::Vs => dynasm!(self ; b.vc => cont),
2342            Condition::Vc => dynasm!(self ; b.vs => cont),
2343            Condition::Hi => dynasm!(self ; b.ls => cont),
2344            Condition::Ls => dynasm!(self ; b.hi => cont),
2345            Condition::Ge => dynasm!(self ; b.lt => cont),
2346            Condition::Lt => dynasm!(self ; b.ge => cont),
2347            Condition::Gt => dynasm!(self ; b.le => cont),
2348            Condition::Le => dynasm!(self ; b.gt => cont),
2349            Condition::Al => { /*nothing*/ }
2350        }
2351        dynasm!(self ; b => label);
2352        self.emit_label(cont)?;
2353        Ok(())
2354    }
2355    fn emit_b_register(&mut self, reg: GPR) -> Result<(), CompileError> {
2356        dynasm!(self ; br X(reg));
2357        Ok(())
2358    }
2359    fn emit_call_label(&mut self, label: Label) -> Result<(), CompileError> {
2360        dynasm!(self ; bl =>label);
2361        Ok(())
2362    }
2363    fn emit_call_register(&mut self, reg: GPR) -> Result<(), CompileError> {
2364        dynasm!(self ; blr X(reg));
2365        Ok(())
2366    }
2367    fn emit_ret(&mut self) -> Result<(), CompileError> {
2368        dynasm!(self ; ret);
2369        Ok(())
2370    }
2371
2372    fn emit_udf(&mut self, payload: u16) -> Result<(), CompileError> {
2373        dynasm!(self ; udf payload as u32);
2374        Ok(())
2375    }
2376    fn emit_dmb(&mut self) -> Result<(), CompileError> {
2377        dynasm!(self ; dmb ish);
2378        Ok(())
2379    }
2380    fn emit_brk(&mut self) -> Result<(), CompileError> {
2381        dynasm!(self ; brk 0);
2382        Ok(())
2383    }
2384
2385    fn emit_fcmp(&mut self, sz: Size, src1: Location, src2: Location) -> Result<(), CompileError> {
2386        match (sz, src1, src2) {
2387            (Size::S32, Location::SIMD(src1), Location::SIMD(src2)) => {
2388                dynasm!(self ; fcmp S(src1), S(src2));
2389            }
2390            (Size::S64, Location::SIMD(src1), Location::SIMD(src2)) => {
2391                dynasm!(self ; fcmp D(src1), D(src2));
2392            }
2393            _ => codegen_error!("singlepass can't emit FCMP {:?} {:?} {:?}", sz, src1, src2),
2394        }
2395        Ok(())
2396    }
2397
2398    fn emit_fneg(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2399        match (sz, src, dst) {
2400            (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2401                dynasm!(self ; fneg S(dst), S(src));
2402            }
2403            (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2404                dynasm!(self ; fneg D(dst), D(src));
2405            }
2406            _ => codegen_error!("singlepass can't emit FNEG {:?} {:?} {:?}", sz, src, dst),
2407        }
2408        Ok(())
2409    }
2410    fn emit_fsqrt(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2411        match (sz, src, dst) {
2412            (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2413                dynasm!(self ; fsqrt S(dst), S(src));
2414            }
2415            (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2416                dynasm!(self ; fsqrt D(dst), D(src));
2417            }
2418            _ => codegen_error!("singlepass can't emit FSQRT {:?} {:?} {:?}", sz, src, dst),
2419        }
2420        Ok(())
2421    }
2422
2423    fn emit_fadd(
2424        &mut self,
2425        sz: Size,
2426        src1: Location,
2427        src2: Location,
2428        dst: Location,
2429    ) -> Result<(), CompileError> {
2430        match (sz, src1, src2, dst) {
2431            (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2432                dynasm!(self ; fadd S(dst), S(src1), S(src2));
2433            }
2434            (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2435                dynasm!(self ; fadd D(dst), D(src1), D(src2));
2436            }
2437            _ => codegen_error!(
2438                "singlepass can't emit FADD {:?} {:?} {:?} {:?}",
2439                sz,
2440                src1,
2441                src2,
2442                dst
2443            ),
2444        }
2445        Ok(())
2446    }
2447    fn emit_fsub(
2448        &mut self,
2449        sz: Size,
2450        src1: Location,
2451        src2: Location,
2452        dst: Location,
2453    ) -> Result<(), CompileError> {
2454        match (sz, src1, src2, dst) {
2455            (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2456                dynasm!(self ; fsub S(dst), S(src1), S(src2));
2457            }
2458            (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2459                dynasm!(self ; fsub D(dst), D(src1), D(src2));
2460            }
2461            _ => codegen_error!(
2462                "singlepass can't emit FSUB {:?} {:?} {:?} {:?}",
2463                sz,
2464                src1,
2465                src2,
2466                dst
2467            ),
2468        }
2469        Ok(())
2470    }
2471    fn emit_fmul(
2472        &mut self,
2473        sz: Size,
2474        src1: Location,
2475        src2: Location,
2476        dst: Location,
2477    ) -> Result<(), CompileError> {
2478        match (sz, src1, src2, dst) {
2479            (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2480                dynasm!(self ; fmul S(dst), S(src1), S(src2));
2481            }
2482            (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2483                dynasm!(self ; fmul D(dst), D(src1), D(src2));
2484            }
2485            _ => codegen_error!(
2486                "singlepass can't emit FMUL {:?} {:?} {:?} {:?}",
2487                sz,
2488                src1,
2489                src2,
2490                dst
2491            ),
2492        }
2493        Ok(())
2494    }
2495    fn emit_fdiv(
2496        &mut self,
2497        sz: Size,
2498        src1: Location,
2499        src2: Location,
2500        dst: Location,
2501    ) -> Result<(), CompileError> {
2502        match (sz, src1, src2, dst) {
2503            (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2504                dynasm!(self ; fdiv S(dst), S(src1), S(src2));
2505            }
2506            (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2507                dynasm!(self ; fdiv D(dst), D(src1), D(src2));
2508            }
2509            _ => codegen_error!(
2510                "singlepass can't emit FDIV {:?} {:?} {:?} {:?}",
2511                sz,
2512                src1,
2513                src2,
2514                dst
2515            ),
2516        }
2517        Ok(())
2518    }
2519
2520    fn emit_fmin(
2521        &mut self,
2522        sz: Size,
2523        src1: Location,
2524        src2: Location,
2525        dst: Location,
2526    ) -> Result<(), CompileError> {
2527        match (sz, src1, src2, dst) {
2528            (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2529                dynasm!(self ; fmin S(dst), S(src1), S(src2));
2530            }
2531            (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2532                dynasm!(self ; fmin D(dst), D(src1), D(src2));
2533            }
2534            _ => codegen_error!(
2535                "singlepass can't emit FMIN {:?} {:?} {:?} {:?}",
2536                sz,
2537                src1,
2538                src2,
2539                dst
2540            ),
2541        }
2542        Ok(())
2543    }
2544    fn emit_fmax(
2545        &mut self,
2546        sz: Size,
2547        src1: Location,
2548        src2: Location,
2549        dst: Location,
2550    ) -> Result<(), CompileError> {
2551        match (sz, src1, src2, dst) {
2552            (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2553                dynasm!(self ; fmax S(dst), S(src1), S(src2));
2554            }
2555            (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2556                dynasm!(self ; fmax D(dst), D(src1), D(src2));
2557            }
2558            _ => codegen_error!(
2559                "singlepass can't emit FMAX {:?} {:?} {:?} {:?}",
2560                sz,
2561                src1,
2562                src2,
2563                dst
2564            ),
2565        }
2566        Ok(())
2567    }
2568
2569    fn emit_frintz(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2570        match (sz, src, dst) {
2571            (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2572                dynasm!(self ; frintz S(dst), S(src));
2573            }
2574            (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2575                dynasm!(self ; frintz D(dst), D(src));
2576            }
2577            _ => codegen_error!("singlepass can't emit FRINTZ {:?} {:?} {:?}", sz, src, dst),
2578        }
2579        Ok(())
2580    }
2581    fn emit_frintn(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2582        match (sz, src, dst) {
2583            (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2584                dynasm!(self ; frintn S(dst), S(src));
2585            }
2586            (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2587                dynasm!(self ; frintn D(dst), D(src));
2588            }
2589            _ => codegen_error!("singlepass can't emit FRINTN {:?} {:?} {:?}", sz, src, dst),
2590        }
2591        Ok(())
2592    }
2593    fn emit_frintm(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2594        match (sz, src, dst) {
2595            (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2596                dynasm!(self ; frintm S(dst), S(src));
2597            }
2598            (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2599                dynasm!(self ; frintm D(dst), D(src));
2600            }
2601            _ => codegen_error!("singlepass can't emit FRINTM {:?} {:?} {:?}", sz, src, dst),
2602        }
2603        Ok(())
2604    }
2605    fn emit_frintp(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2606        match (sz, src, dst) {
2607            (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2608                dynasm!(self ; frintp S(dst), S(src));
2609            }
2610            (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2611                dynasm!(self ; frintp D(dst), D(src));
2612            }
2613            _ => codegen_error!("singlepass can't emit FRINTP {:?} {:?} {:?}", sz, src, dst),
2614        }
2615        Ok(())
2616    }
2617
2618    fn emit_scvtf(
2619        &mut self,
2620        sz_in: Size,
2621        src: Location,
2622        sz_out: Size,
2623        dst: Location,
2624    ) -> Result<(), CompileError> {
2625        match (sz_in, src, sz_out, dst) {
2626            (Size::S32, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
2627                dynasm!(self ; scvtf S(dst), W(src));
2628            }
2629            (Size::S64, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
2630                dynasm!(self ; scvtf S(dst), X(src));
2631            }
2632            (Size::S32, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
2633                dynasm!(self ; scvtf D(dst), W(src));
2634            }
2635            (Size::S64, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
2636                dynasm!(self ; scvtf D(dst), X(src));
2637            }
2638            _ => codegen_error!(
2639                "singlepass can't emit SCVTF {:?} {:?} {:?} {:?}",
2640                sz_in,
2641                src,
2642                sz_out,
2643                dst
2644            ),
2645        }
2646        Ok(())
2647    }
2648    fn emit_ucvtf(
2649        &mut self,
2650        sz_in: Size,
2651        src: Location,
2652        sz_out: Size,
2653        dst: Location,
2654    ) -> Result<(), CompileError> {
2655        match (sz_in, src, sz_out, dst) {
2656            (Size::S32, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
2657                dynasm!(self ; ucvtf S(dst), W(src));
2658            }
2659            (Size::S64, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
2660                dynasm!(self ; ucvtf S(dst), X(src));
2661            }
2662            (Size::S32, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
2663                dynasm!(self ; ucvtf D(dst), W(src));
2664            }
2665            (Size::S64, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
2666                dynasm!(self ; ucvtf D(dst), X(src));
2667            }
2668            _ => codegen_error!(
2669                "singlepass can't emit UCVTF {:?} {:?} {:?} {:?}",
2670                sz_in,
2671                src,
2672                sz_out,
2673                dst
2674            ),
2675        }
2676        Ok(())
2677    }
2678    fn emit_fcvt(&mut self, sz_in: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2679        match (sz_in, src, dst) {
2680            (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2681                dynasm!(self ; fcvt D(dst), S(src));
2682            }
2683            (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2684                dynasm!(self ; fcvt S(dst), D(src));
2685            }
2686            _ => codegen_error!(
2687                "singlepass can't emit UCVTF {:?} {:?} {:?}",
2688                sz_in,
2689                src,
2690                dst
2691            ),
2692        }
2693        Ok(())
2694    }
2695    fn emit_fcvtzs(
2696        &mut self,
2697        sz_in: Size,
2698        src: Location,
2699        sz_out: Size,
2700        dst: Location,
2701    ) -> Result<(), CompileError> {
2702        match (sz_in, src, sz_out, dst) {
2703            (Size::S32, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
2704                dynasm!(self ; fcvtzs W(dst), S(src));
2705            }
2706            (Size::S64, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
2707                dynasm!(self ; fcvtzs W(dst), D(src));
2708            }
2709            (Size::S32, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
2710                dynasm!(self ; fcvtzs X(dst), S(src));
2711            }
2712            (Size::S64, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
2713                dynasm!(self ; fcvtzs X(dst), D(src));
2714            }
2715            _ => codegen_error!(
2716                "singlepass can't emit FCVTZS {:?} {:?} {:?} {:?}",
2717                sz_in,
2718                src,
2719                sz_out,
2720                dst
2721            ),
2722        }
2723        Ok(())
2724    }
2725    fn emit_fcvtzu(
2726        &mut self,
2727        sz_in: Size,
2728        src: Location,
2729        sz_out: Size,
2730        dst: Location,
2731    ) -> Result<(), CompileError> {
2732        match (sz_in, src, sz_out, dst) {
2733            (Size::S32, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
2734                dynasm!(self ; fcvtzu W(dst), S(src));
2735            }
2736            (Size::S64, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
2737                dynasm!(self ; fcvtzu W(dst), D(src));
2738            }
2739            (Size::S32, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
2740                dynasm!(self ; fcvtzu X(dst), S(src));
2741            }
2742            (Size::S64, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
2743                dynasm!(self ; fcvtzu X(dst), D(src));
2744            }
2745            _ => codegen_error!(
2746                "singlepass can't emit FCVTZU {:?} {:?} {:?} {:?}",
2747                sz_in,
2748                src,
2749                sz_out,
2750                dst
2751            ),
2752        }
2753        Ok(())
2754    }
2755
2756    // 1 011 0100 0100 000 => fpcr
2757    fn emit_read_fpcr(&mut self, reg: GPR) -> Result<(), CompileError> {
2758        dynasm!(self ; mrs X(reg), 0b1_011_0100_0100_000);
2759        Ok(())
2760    }
2761    fn emit_write_fpcr(&mut self, reg: GPR) -> Result<(), CompileError> {
2762        dynasm!(self ; msr 0b1_011_0100_0100_000, X(reg));
2763        Ok(())
2764    }
2765    // 1 011 0100 0100 001 => fpsr
2766    fn emit_read_fpsr(&mut self, reg: GPR) -> Result<(), CompileError> {
2767        dynasm!(self ; mrs X(reg), 0b1_011_0100_0100_001);
2768        Ok(())
2769    }
2770    fn emit_write_fpsr(&mut self, reg: GPR) -> Result<(), CompileError> {
2771        dynasm!(self ; msr 0b1_011_0100_0100_001, X(reg));
2772        Ok(())
2773    }
2774
2775    fn emit_cnt(&mut self, src: NEON, dst: NEON) -> Result<(), CompileError> {
2776        dynasm!(self ; cnt V(dst).B8, V(src).B8);
2777        Ok(())
2778    }
2779
2780    fn emit_addv(&mut self, src: NEON, dst: NEON) -> Result<(), CompileError> {
2781        dynasm!(self ; addv B(dst), V(src).B8);
2782        Ok(())
2783    }
2784
2785    #[allow(clippy::unit_arg)]
2786    fn emit_fmov(
2787        &mut self,
2788        src_size: Size,
2789        src: Location,
2790        dst_size: Size,
2791        dst: Location,
2792    ) -> Result<(), CompileError> {
2793        let mut err = false;
2794        match (src, dst) {
2795            (AbstractLocation::GPR(src), AbstractLocation::SIMD(dst)) => {
2796                match (src_size, dst_size) {
2797                    (Size::S32, Size::S32) => dynasm!(self ; fmov S(dst), W(src)),
2798                    (Size::S64, Size::S64) => dynasm!(self ; fmov D(dst), X(src)),
2799                    _ => {
2800                        err = true;
2801                    }
2802                }
2803            }
2804            (AbstractLocation::SIMD(src), AbstractLocation::GPR(dst)) => {
2805                match (src_size, dst_size) {
2806                    (Size::S32, Size::S32) => dynasm!(self ; fmov W(dst), S(src)),
2807                    (Size::S64, Size::S64) => dynasm!(self ; fmov X(dst), D(src)),
2808                    _ => {
2809                        err = true;
2810                    }
2811                }
2812            }
2813            // (AbstractLocation::SIMD(src), AbstractLocation::SIMD(dst)) => todo!(),
2814            // (AbstractLocation::SIMD(src), AbstractLocation::Imm32(dst)) => todo!(),
2815            // (AbstractLocation::SIMD(src), AbstractLocation::Imm64(dst)) => todo!(),
2816            _ => {
2817                err = true;
2818            }
2819        };
2820
2821        if err {
2822            codegen_error!(
2823                "singlepass can't generate fmov instruction for src_size: {:?}, src: {:?}, dst_size: {:?}, dst: {:?}",
2824                src_size,
2825                src,
2826                dst_size,
2827                dst,
2828            )
2829        }
2830        Ok(())
2831    }
2832}
2833
2834pub fn gen_std_trampoline_arm64(
2835    sig: &FunctionType,
2836    calling_convention: CallingConvention,
2837) -> Result<FunctionBody, CompileError> {
2838    let mut a = Assembler::new(0);
2839
2840    let fptr = GPR::X27;
2841    let args = GPR::X28;
2842
2843    dynasm!(a
2844        ; sub sp, sp, 32
2845        ; stp x29, x30, [sp]
2846        ; stp X(fptr), X(args), [sp, 16]
2847        ; mov x29, sp
2848        ; mov X(fptr), x1
2849        ; mov X(args), x2
2850    );
2851
2852    let stack_args = sig.params().len().saturating_sub(7); //1st arg is ctx, not an actual arg
2853    let mut stack_offset = stack_args as u32 * 8;
2854    if stack_args > 0 {
2855        if stack_offset % 16 != 0 {
2856            stack_offset += 8;
2857            assert!(stack_offset % 16 == 0);
2858        }
2859        dynasm!(a ; sub sp, sp, stack_offset);
2860    }
2861
2862    // Move arguments to their locations.
2863    // `callee_vmctx` is already in the first argument register, so no need to move.
2864    let mut caller_stack_offset: i32 = 0;
2865    for (i, param) in sig.params().iter().enumerate() {
2866        let sz = match *param {
2867            Type::I32 | Type::F32 => Size::S32,
2868            Type::I64 | Type::F64 => Size::S64,
2869            Type::ExternRef => Size::S64,
2870            Type::FuncRef => Size::S64,
2871            _ => codegen_error!(
2872                "singlepass unsupported param type for trampoline {:?}",
2873                *param
2874            ),
2875        };
2876        match i {
2877            0..=6 => {
2878                a.emit_ldr(
2879                    sz,
2880                    Location::GPR(GPR::from_index(i + 1).unwrap()),
2881                    Location::Memory(args, (i * 16) as i32),
2882                )?;
2883            }
2884            _ => {
2885                #[allow(clippy::single_match)]
2886                match calling_convention {
2887                    CallingConvention::AppleAarch64 => {
2888                        let sz = 1
2889                            << match sz {
2890                                Size::S8 => 0,
2891                                Size::S16 => 1,
2892                                Size::S32 => 2,
2893                                Size::S64 => 3,
2894                            };
2895                        // align first
2896                        if sz > 1 && caller_stack_offset & (sz - 1) != 0 {
2897                            caller_stack_offset = (caller_stack_offset + (sz - 1)) & !(sz - 1);
2898                        }
2899                    }
2900                    _ => (),
2901                };
2902                // using X16 as scratch reg
2903                a.emit_ldr(
2904                    sz,
2905                    Location::GPR(GPR::X16),
2906                    Location::Memory(args, (i * 16) as i32),
2907                )?;
2908                a.emit_str(
2909                    sz,
2910                    Location::GPR(GPR::X16),
2911                    Location::Memory(GPR::XzrSp, caller_stack_offset),
2912                )?;
2913                match calling_convention {
2914                    CallingConvention::AppleAarch64 => {
2915                        caller_stack_offset += 1
2916                            << match sz {
2917                                Size::S8 => 0,
2918                                Size::S16 => 1,
2919                                Size::S32 => 2,
2920                                Size::S64 => 3,
2921                            };
2922                    }
2923                    _ => {
2924                        caller_stack_offset += 8;
2925                    }
2926                }
2927            }
2928        }
2929    }
2930
2931    dynasm!(a  ; blr X(fptr));
2932
2933    // Write return value.
2934    if !sig.results().is_empty() {
2935        a.emit_str(Size::S64, Location::GPR(GPR::X0), Location::Memory(args, 0))?;
2936    }
2937
2938    // Restore stack.
2939    dynasm!(a
2940        ; ldp X(fptr), X(args), [x29, 16]
2941        ; ldp x29, x30, [x29]
2942        ; add sp, sp, 32 + stack_offset as u32
2943        ; ret
2944    );
2945
2946    let mut body = a.finalize().unwrap();
2947    body.shrink_to_fit();
2948    Ok(FunctionBody {
2949        body,
2950        unwind_info: None,
2951    })
2952}
2953// Generates dynamic import function call trampoline for a function type.
2954pub fn gen_std_dynamic_import_trampoline_arm64(
2955    vmoffsets: &VMOffsets,
2956    sig: &FunctionType,
2957    calling_convention: CallingConvention,
2958) -> Result<FunctionBody, CompileError> {
2959    let mut a = Assembler::new(0);
2960    // Allocate argument array.
2961    let stack_offset: usize = 16 * std::cmp::max(sig.params().len(), sig.results().len());
2962    // Save LR and X26, as scratch register
2963    a.emit_stpdb(
2964        Size::S64,
2965        Location::GPR(GPR::X30),
2966        Location::GPR(GPR::X26),
2967        GPR::XzrSp,
2968        16,
2969    )?;
2970
2971    if stack_offset != 0 {
2972        if stack_offset < 0x1000 {
2973            a.emit_sub(
2974                Size::S64,
2975                Location::GPR(GPR::XzrSp),
2976                Location::Imm32(stack_offset as _),
2977                Location::GPR(GPR::XzrSp),
2978            )?;
2979        } else {
2980            a.emit_mov_imm(Location::GPR(GPR::X26), stack_offset as u64)?;
2981            a.emit_sub(
2982                Size::S64,
2983                Location::GPR(GPR::XzrSp),
2984                Location::GPR(GPR::X26),
2985                Location::GPR(GPR::XzrSp),
2986            )?;
2987        }
2988    }
2989
2990    // Copy arguments.
2991    if !sig.params().is_empty() {
2992        let mut argalloc = ArgumentRegisterAllocator::default();
2993        argalloc.next(Type::I64, calling_convention).unwrap(); // skip VMContext
2994
2995        let mut stack_param_count: usize = 0;
2996
2997        for (i, ty) in sig.params().iter().enumerate() {
2998            let source_loc = match argalloc.next(*ty, calling_convention)? {
2999                Some(ARM64Register::GPR(gpr)) => Location::GPR(gpr),
3000                Some(ARM64Register::NEON(neon)) => Location::SIMD(neon),
3001                None => {
3002                    let sz = match calling_convention {
3003                        CallingConvention::AppleAarch64 => match *ty {
3004                            Type::I32 | Type::F32 => Size::S32,
3005                            _ => {
3006                                if stack_param_count & 7 != 0 {
3007                                    stack_param_count = (stack_param_count + 7) & !7;
3008                                };
3009                                Size::S64
3010                            }
3011                        },
3012                        _ => Size::S64,
3013                    };
3014                    a.emit_ldr(
3015                        sz,
3016                        Location::GPR(GPR::X26),
3017                        Location::Memory(GPR::XzrSp, (stack_offset + 16 + stack_param_count) as _),
3018                    )?;
3019                    stack_param_count += match sz {
3020                        Size::S32 => 4,
3021                        Size::S64 => 8,
3022                        _ => codegen_error!(
3023                            "singlepass unreachable in gen_std_dynamic_import_trampoline_arm64"
3024                        ),
3025                    };
3026                    Location::GPR(GPR::X26)
3027                }
3028            };
3029            a.emit_str(
3030                Size::S64,
3031                source_loc,
3032                Location::Memory(GPR::XzrSp, (i * 16) as _),
3033            )?;
3034
3035            // Zero upper 64 bits.
3036            a.emit_str(
3037                Size::S64,
3038                Location::GPR(GPR::XzrSp),                       // XZR here
3039                Location::Memory(GPR::XzrSp, (i * 16 + 8) as _), // XSP here
3040            )?;
3041        }
3042    }
3043
3044    #[allow(clippy::match_single_binding)]
3045    match calling_convention {
3046        _ => {
3047            // Load target address.
3048            let offset = vmoffsets.vmdynamicfunction_import_context_address();
3049            a.emit_ldur(Size::S64, Location::GPR(GPR::X26), GPR::X0, offset as i32)?;
3050            // Load values array.
3051            a.emit_add(
3052                Size::S64,
3053                Location::GPR(GPR::XzrSp),
3054                Location::Imm8(0),
3055                Location::GPR(GPR::X1),
3056            )?;
3057        }
3058    };
3059
3060    // Call target.
3061    a.emit_call_register(GPR::X26)?;
3062
3063    // Fetch return value.
3064    if !sig.results().is_empty() {
3065        assert_eq!(sig.results().len(), 1);
3066        a.emit_ldr(
3067            Size::S64,
3068            Location::GPR(GPR::X0),
3069            Location::Memory(GPR::XzrSp, 0),
3070        )?;
3071    }
3072
3073    // Release values array.
3074    if stack_offset != 0 {
3075        if stack_offset < 0x1000 {
3076            a.emit_add(
3077                Size::S64,
3078                Location::GPR(GPR::XzrSp),
3079                Location::Imm32(stack_offset as _),
3080                Location::GPR(GPR::XzrSp),
3081            )?;
3082        } else {
3083            a.emit_mov_imm(Location::GPR(GPR::X26), stack_offset as u64)?;
3084            a.emit_add(
3085                Size::S64,
3086                Location::GPR(GPR::XzrSp),
3087                Location::GPR(GPR::X26),
3088                Location::GPR(GPR::XzrSp),
3089            )?;
3090        }
3091    }
3092    a.emit_ldpia(
3093        Size::S64,
3094        Location::GPR(GPR::X30),
3095        Location::GPR(GPR::X26),
3096        GPR::XzrSp,
3097        16,
3098    )?;
3099
3100    // Return.
3101    a.emit_ret()?;
3102
3103    let mut body = a.finalize().unwrap();
3104    body.shrink_to_fit();
3105    Ok(FunctionBody {
3106        body,
3107        unwind_info: None,
3108    })
3109}
3110// Singlepass calls import functions through a trampoline.
3111pub fn gen_import_call_trampoline_arm64(
3112    vmoffsets: &VMOffsets,
3113    index: FunctionIndex,
3114    sig: &FunctionType,
3115    calling_convention: CallingConvention,
3116) -> Result<CustomSection, CompileError> {
3117    let mut a = Assembler::new(0);
3118
3119    // Singlepass internally treats all arguments as integers
3120    // For the standard System V calling convention requires
3121    //  floating point arguments to be passed in NEON registers.
3122    //  Translation is expensive, so only do it if needed.
3123    if sig
3124        .params()
3125        .iter()
3126        .any(|&x| x == Type::F32 || x == Type::F64)
3127    {
3128        #[allow(clippy::match_single_binding)]
3129        match calling_convention {
3130            _ => {
3131                // Allocate stack space for arguments.
3132                let stack_offset: i32 = if sig.params().len() > 7 {
3133                    7 * 8
3134                } else {
3135                    (sig.params().len() as i32) * 8
3136                };
3137                let stack_offset = if stack_offset & 15 != 0 {
3138                    stack_offset + 8
3139                } else {
3140                    stack_offset
3141                };
3142                if stack_offset > 0 {
3143                    if stack_offset < 0x1000 {
3144                        a.emit_sub(
3145                            Size::S64,
3146                            Location::GPR(GPR::XzrSp),
3147                            Location::Imm32(stack_offset as u32),
3148                            Location::GPR(GPR::XzrSp),
3149                        )?;
3150                    } else {
3151                        a.emit_mov_imm(Location::GPR(GPR::X16), stack_offset as u64)?;
3152                        a.emit_sub(
3153                            Size::S64,
3154                            Location::GPR(GPR::XzrSp),
3155                            Location::GPR(GPR::X16),
3156                            Location::GPR(GPR::XzrSp),
3157                        )?;
3158                    }
3159                }
3160
3161                // Store all arguments to the stack to prevent overwrite.
3162                static PARAM_REGS: &[GPR] = &[
3163                    GPR::X1,
3164                    GPR::X2,
3165                    GPR::X3,
3166                    GPR::X4,
3167                    GPR::X5,
3168                    GPR::X6,
3169                    GPR::X7,
3170                ];
3171                let mut param_locations = vec![];
3172                /* Clippy is wrong about using `i` to index `PARAM_REGS` here. */
3173                #[allow(clippy::needless_range_loop)]
3174                for i in 0..sig.params().len() {
3175                    let loc = match i {
3176                        0..=6 => {
3177                            let loc = Location::Memory(GPR::XzrSp, (i * 8) as i32);
3178                            a.emit_str(Size::S64, Location::GPR(PARAM_REGS[i]), loc)?;
3179                            loc
3180                        }
3181                        _ => Location::Memory(GPR::XzrSp, stack_offset + ((i - 7) * 8) as i32),
3182                    };
3183                    param_locations.push(loc);
3184                }
3185
3186                // Copy arguments.
3187                let mut caller_stack_offset: i32 = 0;
3188                let mut argalloc = ArgumentRegisterAllocator::default();
3189                argalloc.next(Type::I64, calling_convention).unwrap(); // skip VMContext
3190                for (i, ty) in sig.params().iter().enumerate() {
3191                    let prev_loc = param_locations[i];
3192                    let targ = match argalloc.next(*ty, calling_convention)? {
3193                        Some(ARM64Register::GPR(gpr)) => Location::GPR(gpr),
3194                        Some(ARM64Register::NEON(neon)) => Location::SIMD(neon),
3195                        None => {
3196                            // No register can be allocated. Put this argument on the stack.
3197                            a.emit_ldr(Size::S64, Location::GPR(GPR::X16), prev_loc)?;
3198                            a.emit_str(
3199                                Size::S64,
3200                                Location::GPR(GPR::X16),
3201                                Location::Memory(GPR::XzrSp, stack_offset + caller_stack_offset),
3202                            )?;
3203                            caller_stack_offset += 8;
3204                            continue;
3205                        }
3206                    };
3207                    a.emit_ldr(Size::S64, targ, prev_loc)?;
3208                }
3209
3210                // Restore stack pointer.
3211                if stack_offset > 0 {
3212                    if stack_offset < 0x1000 {
3213                        a.emit_add(
3214                            Size::S64,
3215                            Location::GPR(GPR::XzrSp),
3216                            Location::Imm32(stack_offset as u32),
3217                            Location::GPR(GPR::XzrSp),
3218                        )?;
3219                    } else {
3220                        a.emit_mov_imm(Location::GPR(GPR::X16), stack_offset as u64)?;
3221                        a.emit_add(
3222                            Size::S64,
3223                            Location::GPR(GPR::XzrSp),
3224                            Location::GPR(GPR::X16),
3225                            Location::GPR(GPR::XzrSp),
3226                        )?;
3227                    }
3228                }
3229            }
3230        }
3231    }
3232
3233    // Emits a tail call trampoline that loads the address of the target import function
3234    // from Ctx and jumps to it.
3235
3236    let offset = vmoffsets.vmctx_vmfunction_import(index);
3237    // for ldr, offset needs to be a multiple of 8, wich often is not
3238    // so use ldur, but then offset is limited to -255 .. +255. It will be positive here
3239    let offset =
3240        if (offset > 0 && offset < 0xF8) || (offset > 0 && offset < 0x7FF8 && (offset & 7) == 0) {
3241            offset
3242        } else {
3243            a.emit_mov_imm(Location::GPR(GPR::X16), (offset as i64) as u64)?;
3244            a.emit_add(
3245                Size::S64,
3246                Location::GPR(GPR::X0),
3247                Location::GPR(GPR::X16),
3248                Location::GPR(GPR::X0),
3249            )?;
3250            0
3251        };
3252    #[allow(clippy::match_single_binding)]
3253    match calling_convention {
3254        _ => {
3255            if (offset & 7) == 0 {
3256                a.emit_ldr(
3257                    Size::S64,
3258                    Location::GPR(GPR::X16),
3259                    Location::Memory(GPR::X0, offset as i32), // function pointer
3260                )?;
3261                a.emit_ldr(
3262                    Size::S64,
3263                    Location::GPR(GPR::X0),
3264                    Location::Memory(GPR::X0, offset as i32 + 8), // target vmctx
3265                )?;
3266            } else {
3267                a.emit_ldur(
3268                    Size::S64,
3269                    Location::GPR(GPR::X16),
3270                    GPR::X0,
3271                    offset as i32, // function pointer
3272                )?;
3273                a.emit_ldur(
3274                    Size::S64,
3275                    Location::GPR(GPR::X0),
3276                    GPR::X0,
3277                    offset as i32 + 8, // target vmctx
3278                )?;
3279            }
3280        }
3281    }
3282    a.emit_b_register(GPR::X16)?;
3283
3284    let mut contents = a.finalize().unwrap();
3285    contents.shrink_to_fit();
3286    let section_body = SectionBody::new_with_vec(contents);
3287
3288    Ok(CustomSection {
3289        protection: CustomSectionProtection::ReadExecute,
3290        alignment: None,
3291        bytes: section_body,
3292        relocations: vec![],
3293    })
3294}