wasmer_compiler_singlepass/
emitter_arm64.rs

1pub use crate::{
2    arm64_decl::{ARM64Register, ArgumentRegisterAllocator, GPR, NEON},
3    location::{Multiplier, Reg},
4    machine::{Label, Offset},
5};
6use crate::{codegen_error, common_decl::Size, location::Location as AbstractLocation};
7use dynasm::dynasm;
8pub use dynasmrt::aarch64::{encode_logical_immediate_32bit, encode_logical_immediate_64bit};
9use dynasmrt::{
10    AssemblyOffset, DynamicLabel, DynasmApi, DynasmLabelApi, VecAssembler,
11    aarch64::Aarch64Relocation,
12};
13use wasmer_compiler::types::{
14    function::FunctionBody,
15    section::{CustomSection, CustomSectionProtection, SectionBody},
16};
17use wasmer_types::{
18    CompileError, FunctionIndex, FunctionType, Type, VMOffsets, target::CallingConvention,
19};
20
21type Assembler = VecAssembler<Aarch64Relocation>;
22
23/// Force `dynasm!` to use the correct arch (aarch64) when cross-compiling.
24/// `dynasm!` proc-macro tries to auto-detect it by default by looking at the
25/// `target_arch`, but it sees the `target_arch` of the proc-macro itself, which
26/// is always equal to host, even when cross-compiling.
27macro_rules! dynasm {
28    ($a:expr_2021 ; $($tt:tt)*) => {
29        dynasm::dynasm!(
30            $a
31            ; .arch aarch64
32            ; $($tt)*
33        )
34    };
35}
36
37pub type Location = AbstractLocation<GPR, NEON>;
38
39#[derive(Copy, Clone, Debug, Eq, PartialEq)]
40#[allow(dead_code)]
41#[repr(u8)]
42pub enum Condition {
43    // meaning for cmp or sub
44    /// Equal
45    Eq = 0,
46    /// Not equal
47    Ne = 1,
48    /// Unsigned higher or same (or carry set)
49    Cs = 2,
50    /// Unsigned lower (or carry clear)
51    Cc = 3,
52    /// Negative. The mnemonic stands for "minus"
53    Mi = 4,
54    /// Positive or zero. The mnemonic stands for "plus"
55    Pl = 5,
56    /// Signed overflow. The mnemonic stands for "V set"
57    Vs = 6,
58    /// No signed overflow. The mnemonic stands for "V clear"
59    Vc = 7,
60    /// Unsigned higher
61    Hi = 8,
62    /// Unsigned lower or same
63    Ls = 9,
64    /// Signed greater than or equal
65    Ge = 10,
66    /// Signed less than
67    Lt = 11,
68    /// Signed greater than
69    Gt = 12,
70    /// Signed less than or equal
71    Le = 13,
72    /// Always executed
73    Al = 14,
74}
75
76#[derive(Copy, Clone, Debug, Eq, PartialEq)]
77#[allow(dead_code, clippy::upper_case_acronyms)]
78pub enum NeonOrMemory {
79    NEON(NEON),
80    Memory(GPR, i32),
81}
82
83#[derive(Copy, Clone, Debug)]
84#[allow(dead_code, clippy::upper_case_acronyms)]
85pub enum GPROrMemory {
86    GPR(GPR),
87    Memory(GPR, i32),
88}
89
90#[allow(unused)]
91pub trait EmitterARM64 {
92    fn get_label(&mut self) -> Label;
93    fn get_offset(&self) -> Offset;
94    fn get_jmp_instr_size(&self) -> u8;
95
96    fn finalize_function(&mut self);
97
98    fn emit_str(&mut self, sz: Size, reg: Location, addr: Location) -> Result<(), CompileError>;
99    fn emit_ldr(&mut self, sz: Size, reg: Location, addr: Location) -> Result<(), CompileError>;
100    fn emit_stur(
101        &mut self,
102        sz: Size,
103        reg: Location,
104        addr: GPR,
105        offset: i32,
106    ) -> Result<(), CompileError>;
107    fn emit_ldur(
108        &mut self,
109        sz: Size,
110        reg: Location,
111        addr: GPR,
112        offset: i32,
113    ) -> Result<(), CompileError>;
114    fn emit_strdb(
115        &mut self,
116        sz: Size,
117        reg: Location,
118        addr: GPR,
119        offset: u32,
120    ) -> Result<(), CompileError>;
121    fn emit_stria(
122        &mut self,
123        sz: Size,
124        reg: Location,
125        addr: GPR,
126        offset: u32,
127    ) -> Result<(), CompileError>;
128    fn emit_ldria(
129        &mut self,
130        sz: Size,
131        reg: Location,
132        addr: GPR,
133        offset: u32,
134    ) -> Result<(), CompileError>;
135    fn emit_stpdb(
136        &mut self,
137        sz: Size,
138        reg1: Location,
139        reg2: Location,
140        addr: GPR,
141        offset: u32,
142    ) -> Result<(), CompileError>;
143    fn emit_ldpia(
144        &mut self,
145        sz: Size,
146        reg1: Location,
147        reg2: Location,
148        addr: GPR,
149        offset: u32,
150    ) -> Result<(), CompileError>;
151
152    fn emit_ldrb(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
153    fn emit_ldrh(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
154    fn emit_ldrsb(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
155    fn emit_ldrsh(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
156    fn emit_ldrsw(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
157    fn emit_strb(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
158    fn emit_strh(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
159
160    fn emit_ldaxr(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
161    fn emit_ldaxrb(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
162    fn emit_ldaxrh(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
163    fn emit_stlxr(
164        &mut self,
165        sz: Size,
166        status: Location,
167        reg: Location,
168        dst: Location,
169    ) -> Result<(), CompileError>;
170    fn emit_stlxrb(
171        &mut self,
172        sz: Size,
173        status: Location,
174        reg: Location,
175        dst: Location,
176    ) -> Result<(), CompileError>;
177    fn emit_stlxrh(
178        &mut self,
179        sz: Size,
180        status: Location,
181        reg: Location,
182        dst: Location,
183    ) -> Result<(), CompileError>;
184
185    fn emit_mov(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
186
187    fn emit_movn(&mut self, sz: Size, reg: Location, val: u32) -> Result<(), CompileError>;
188    fn emit_movz(&mut self, reg: Location, val: u32) -> Result<(), CompileError>;
189    fn emit_movk(&mut self, reg: Location, val: u32, shift: u32) -> Result<(), CompileError>;
190
191    fn emit_mov_imm(&mut self, dst: Location, val: u64) -> Result<(), CompileError>;
192
193    fn emit_add(
194        &mut self,
195        sz: Size,
196        src1: Location,
197        src2: Location,
198        dst: Location,
199    ) -> Result<(), CompileError>;
200    fn emit_sub(
201        &mut self,
202        sz: Size,
203        src1: Location,
204        src2: Location,
205        dst: Location,
206    ) -> Result<(), CompileError>;
207    fn emit_mul(
208        &mut self,
209        sz: Size,
210        src1: Location,
211        src2: Location,
212        dst: Location,
213    ) -> Result<(), CompileError>;
214    fn emit_adds(
215        &mut self,
216        sz: Size,
217        src1: Location,
218        src2: Location,
219        dst: Location,
220    ) -> Result<(), CompileError>;
221    fn emit_subs(
222        &mut self,
223        sz: Size,
224        src1: Location,
225        src2: Location,
226        dst: Location,
227    ) -> Result<(), CompileError>;
228
229    fn emit_add_lsl(
230        &mut self,
231        sz: Size,
232        src1: Location,
233        src2: Location,
234        lsl: u32,
235        dst: Location,
236    ) -> Result<(), CompileError>;
237
238    fn emit_cmp(&mut self, sz: Size, left: Location, right: Location) -> Result<(), CompileError>;
239    fn emit_tst(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
240
241    fn emit_lsl(
242        &mut self,
243        sz: Size,
244        src1: Location,
245        src2: Location,
246        dst: Location,
247    ) -> Result<(), CompileError>;
248    fn emit_lsr(
249        &mut self,
250        sz: Size,
251        src1: Location,
252        src2: Location,
253        dst: Location,
254    ) -> Result<(), CompileError>;
255    fn emit_asr(
256        &mut self,
257        sz: Size,
258        src1: Location,
259        src2: Location,
260        dst: Location,
261    ) -> Result<(), CompileError>;
262    fn emit_ror(
263        &mut self,
264        sz: Size,
265        src1: Location,
266        src2: Location,
267        dst: Location,
268    ) -> Result<(), CompileError>;
269
270    fn emit_or(
271        &mut self,
272        sz: Size,
273        src1: Location,
274        src2: Location,
275        dst: Location,
276    ) -> Result<(), CompileError>;
277    fn emit_and(
278        &mut self,
279        sz: Size,
280        src1: Location,
281        src2: Location,
282        dst: Location,
283    ) -> Result<(), CompileError>;
284    fn emit_eor(
285        &mut self,
286        sz: Size,
287        src1: Location,
288        src2: Location,
289        dst: Location,
290    ) -> Result<(), CompileError>;
291
292    fn emit_bfc(
293        &mut self,
294        se: Size,
295        lsb: u32,
296        width: u32,
297        dst: Location,
298    ) -> Result<(), CompileError>;
299    fn emit_bfi(
300        &mut self,
301        se: Size,
302        src: Location,
303        lsb: u32,
304        width: u32,
305        dst: Location,
306    ) -> Result<(), CompileError>;
307
308    fn emit_udiv(
309        &mut self,
310        sz: Size,
311        src1: Location,
312        src2: Location,
313        dst: Location,
314    ) -> Result<(), CompileError>;
315    fn emit_sdiv(
316        &mut self,
317        sz: Size,
318        src1: Location,
319        src2: Location,
320        dst: Location,
321    ) -> Result<(), CompileError>;
322    /// msub : c - a*b -> dst
323    fn emit_msub(
324        &mut self,
325        sz: Size,
326        a: Location,
327        b: Location,
328        c: Location,
329        dst: Location,
330    ) -> Result<(), CompileError>;
331
332    fn emit_sxtb(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
333    fn emit_sxth(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
334    fn emit_sxtw(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
335    fn emit_uxtb(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
336    fn emit_uxth(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
337
338    fn emit_cset(&mut self, sz: Size, dst: Location, cond: Condition) -> Result<(), CompileError>;
339    fn emit_csetm(&mut self, sz: Size, dst: Location, cond: Condition) -> Result<(), CompileError>;
340    fn emit_cinc(
341        &mut self,
342        sz: Size,
343        src: Location,
344        dst: Location,
345        cond: Condition,
346    ) -> Result<(), CompileError>;
347    fn emit_clz(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
348    fn emit_rbit(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
349
350    fn emit_label(&mut self, label: Label) -> Result<(), CompileError>;
351    fn emit_load_label(&mut self, reg: GPR, label: Label) -> Result<(), CompileError>;
352    fn emit_b_label(&mut self, label: Label) -> Result<(), CompileError>;
353    fn emit_cbz_label(&mut self, sz: Size, reg: Location, label: Label)
354    -> Result<(), CompileError>;
355    fn emit_cbnz_label(
356        &mut self,
357        sz: Size,
358        reg: Location,
359        label: Label,
360    ) -> Result<(), CompileError>;
361    fn emit_cbz_label_far(
362        &mut self,
363        sz: Size,
364        reg: Location,
365        label: Label,
366    ) -> Result<(), CompileError>;
367    fn emit_tbz_label(
368        &mut self,
369        sz: Size,
370        reg: Location,
371        n: u32,
372        label: Label,
373    ) -> Result<(), CompileError>;
374    fn emit_tbnz_label(
375        &mut self,
376        sz: Size,
377        reg: Location,
378        n: u32,
379        label: Label,
380    ) -> Result<(), CompileError>;
381    fn emit_bcond_label(&mut self, condition: Condition, label: Label) -> Result<(), CompileError>;
382    fn emit_bcond_label_far(
383        &mut self,
384        condition: Condition,
385        label: Label,
386    ) -> Result<(), CompileError>;
387    fn emit_b_register(&mut self, reg: GPR) -> Result<(), CompileError>;
388    fn emit_call_label(&mut self, label: Label) -> Result<(), CompileError>;
389    fn emit_call_register(&mut self, reg: GPR) -> Result<(), CompileError>;
390    fn emit_ret(&mut self) -> Result<(), CompileError>;
391
392    fn emit_udf(&mut self, payload: u16) -> Result<(), CompileError>;
393    fn emit_dmb(&mut self) -> Result<(), CompileError>;
394    fn emit_brk(&mut self) -> Result<(), CompileError>;
395
396    fn emit_fcmp(&mut self, sz: Size, src1: Location, src2: Location) -> Result<(), CompileError>;
397    fn emit_fneg(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
398    fn emit_fsqrt(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
399
400    fn emit_fadd(
401        &mut self,
402        sz: Size,
403        src1: Location,
404        src2: Location,
405        dst: Location,
406    ) -> Result<(), CompileError>;
407    fn emit_fsub(
408        &mut self,
409        sz: Size,
410        src1: Location,
411        src2: Location,
412        dst: Location,
413    ) -> Result<(), CompileError>;
414    fn emit_fmul(
415        &mut self,
416        sz: Size,
417        src1: Location,
418        src2: Location,
419        dst: Location,
420    ) -> Result<(), CompileError>;
421    fn emit_fdiv(
422        &mut self,
423        sz: Size,
424        src1: Location,
425        src2: Location,
426        dst: Location,
427    ) -> Result<(), CompileError>;
428
429    fn emit_fmin(
430        &mut self,
431        sz: Size,
432        src1: Location,
433        src2: Location,
434        dst: Location,
435    ) -> Result<(), CompileError>;
436    fn emit_fmax(
437        &mut self,
438        sz: Size,
439        src1: Location,
440        src2: Location,
441        dst: Location,
442    ) -> Result<(), CompileError>;
443
444    fn emit_frintz(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
445    fn emit_frintn(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
446    fn emit_frintm(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
447    fn emit_frintp(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
448
449    fn emit_scvtf(
450        &mut self,
451        sz_in: Size,
452        src: Location,
453        sz_out: Size,
454        dst: Location,
455    ) -> Result<(), CompileError>;
456    fn emit_ucvtf(
457        &mut self,
458        sz_in: Size,
459        src: Location,
460        sz_out: Size,
461        dst: Location,
462    ) -> Result<(), CompileError>;
463    fn emit_fcvt(&mut self, sz_in: Size, src: Location, dst: Location) -> Result<(), CompileError>;
464    fn emit_fcvtzs(
465        &mut self,
466        sz_in: Size,
467        src: Location,
468        sz_out: Size,
469        dst: Location,
470    ) -> Result<(), CompileError>;
471    fn emit_fcvtzu(
472        &mut self,
473        sz_in: Size,
474        src: Location,
475        sz_out: Size,
476        dst: Location,
477    ) -> Result<(), CompileError>;
478
479    fn emit_fmov(
480        &mut self,
481        src_size: Size,
482        src: Location,
483        dst_size: Size,
484        dst: Location,
485    ) -> Result<(), CompileError>;
486    fn emit_cnt(&mut self, src: NEON, dst: NEON) -> Result<(), CompileError>;
487    fn emit_addv(&mut self, src: NEON, dst: NEON) -> Result<(), CompileError>;
488    fn emit_read_fpcr(&mut self, reg: GPR) -> Result<(), CompileError>;
489    fn emit_write_fpcr(&mut self, reg: GPR) -> Result<(), CompileError>;
490    fn emit_read_fpsr(&mut self, reg: GPR) -> Result<(), CompileError>;
491    fn emit_write_fpsr(&mut self, reg: GPR) -> Result<(), CompileError>;
492
493    fn arch_supports_canonicalize_nan(&self) -> bool {
494        true
495    }
496
497    fn arch_requires_indirect_call_trampoline(&self) -> bool {
498        false
499    }
500
501    fn arch_emit_indirect_call_with_trampoline(
502        &mut self,
503        _loc: Location,
504    ) -> Result<(), CompileError> {
505        codegen_error!("singlepass arch_emit_indirect_call_with_trampoline unimplemented")
506    }
507}
508
509impl EmitterARM64 for Assembler {
510    fn get_label(&mut self) -> DynamicLabel {
511        self.new_dynamic_label()
512    }
513
514    fn get_offset(&self) -> AssemblyOffset {
515        self.offset()
516    }
517
518    fn get_jmp_instr_size(&self) -> u8 {
519        4 // relative jump, not full 32bits capable
520    }
521
522    fn finalize_function(&mut self) {
523        dynasm!(
524            self
525            ; const_neg_one_32:
526            ; .i32 -1
527            ; const_zero_32:
528            ; .i32 0
529            ; const_pos_one_32:
530            ; .i32 1
531        );
532    }
533
534    fn emit_str(&mut self, sz: Size, reg: Location, addr: Location) -> Result<(), CompileError> {
535        match (sz, reg, addr) {
536            (Size::S64, Location::GPR(reg), Location::Memory(addr, disp)) => {
537                let disp = disp as u32;
538                assert!((disp & 0x7) == 0 && (disp < 0x8000));
539                dynasm!(self ; str X(reg), [X(addr), disp]);
540            }
541            (Size::S32, Location::GPR(reg), Location::Memory(addr, disp)) => {
542                let disp = disp as u32;
543                assert!((disp & 0x3) == 0 && (disp < 0x4000));
544                dynasm!(self ; str W(reg), [X(addr), disp]);
545            }
546            (Size::S16, Location::GPR(reg), Location::Memory(addr, disp)) => {
547                let disp = disp as u32;
548                assert!((disp & 0x1) == 0 && (disp < 0x2000));
549                dynasm!(self ; strh W(reg), [X(addr), disp]);
550            }
551            (Size::S8, Location::GPR(reg), Location::Memory(addr, disp)) => {
552                let disp = disp as u32;
553                assert!(disp < 0x1000);
554                dynasm!(self ; strb W(reg), [X(addr), disp]);
555            }
556            (Size::S64, Location::SIMD(reg), Location::Memory(addr, disp)) => {
557                let disp = disp as u32;
558                assert!((disp & 0x7) == 0 && (disp < 0x8000));
559                dynasm!(self ; str D(reg), [X(addr), disp]);
560            }
561            (Size::S32, Location::SIMD(reg), Location::Memory(addr, disp)) => {
562                let disp = disp as u32;
563                assert!((disp & 0x3) == 0 && (disp < 0x4000));
564                dynasm!(self ; str S(reg), [X(addr), disp]);
565            }
566            (Size::S64, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
567                assert!(offs == 0);
568                let mult = mult as u32;
569                match mult {
570                    0 => dynasm!(self ; str X(reg), [X(addr)]),
571                    1 => dynasm!(self ; str X(reg), [X(addr), X(r2)]),
572                    _ => dynasm!(self ; str X(reg), [X(addr), X(r2), LSL mult]),
573                };
574            }
575            (Size::S32, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
576                assert!(offs == 0);
577                let mult = mult as u32;
578                match mult {
579                    0 => dynasm!(self ; str W(reg), [X(addr)]),
580                    1 => dynasm!(self ; str W(reg), [X(addr), X(r2)]),
581                    _ => dynasm!(self ; str W(reg), [X(addr), X(r2), LSL mult]),
582                };
583            }
584            _ => codegen_error!("singlepass can't emit STR {:?}, {:?}, {:?}", sz, reg, addr),
585        }
586        Ok(())
587    }
588    fn emit_ldr(&mut self, sz: Size, reg: Location, addr: Location) -> Result<(), CompileError> {
589        match (sz, reg, addr) {
590            (Size::S64, Location::GPR(reg), Location::Memory(addr, disp)) => {
591                assert!((disp & 0x7) == 0 && (disp < 0x8000));
592                let disp = disp as u32;
593                dynasm!(self ; ldr X(reg), [X(addr), disp]);
594            }
595            (Size::S32, Location::GPR(reg), Location::Memory(addr, disp)) => {
596                assert!((disp & 0x3) == 0 && (disp < 0x4000));
597                let disp = disp as u32;
598                dynasm!(self ; ldr W(reg), [X(addr), disp]);
599            }
600            (Size::S16, Location::GPR(reg), Location::Memory(addr, disp)) => {
601                assert!((disp & 0x1 == 0) && (disp < 0x2000));
602                let disp = disp as u32;
603                dynasm!(self ; ldrh W(reg), [X(addr), disp]);
604            }
605            (Size::S8, Location::GPR(reg), Location::Memory(addr, disp)) => {
606                assert!(disp < 0x1000);
607                let disp = disp as u32;
608                dynasm!(self ; ldrb W(reg), [X(addr), disp]);
609            }
610            (Size::S64, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
611                assert!(offs == 0);
612                let mult = mult as u32;
613                match mult {
614                    0 => dynasm!(self ; ldr X(reg), [X(addr)]),
615                    1 => dynasm!(self ; ldr X(reg), [X(addr), X(r2)]),
616                    _ => dynasm!(self ; ldr X(reg), [X(addr), X(r2), LSL mult]),
617                };
618            }
619            (Size::S32, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
620                assert!(offs == 0);
621                let mult = mult as u32;
622                match mult {
623                    0 => dynasm!(self ; ldr W(reg), [X(addr)]),
624                    1 => dynasm!(self ; ldr W(reg), [X(addr), X(r2)]),
625                    _ => dynasm!(self ; ldr W(reg), [X(addr), X(r2), LSL mult]),
626                };
627            }
628            (Size::S64, Location::SIMD(reg), Location::Memory(addr, disp)) => {
629                let disp = disp as u32;
630                assert!((disp & 0x7) == 0 && (disp < 0x8000));
631                dynasm!(self ; ldr D(reg), [X(addr), disp]);
632            }
633            (Size::S32, Location::SIMD(reg), Location::Memory(addr, disp)) => {
634                let disp = disp as u32;
635                assert!((disp & 0x3) == 0 && (disp < 0x4000));
636                dynasm!(self ; ldr S(reg), [X(addr), disp]);
637            }
638            (Size::S64, Location::SIMD(reg), Location::Memory2(addr, r2, mult, offs)) => {
639                assert!(offs == 0);
640                let mult = mult as u32;
641                match mult {
642                    0 => dynasm!(self ; ldr D(reg), [X(addr)]),
643                    1 => dynasm!(self ; ldr D(reg), [X(addr), X(r2)]),
644                    _ => dynasm!(self ; ldr D(reg), [X(addr), X(r2), LSL mult]),
645                };
646            }
647            (Size::S32, Location::SIMD(reg), Location::Memory2(addr, r2, mult, offs)) => {
648                assert!(offs == 0);
649                let mult = mult as u32;
650                match mult {
651                    0 => dynasm!(self ; ldr S(reg), [X(addr)]),
652                    1 => dynasm!(self ; ldr S(reg), [X(addr), X(r2)]),
653                    _ => dynasm!(self ; ldr S(reg), [X(addr), X(r2), LSL mult]),
654                };
655            }
656            _ => codegen_error!("singlepass can't emit LDR {:?}, {:?}, {:?}", sz, reg, addr),
657        }
658        Ok(())
659    }
660    fn emit_stur(
661        &mut self,
662        sz: Size,
663        reg: Location,
664        addr: GPR,
665        offset: i32,
666    ) -> Result<(), CompileError> {
667        assert!((-255..=255).contains(&offset));
668        match (sz, reg) {
669            (Size::S64, Location::GPR(reg)) => {
670                dynasm!(self ; stur X(reg), [X(addr), offset]);
671            }
672            (Size::S32, Location::GPR(reg)) => {
673                dynasm!(self ; stur W(reg), [X(addr), offset]);
674            }
675            (Size::S64, Location::SIMD(reg)) => {
676                dynasm!(self ; stur D(reg), [X(addr), offset]);
677            }
678            (Size::S32, Location::SIMD(reg)) => {
679                dynasm!(self ; stur S(reg), [X(addr), offset]);
680            }
681            _ => codegen_error!(
682                "singlepass can't emit STUR {:?}, {:?}, {:?}, {:?}",
683                sz,
684                reg,
685                addr,
686                offset
687            ),
688        }
689        Ok(())
690    }
691    fn emit_ldur(
692        &mut self,
693        sz: Size,
694        reg: Location,
695        addr: GPR,
696        offset: i32,
697    ) -> Result<(), CompileError> {
698        assert!((-255..=255).contains(&offset));
699        match (sz, reg) {
700            (Size::S64, Location::GPR(reg)) => {
701                dynasm!(self ; ldur X(reg), [X(addr), offset]);
702            }
703            (Size::S32, Location::GPR(reg)) => {
704                dynasm!(self ; ldur W(reg), [X(addr), offset]);
705            }
706            (Size::S64, Location::SIMD(reg)) => {
707                dynasm!(self ; ldur D(reg), [X(addr), offset]);
708            }
709            (Size::S32, Location::SIMD(reg)) => {
710                dynasm!(self ; ldur S(reg), [X(addr), offset]);
711            }
712            _ => codegen_error!(
713                "singlepass can't emit LDUR {:?}, {:?}, {:?}, {:?}",
714                sz,
715                reg,
716                addr,
717                offset
718            ),
719        }
720        Ok(())
721    }
722
723    fn emit_strdb(
724        &mut self,
725        sz: Size,
726        reg: Location,
727        addr: GPR,
728        offset: u32,
729    ) -> Result<(), CompileError> {
730        assert!(offset <= 255);
731        match (sz, reg) {
732            (Size::S64, Location::GPR(reg)) => {
733                dynasm!(self ; str X(reg), [X(addr), -(offset as i32)]!);
734            }
735            (Size::S64, Location::SIMD(reg)) => {
736                dynasm!(self ; str D(reg), [X(addr), -(offset as i32)]!);
737            }
738            _ => codegen_error!("singlepass can't emit STRDB"),
739        }
740        Ok(())
741    }
742    fn emit_stria(
743        &mut self,
744        sz: Size,
745        reg: Location,
746        addr: GPR,
747        offset: u32,
748    ) -> Result<(), CompileError> {
749        assert!(offset <= 255);
750        match (sz, reg) {
751            (Size::S64, Location::GPR(reg)) => {
752                dynasm!(self ; str X(reg), [X(addr)], offset as i32);
753            }
754            (Size::S64, Location::SIMD(reg)) => {
755                dynasm!(self ; str D(reg), [X(addr)], offset as i32);
756            }
757            _ => codegen_error!("singlepass can't emit STRIA"),
758        }
759        Ok(())
760    }
761    fn emit_ldria(
762        &mut self,
763        sz: Size,
764        reg: Location,
765        addr: GPR,
766        offset: u32,
767    ) -> Result<(), CompileError> {
768        assert!(offset <= 255);
769        match (sz, reg) {
770            (Size::S64, Location::GPR(reg)) => {
771                dynasm!(self ; ldr X(reg), [X(addr)], offset as _);
772            }
773            (Size::S64, Location::SIMD(reg)) => {
774                dynasm!(self ; ldr D(reg), [X(addr)], offset as _);
775            }
776            _ => codegen_error!("singlepass can't emit LDRIA"),
777        }
778        Ok(())
779    }
780
781    fn emit_stpdb(
782        &mut self,
783        sz: Size,
784        reg1: Location,
785        reg2: Location,
786        addr: GPR,
787        offset: u32,
788    ) -> Result<(), CompileError> {
789        assert!(offset <= 255);
790        match (sz, reg1, reg2) {
791            (Size::S64, Location::GPR(reg1), Location::GPR(reg2)) => {
792                dynasm!(self ; stp X(reg1), X(reg2), [X(addr), -(offset as i32)]!);
793            }
794            _ => codegen_error!("singlepass can't emit STPDB"),
795        }
796        Ok(())
797    }
798    fn emit_ldpia(
799        &mut self,
800        sz: Size,
801        reg1: Location,
802        reg2: Location,
803        addr: GPR,
804        offset: u32,
805    ) -> Result<(), CompileError> {
806        assert!(offset <= 255);
807        match (sz, reg1, reg2) {
808            (Size::S64, Location::GPR(reg1), Location::GPR(reg2)) => {
809                dynasm!(self ; ldp X(reg1), X(reg2), [X(addr)], offset as _);
810            }
811            _ => codegen_error!("singlepass can't emit LDPIA"),
812        }
813        Ok(())
814    }
815
816    fn emit_ldrb(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
817        match (reg, dst) {
818            (Location::GPR(reg), Location::Memory(addr, offset)) => {
819                let offset = offset as u32;
820                assert!(offset < 0x1000);
821                dynasm!(self ; ldrb W(reg), [X(addr), offset]);
822            }
823            (Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
824                assert!(offs == 0);
825                let mult = mult as u32;
826                match mult {
827                    0 => dynasm!(self ; ldrb W(reg), [X(addr)]),
828                    1 => dynasm!(self ; ldrb W(reg), [X(addr), X(r2)]),
829                    _ => dynasm!(self ; ldrb W(reg), [X(addr), X(r2), LSL mult]),
830                };
831            }
832            _ => codegen_error!("singlepass can't emit LDRB {:?}, {:?}", reg, dst),
833        }
834        Ok(())
835    }
836    fn emit_ldrh(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
837        match (reg, dst) {
838            (Location::GPR(reg), Location::Memory(addr, offset)) => {
839                let offset = offset as u32;
840                assert!((offset & 1 == 0) && (offset < 0x2000));
841                dynasm!(self ; ldrh W(reg), [X(addr), offset]);
842            }
843            (Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
844                assert!(offs == 0);
845                let mult = mult as u32;
846                match mult {
847                    0 => dynasm!(self ; ldrh W(reg), [X(addr)]),
848                    1 => dynasm!(self ; ldrh W(reg), [X(addr), X(r2)]),
849                    _ => dynasm!(self ; ldrh W(reg), [X(addr), X(r2), LSL mult]),
850                };
851            }
852            _ => codegen_error!("singlepass can't emit LDRH {:?}, {:?}", reg, dst),
853        }
854        Ok(())
855    }
856    fn emit_ldrsb(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
857        match (sz, reg, dst) {
858            (Size::S64, Location::GPR(reg), Location::Memory(addr, offset)) => {
859                let offset = offset as u32;
860                assert!(offset < 0x1000);
861                dynasm!(self ; ldrsb X(reg), [X(addr), offset]);
862            }
863            (Size::S32, Location::GPR(reg), Location::Memory(addr, offset)) => {
864                let offset = offset as u32;
865                assert!(offset < 0x1000);
866                dynasm!(self ; ldrsb W(reg), [X(addr), offset]);
867            }
868            (Size::S64, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
869                assert!(offs == 0);
870                let mult = mult as u32;
871                match mult {
872                    0 => dynasm!(self ; ldrsb X(reg), [X(addr)]),
873                    1 => dynasm!(self ; ldrsb X(reg), [X(addr), X(r2)]),
874                    _ => dynasm!(self ; ldrsb X(reg), [X(addr), X(r2), LSL mult]),
875                };
876            }
877            (Size::S32, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
878                assert!(offs == 0);
879                let mult = mult as u32;
880                match mult {
881                    0 => dynasm!(self ; ldrsb W(reg), [X(addr)]),
882                    1 => dynasm!(self ; ldrsb W(reg), [X(addr), X(r2)]),
883                    _ => dynasm!(self ; ldrsb W(reg), [X(addr), X(r2), LSL mult]),
884                };
885            }
886            _ => codegen_error!("singlepass can't emit LDRSB {:?}, {:?}, {:?}", sz, reg, dst),
887        }
888        Ok(())
889    }
890    fn emit_ldrsh(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
891        match (sz, reg, dst) {
892            (Size::S64, Location::GPR(reg), Location::Memory(addr, offset)) => {
893                let offset = offset as u32;
894                assert!((offset & 1 == 0) && (offset < 0x2000));
895                dynasm!(self ; ldrsh X(reg), [X(addr), offset]);
896            }
897            (Size::S32, Location::GPR(reg), Location::Memory(addr, offset)) => {
898                let offset = offset as u32;
899                assert!((offset & 1 == 0) && (offset < 0x2000));
900                dynasm!(self ; ldrsh W(reg), [X(addr), offset]);
901            }
902            (Size::S64, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
903                assert!(offs == 0);
904                let mult = mult as u32;
905                match mult {
906                    0 => dynasm!(self ; ldrsh X(reg), [X(addr)]),
907                    1 => dynasm!(self ; ldrsh X(reg), [X(addr), X(r2)]),
908                    _ => dynasm!(self ; ldrsh X(reg), [X(addr), X(r2), LSL mult]),
909                };
910            }
911            (Size::S32, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
912                assert!(offs == 0);
913                let mult = mult as u32;
914                match mult {
915                    0 => dynasm!(self ; ldrsh W(reg), [X(addr)]),
916                    1 => dynasm!(self ; ldrsh W(reg), [X(addr), X(r2)]),
917                    _ => dynasm!(self ; ldrsh W(reg), [X(addr), X(r2), LSL mult]),
918                };
919            }
920            _ => codegen_error!("singlepass can't emit LDRSH {:?}, {:?}, {:?}", sz, reg, dst),
921        }
922        Ok(())
923    }
924    fn emit_ldrsw(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
925        match (sz, reg, dst) {
926            (Size::S64, Location::GPR(reg), Location::Memory(addr, offset)) => {
927                let offset = offset as u32;
928                assert!((offset & 3 == 0) && (offset < 0x4000));
929                dynasm!(self ; ldrsw X(reg), [X(addr), offset]);
930            }
931            (Size::S64, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
932                assert!(offs == 0);
933                let mult = mult as u32;
934                match mult {
935                    0 => dynasm!(self ; ldrsw X(reg), [X(addr)]),
936                    1 => dynasm!(self ; ldrsw X(reg), [X(addr), X(r2)]),
937                    _ => dynasm!(self ; ldrsw X(reg), [X(addr), X(r2), LSL mult]),
938                };
939            }
940            _ => codegen_error!("singlepass can't emit LDRSW {:?}, {:?}, {:?}", sz, reg, dst),
941        }
942        Ok(())
943    }
944    fn emit_strb(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
945        match (reg, dst) {
946            (Location::GPR(reg), Location::Memory(addr, offset)) => {
947                let offset = offset as u32;
948                assert!(offset < 0x1000);
949                dynasm!(self ; strb W(reg), [X(addr), offset]);
950            }
951            (Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
952                assert!(offs == 0);
953                let mult = mult as u32;
954                match mult {
955                    0 => dynasm!(self ; strb W(reg), [X(addr)]),
956                    1 => dynasm!(self ; strb W(reg), [X(addr), X(r2)]),
957                    _ => dynasm!(self ; strb W(reg), [X(addr), X(r2), LSL mult]),
958                };
959            }
960            _ => codegen_error!("singlepass can't emit STRB {:?}, {:?}", reg, dst),
961        }
962        Ok(())
963    }
964    fn emit_strh(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
965        match (reg, dst) {
966            (Location::GPR(reg), Location::Memory(addr, offset)) => {
967                let offset = offset as u32;
968                assert!((offset & 1 == 0) && (offset < 0x2000));
969                dynasm!(self ; strh W(reg), [X(addr), offset]);
970            }
971            (Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
972                assert!(offs == 0);
973                let mult = mult as u32;
974                match mult {
975                    0 => dynasm!(self ; strh W(reg), [X(addr)]),
976                    1 => dynasm!(self ; strh W(reg), [X(addr), X(r2)]),
977                    _ => dynasm!(self ; strh W(reg), [X(addr), X(r2), LSL mult]),
978                };
979            }
980            _ => codegen_error!("singlepass can't emit STRH {:?}, {:?}", reg, dst),
981        }
982        Ok(())
983    }
984
985    fn emit_ldaxr(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
986        match (sz, reg, dst) {
987            (Size::S32, Location::GPR(reg), Location::GPR(dst)) => {
988                dynasm!(self ; ldaxr W(reg), [X(dst)]);
989            }
990            (Size::S64, Location::GPR(reg), Location::GPR(dst)) => {
991                dynasm!(self ; ldaxr X(reg), [X(dst)]);
992            }
993            _ => codegen_error!("singlepass can't emit LDAXR {:?}, {:?}", reg, dst),
994        }
995        Ok(())
996    }
997    fn emit_ldaxrb(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
998        match (reg, dst) {
999            (Location::GPR(reg), Location::GPR(dst)) => {
1000                dynasm!(self ; ldaxrb W(reg), [X(dst)]);
1001            }
1002            _ => codegen_error!("singlepass can't emit LDAXRB {:?}, {:?}", reg, dst),
1003        }
1004        Ok(())
1005    }
1006    fn emit_ldaxrh(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
1007        match (reg, dst) {
1008            (Location::GPR(reg), Location::GPR(dst)) => {
1009                dynasm!(self ; ldaxrh W(reg), [X(dst)]);
1010            }
1011            _ => codegen_error!("singlepass can't emit LDAXRH {:?}, {:?}", reg, dst),
1012        }
1013        Ok(())
1014    }
1015    fn emit_stlxr(
1016        &mut self,
1017        sz: Size,
1018        status: Location,
1019        reg: Location,
1020        dst: Location,
1021    ) -> Result<(), CompileError> {
1022        match (sz, status, reg, dst) {
1023            (Size::S32, Location::GPR(status), Location::GPR(reg), Location::GPR(dst)) => {
1024                dynasm!(self ; stlxr W(status), W(reg), [X(dst)]);
1025            }
1026            (Size::S64, Location::GPR(status), Location::GPR(reg), Location::GPR(dst)) => {
1027                dynasm!(self ; stlxr W(status), X(reg), [X(dst)]);
1028            }
1029            _ => codegen_error!("singlepass can't emit STLXR {:?}, {:?}", reg, dst),
1030        }
1031        Ok(())
1032    }
1033    fn emit_stlxrb(
1034        &mut self,
1035        _sz: Size,
1036        status: Location,
1037        reg: Location,
1038        dst: Location,
1039    ) -> Result<(), CompileError> {
1040        match (status, reg, dst) {
1041            (Location::GPR(status), Location::GPR(reg), Location::GPR(dst)) => {
1042                dynasm!(self ; stlxrb W(status), W(reg), [X(dst)]);
1043            }
1044            _ => codegen_error!("singlepass can't emit STLXRB {:?}, {:?}", reg, dst),
1045        }
1046        Ok(())
1047    }
1048    fn emit_stlxrh(
1049        &mut self,
1050        _sz: Size,
1051        status: Location,
1052        reg: Location,
1053        dst: Location,
1054    ) -> Result<(), CompileError> {
1055        match (status, reg, dst) {
1056            (Location::GPR(status), Location::GPR(reg), Location::GPR(dst)) => {
1057                dynasm!(self ; stlxrh W(status), W(reg), [X(dst)]);
1058            }
1059            _ => codegen_error!("singlepass can't emit STLXRH {:?}, {:?}", reg, dst),
1060        }
1061        Ok(())
1062    }
1063
1064    fn emit_mov(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1065        match (sz, src, dst) {
1066            (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1067                dynasm!(self ; mov X(dst), X(src));
1068            }
1069            (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1070                dynasm!(self ; mov W(dst), W(src));
1071            }
1072            (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
1073                dynasm!(self ; mov V(dst).D[0], V(src).D[0]);
1074            }
1075            (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
1076                dynasm!(self ; mov V(dst).S[0], V(src).S[0]);
1077            }
1078            (Size::S64, Location::GPR(src), Location::SIMD(dst)) => {
1079                dynasm!(self ; mov V(dst).D[0], X(src));
1080            }
1081            (Size::S32, Location::GPR(src), Location::SIMD(dst)) => {
1082                dynasm!(self ; mov V(dst).S[0], W(src));
1083            }
1084            (Size::S64, Location::SIMD(src), Location::GPR(dst)) => {
1085                dynasm!(self ; mov X(dst), V(src).D[0]);
1086            }
1087            (Size::S32, Location::SIMD(src), Location::GPR(dst)) => {
1088                dynasm!(self ; mov W(dst), V(src).S[0]);
1089            }
1090            (Size::S32, Location::Imm32(val), Location::GPR(dst)) => {
1091                if val < 0x1000 || encode_logical_immediate_32bit(val as _).is_some() {
1092                    dynasm!(self ; mov W(dst), val);
1093                } else {
1094                    dynasm!(self
1095                        ; movz W(dst), val
1096                        ; movk W(dst), val, lsl #16);
1097                }
1098            }
1099            (Size::S64, Location::Imm32(val), Location::GPR(dst)) => {
1100                if val < 0x1000 {
1101                    dynasm!(self ; mov W(dst), val);
1102                } else if encode_logical_immediate_64bit(val as _).is_some() {
1103                    dynasm!(self ; mov X(dst), val as _);
1104                } else {
1105                    dynasm!(self
1106                        ; movz X(dst), val
1107                        ; movk X(dst), val, lsl #16);
1108                }
1109            }
1110            (Size::S64, Location::Imm64(val), Location::GPR(dst)) => {
1111                if val < 0x1000 || encode_logical_immediate_64bit(val as _).is_some() {
1112                    dynasm!(self ; mov X(dst), val as _);
1113                } else {
1114                    dynasm!(self
1115                        ; movz X(dst), val as _
1116                        ; movk X(dst), val as _, lsl #16
1117                        ; movk X(dst), val as _, lsl #32
1118                        ; movk X(dst), val as _, lsl #48);
1119                }
1120            }
1121            _ => codegen_error!("singlepass can't emit MOV {:?}, {:?}, {:?}", sz, src, dst),
1122        }
1123        Ok(())
1124    }
1125
1126    fn emit_movn(&mut self, sz: Size, reg: Location, val: u32) -> Result<(), CompileError> {
1127        match (sz, reg) {
1128            (Size::S32, Location::GPR(reg)) => {
1129                dynasm!(self ; movn W(reg), val);
1130            }
1131            (Size::S64, Location::GPR(reg)) => {
1132                dynasm!(self ; movn X(reg), val);
1133            }
1134            _ => codegen_error!("singlepass can't emit MOVN"),
1135        }
1136        Ok(())
1137    }
1138    fn emit_movz(&mut self, reg: Location, val: u32) -> Result<(), CompileError> {
1139        match reg {
1140            Location::GPR(reg) => {
1141                dynasm!(self ; movz W(reg), val);
1142            }
1143            _ => codegen_error!("singlepass can't emit MOVZ"),
1144        }
1145        Ok(())
1146    }
1147    fn emit_movk(&mut self, reg: Location, val: u32, shift: u32) -> Result<(), CompileError> {
1148        match reg {
1149            Location::GPR(reg) => {
1150                dynasm!(self ; movk X(reg), val, LSL shift);
1151            }
1152            _ => codegen_error!("singlepass can't emit MOVK"),
1153        }
1154        Ok(())
1155    }
1156
1157    fn emit_mov_imm(&mut self, dst: Location, val: u64) -> Result<(), CompileError> {
1158        match dst {
1159            Location::GPR(dst) => {
1160                let offset = val.trailing_zeros() & 48;
1161                let masked = 0xffff & (val >> offset);
1162                if (masked << offset) == val {
1163                    dynasm!(self ; movz X(dst), masked as u32, LSL offset);
1164                } else if val >> 16 == 0xffff_ffff_ffff {
1165                    let val: u16 = !((val & 0xffff) as u16);
1166                    dynasm!(self ; movn X(dst), val as u32);
1167                } else if val >> 16 == 0xffff {
1168                    let val: u16 = !((val & 0xffff) as u16);
1169                    dynasm!(self ; movn W(dst), val as u32);
1170                } else {
1171                    dynasm!(self ; movz W(dst), (val&0xffff) as u32);
1172                    let val = val >> 16;
1173                    if val != 0 {
1174                        dynasm!(self ; movk X(dst), (val&0xffff) as u32, LSL 16);
1175                        let val = val >> 16;
1176                        if val != 0 {
1177                            dynasm!(self ; movk X(dst), (val&0xffff) as u32, LSL 32);
1178                            let val = val >> 16;
1179                            if val != 0 {
1180                                dynasm!(self ; movk X(dst), (val&0xffff) as u32, LSL 48);
1181                            }
1182                        }
1183                    }
1184                }
1185            }
1186            _ => codegen_error!("singlepass can't emit MOVW {:?}", dst),
1187        }
1188        Ok(())
1189    }
1190
1191    fn emit_add(
1192        &mut self,
1193        sz: Size,
1194        src1: Location,
1195        src2: Location,
1196        dst: Location,
1197    ) -> Result<(), CompileError> {
1198        match (sz, src1, src2, dst) {
1199            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1200                dynasm!(self ; add XSP(dst), XSP(src1), X(src2), UXTX);
1201            }
1202            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1203                dynasm!(self ; add WSP(dst), WSP(src1), W(src2), UXTX);
1204            }
1205            (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1206            | (Size::S64, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1207                dynasm!(self ; add XSP(dst), XSP(src1), imm as _);
1208            }
1209            (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
1210            | (Size::S64, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
1211                if imm >= 0x1000 {
1212                    unreachable!();
1213                }
1214                dynasm!(self ; add XSP(dst), XSP(src1), imm);
1215            }
1216            (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst))
1217            | (Size::S64, Location::Imm64(imm), Location::GPR(src1), Location::GPR(dst)) => {
1218                if imm >= 0x1000 {
1219                    unreachable!();
1220                }
1221                let imm = imm as u32;
1222                dynasm!(self ; add XSP(dst), XSP(src1), imm);
1223            }
1224            (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1225            | (Size::S32, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1226                dynasm!(self ; add WSP(dst), WSP(src1), imm as u32);
1227            }
1228            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
1229            | (Size::S32, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
1230                if imm >= 0x1000 {
1231                    unreachable!();
1232                }
1233                dynasm!(self ; add WSP(dst), WSP(src1), imm);
1234            }
1235            _ => codegen_error!(
1236                "singlepass can't emit ADD {:?} {:?} {:?} {:?}",
1237                sz,
1238                src1,
1239                src2,
1240                dst
1241            ),
1242        }
1243        Ok(())
1244    }
1245    fn emit_sub(
1246        &mut self,
1247        sz: Size,
1248        src1: Location,
1249        src2: Location,
1250        dst: Location,
1251    ) -> Result<(), CompileError> {
1252        match (sz, src1, src2, dst) {
1253            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1254                dynasm!(self; sub XSP(dst), XSP(src1), X(src2), UXTX 0);
1255            }
1256            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1257                dynasm!(self ; sub WSP(dst), WSP(src1), W(src2), UXTX 0);
1258            }
1259            (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1260                dynasm!(self ; sub XSP(dst), XSP(src1), imm as u32);
1261            }
1262            (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1263                dynasm!(self ; sub WSP(dst), WSP(src1), imm as u32);
1264            }
1265            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1266                if imm >= 0x1000 {
1267                    unreachable!();
1268                }
1269                dynasm!(self ; sub WSP(dst), WSP(src1), imm);
1270            }
1271            (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1272                if imm >= 0x1000 {
1273                    unreachable!();
1274                }
1275                dynasm!(self ; sub XSP(dst), XSP(src1), imm);
1276            }
1277            (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
1278                if imm >= 0x1000 {
1279                    unreachable!();
1280                }
1281                dynasm!(self ; sub XSP(dst), XSP(src1), imm as u32);
1282            }
1283            _ => codegen_error!(
1284                "singlepass can't emit SUB {:?} {:?} {:?} {:?}",
1285                sz,
1286                src1,
1287                src2,
1288                dst
1289            ),
1290        }
1291        Ok(())
1292    }
1293    fn emit_mul(
1294        &mut self,
1295        sz: Size,
1296        src1: Location,
1297        src2: Location,
1298        dst: Location,
1299    ) -> Result<(), CompileError> {
1300        match (sz, src1, src2, dst) {
1301            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1302                dynasm!(self ; mul X(dst), X(src1), X(src2));
1303            }
1304            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1305                dynasm!(self ; mul W(dst), W(src1), W(src2));
1306            }
1307            _ => codegen_error!(
1308                "singlepass can't emit MUL {:?} {:?} {:?} {:?}",
1309                sz,
1310                src1,
1311                src2,
1312                dst
1313            ),
1314        }
1315        Ok(())
1316    }
1317    fn emit_adds(
1318        &mut self,
1319        sz: Size,
1320        src1: Location,
1321        src2: Location,
1322        dst: Location,
1323    ) -> Result<(), CompileError> {
1324        match (sz, src1, src2, dst) {
1325            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1326                dynasm!(self ; adds X(dst), X(src1), X(src2));
1327            }
1328            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1329                dynasm!(self ; adds W(dst), W(src1), W(src2));
1330            }
1331            (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1332            | (Size::S64, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1333                dynasm!(self ; adds X(dst), XSP(src1), imm as u32);
1334            }
1335            (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
1336            | (Size::S64, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
1337                if imm >= 0x1000 {
1338                    codegen_error!("singlepass ADD.S with imm too large {}", imm);
1339                }
1340                dynasm!(self ; adds X(dst), XSP(src1), imm);
1341            }
1342            (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1343            | (Size::S32, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1344                dynasm!(self ; adds W(dst), WSP(src1), imm as u32);
1345            }
1346            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
1347            | (Size::S32, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
1348                if imm >= 0x1000 {
1349                    codegen_error!("singlepass ADD.S with imm too large {}", imm);
1350                }
1351                dynasm!(self ; adds W(dst), WSP(src1), imm);
1352            }
1353            _ => codegen_error!(
1354                "singlepass can't emit ADD.S {:?} {:?} {:?} {:?}",
1355                sz,
1356                src1,
1357                src2,
1358                dst
1359            ),
1360        }
1361        Ok(())
1362    }
1363    fn emit_subs(
1364        &mut self,
1365        sz: Size,
1366        src1: Location,
1367        src2: Location,
1368        dst: Location,
1369    ) -> Result<(), CompileError> {
1370        match (sz, src1, src2, dst) {
1371            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1372                dynasm!(self ; subs X(dst), X(src1), X(src2));
1373            }
1374            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1375                dynasm!(self ; subs W(dst), W(src1), W(src2));
1376            }
1377            (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1378                dynasm!(self ; subs X(dst), XSP(src1), imm as u32);
1379            }
1380            (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1381                dynasm!(self ; subs W(dst), WSP(src1), imm as u32);
1382            }
1383            _ => codegen_error!(
1384                "singlepass can't emit SUB.S {:?} {:?} {:?} {:?}",
1385                sz,
1386                src1,
1387                src2,
1388                dst
1389            ),
1390        }
1391        Ok(())
1392    }
1393    fn emit_add_lsl(
1394        &mut self,
1395        sz: Size,
1396        src1: Location,
1397        src2: Location,
1398        lsl: u32,
1399        dst: Location,
1400    ) -> Result<(), CompileError> {
1401        match (sz, src1, src2, dst) {
1402            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1403                dynasm!(self ; add X(dst), X(src1), X(src2), LSL lsl);
1404            }
1405            _ => codegen_error!(
1406                "singlepass can't emit LSL {:?} {:?} {:?} {:?} LSL {:?}",
1407                sz,
1408                src1,
1409                src2,
1410                dst,
1411                lsl
1412            ),
1413        }
1414        Ok(())
1415    }
1416
1417    /// Emit a CMP instruction that compares `left` against `right`.
1418    ///
1419    /// Note: callers sometimes pass operands in the opposite order compared
1420    /// to other binary operators. This function performs the comparison as
1421    /// provided (i.e. it emits `cmp left, right` semantics).
1422    fn emit_cmp(&mut self, sz: Size, left: Location, right: Location) -> Result<(), CompileError> {
1423        match (sz, left, right) {
1424            (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1425                dynasm!(self ; cmp X(dst), X(src));
1426            }
1427            (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1428                dynasm!(self ; cmp W(dst), W(src));
1429            }
1430            (Size::S64, Location::Imm8(imm), Location::GPR(dst)) => {
1431                dynasm!(self ; cmp XSP(dst), imm as u32);
1432            }
1433            (Size::S64, Location::Imm32(imm), Location::GPR(dst)) => {
1434                if imm >= 0x1000 {
1435                    codegen_error!("singlepass CMP with imm too large {}", imm);
1436                }
1437                dynasm!(self ; cmp XSP(dst), imm as u32);
1438            }
1439            (Size::S64, Location::Imm64(imm), Location::GPR(dst)) => {
1440                if imm >= 0x1000 {
1441                    codegen_error!("singlepass CMP with imm too large {}", imm);
1442                }
1443                dynasm!(self ; cmp XSP(dst), imm as u32);
1444            }
1445            (Size::S32, Location::Imm8(imm), Location::GPR(dst)) => {
1446                dynasm!(self ; cmp WSP(dst), imm as u32);
1447            }
1448            (Size::S32, Location::Imm32(imm), Location::GPR(dst)) => {
1449                if imm >= 0x1000 {
1450                    codegen_error!("singlepass CMP with imm too large {}", imm);
1451                }
1452                dynasm!(self ; cmp WSP(dst), imm as u32);
1453            }
1454            _ => codegen_error!("singlepass can't emit CMP {:?} {:?} {:?}", sz, left, right),
1455        }
1456        Ok(())
1457    }
1458
1459    fn emit_tst(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1460        match (sz, src, dst) {
1461            (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1462                dynasm!(self ; tst X(dst), X(src));
1463            }
1464            (Size::S64, Location::Imm32(imm), Location::GPR(dst)) => {
1465                if encode_logical_immediate_64bit(imm as u64).is_none() {
1466                    codegen_error!("singlepass TST with incompatible imm {}", imm);
1467                }
1468                dynasm!(self ; tst X(dst), imm as u64);
1469            }
1470            (Size::S64, Location::Imm64(imm), Location::GPR(dst)) => {
1471                if encode_logical_immediate_64bit(imm as u64).is_none() {
1472                    codegen_error!("singlepass TST with incompatible imm {}", imm);
1473                }
1474                dynasm!(self ; tst X(dst), imm as u64);
1475            }
1476            (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1477                dynasm!(self ; tst W(dst), W(src));
1478            }
1479            (Size::S32, Location::Imm32(imm), Location::GPR(dst)) => {
1480                if encode_logical_immediate_64bit(imm as u64).is_none() {
1481                    codegen_error!("singlepass TST with incompatible imm {}", imm);
1482                }
1483                dynasm!(self ; tst W(dst), imm);
1484            }
1485            _ => codegen_error!("singlepass can't emit TST"),
1486        }
1487        Ok(())
1488    }
1489
1490    fn emit_lsl(
1491        &mut self,
1492        sz: Size,
1493        src1: Location,
1494        src2: Location,
1495        dst: Location,
1496    ) -> Result<(), CompileError> {
1497        match (sz, src1, src2, dst) {
1498            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1499                dynasm!(self ; lsl X(dst), X(src1), X(src2));
1500            }
1501            (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1502                if imm > 63 {
1503                    codegen_error!("singlepass LSL with incompatible imm {}", imm);
1504                }
1505                let imm = imm as u32;
1506
1507                dynasm!(self ; lsl X(dst), X(src1), imm);
1508            }
1509            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1510                dynasm!(self ; lsl W(dst), W(src1), W(src2));
1511            }
1512            (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1513                if imm > 63 {
1514                    codegen_error!("singlepass LSL with incompatible imm {}", imm);
1515                }
1516                dynasm!(self ; lsl X(dst), X(src1), imm as u32);
1517            }
1518            (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
1519                if imm > 63 {
1520                    codegen_error!("singlepass LSL with incompatible imm {}", imm);
1521                }
1522                dynasm!(self ; lsl X(dst), X(src1), imm as u32);
1523            }
1524            (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1525                if imm > 31 {
1526                    codegen_error!("singlepass LSL with incompatible imm {}", imm);
1527                }
1528                dynasm!(self ; lsl W(dst), W(src1), imm as u32);
1529            }
1530            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1531                if imm > 31 {
1532                    codegen_error!("singlepass LSL with incompatible imm {}", imm);
1533                }
1534                dynasm!(self ; lsl W(dst), W(src1), imm as u32);
1535            }
1536            _ => codegen_error!(
1537                "singlepass can't emit LSL {:?} {:?} {:?} {:?}",
1538                sz,
1539                src1,
1540                src2,
1541                dst
1542            ),
1543        }
1544        Ok(())
1545    }
1546    fn emit_asr(
1547        &mut self,
1548        sz: Size,
1549        src1: Location,
1550        src2: Location,
1551        dst: Location,
1552    ) -> Result<(), CompileError> {
1553        match (sz, src1, src2, dst) {
1554            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1555                dynasm!(self ; asr X(dst), X(src1), X(src2));
1556            }
1557            (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1558                let imm = imm as u32;
1559
1560                if imm == 0 || imm > 63 {
1561                    codegen_error!("singlepass ASR with incompatible imm {}", imm);
1562                }
1563                dynasm!(self ; asr X(dst), X(src1), imm);
1564            }
1565            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1566                dynasm!(self ; asr W(dst), W(src1), W(src2));
1567            }
1568            (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1569                if imm == 0 || imm > 63 {
1570                    codegen_error!("singlepass ASR with incompatible imm {}", imm);
1571                }
1572                dynasm!(self ; asr X(dst), X(src1), imm as u32);
1573            }
1574            (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
1575                if imm == 0 || imm > 63 {
1576                    codegen_error!("singlepass ASR with incompatible imm {}", imm);
1577                }
1578                dynasm!(self ; asr X(dst), X(src1), imm as u32);
1579            }
1580            (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1581                if imm == 0 || imm > 31 {
1582                    codegen_error!("singlepass ASR with incompatible imm {}", imm);
1583                }
1584                dynasm!(self ; asr W(dst), W(src1), imm as u32);
1585            }
1586            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1587                if imm == 0 || imm > 31 {
1588                    codegen_error!("singlepass ASR with incompatible imm {}", imm);
1589                }
1590                dynasm!(self ; asr W(dst), W(src1), imm as u32);
1591            }
1592            _ => codegen_error!(
1593                "singlepass can't emit ASR {:?} {:?} {:?} {:?}",
1594                sz,
1595                src1,
1596                src2,
1597                dst
1598            ),
1599        }
1600        Ok(())
1601    }
1602    fn emit_lsr(
1603        &mut self,
1604        sz: Size,
1605        src1: Location,
1606        src2: Location,
1607        dst: Location,
1608    ) -> Result<(), CompileError> {
1609        match (sz, src1, src2, dst) {
1610            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1611                dynasm!(self ; lsr X(dst), X(src1), X(src2));
1612            }
1613            (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1614                let imm = imm as u32;
1615
1616                if imm == 0 || imm > 63 {
1617                    codegen_error!("singlepass LSR with incompatible imm {}", imm);
1618                }
1619                dynasm!(self ; lsr X(dst), X(src1), imm);
1620            }
1621            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1622                dynasm!(self ; lsr W(dst), W(src1), W(src2));
1623            }
1624            (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1625                if imm == 0 || imm > 63 {
1626                    codegen_error!("singlepass LSR with incompatible imm {}", imm);
1627                }
1628                dynasm!(self ; lsr X(dst), X(src1), imm as u32);
1629            }
1630            (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
1631                if imm == 0 || imm > 63 {
1632                    codegen_error!("singlepass LSR with incompatible imm {}", imm);
1633                }
1634                dynasm!(self ; lsr X(dst), X(src1), imm as u32);
1635            }
1636            (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1637                if imm == 0 || imm > 31 {
1638                    codegen_error!("singlepass LSR with incompatible imm {}", imm);
1639                }
1640                dynasm!(self ; lsr W(dst), W(src1), imm as u32);
1641            }
1642            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1643                if imm == 0 || imm > 31 {
1644                    codegen_error!("singlepass LSR with incompatible imm {}", imm);
1645                }
1646                dynasm!(self ; lsr W(dst), W(src1), imm as u32);
1647            }
1648            _ => codegen_error!(
1649                "singlepass can't emit LSR {:?} {:?} {:?} {:?}",
1650                sz,
1651                src1,
1652                src2,
1653                dst
1654            ),
1655        }
1656        Ok(())
1657    }
1658    fn emit_ror(
1659        &mut self,
1660        sz: Size,
1661        src1: Location,
1662        src2: Location,
1663        dst: Location,
1664    ) -> Result<(), CompileError> {
1665        match (sz, src1, src2, dst) {
1666            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1667                dynasm!(self ; ror X(dst), X(src1), X(src2));
1668            }
1669            (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1670                let imm = imm as u32;
1671
1672                if imm == 0 || imm > 63 {
1673                    codegen_error!("singlepass ROR with incompatible imm {}", imm);
1674                }
1675                dynasm!(self ; ror X(dst), X(src1), imm);
1676            }
1677            (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
1678                let imm = imm as u32;
1679
1680                if imm == 0 || imm > 63 {
1681                    codegen_error!("singlepass ROR with incompatible imm {}", imm);
1682                }
1683                dynasm!(self ; ror X(dst), X(src1), imm);
1684            }
1685            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1686                if imm == 0 || imm > 31 {
1687                    codegen_error!("singlepass ROR with incompatible imm {}", imm);
1688                }
1689                dynasm!(self ; ror W(dst), W(src1), imm as u32);
1690            }
1691            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1692                dynasm!(self ; ror W(dst), W(src1), W(src2));
1693            }
1694            (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1695                if imm == 0 || imm > 63 {
1696                    codegen_error!("singlepass ROR with incompatible imm {}", imm);
1697                }
1698                dynasm!(self ; ror X(dst), X(src1), imm as u32);
1699            }
1700            (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1701                if imm == 0 || imm > 31 {
1702                    codegen_error!("singlepass ROR with incompatible imm {}", imm);
1703                }
1704                dynasm!(self ; ror W(dst), W(src1), imm as u32);
1705            }
1706            _ => codegen_error!(
1707                "singlepass can't emit ROR {:?} {:?} {:?} {:?}",
1708                sz,
1709                src1,
1710                src2,
1711                dst
1712            ),
1713        }
1714        Ok(())
1715    }
1716
1717    fn emit_or(
1718        &mut self,
1719        sz: Size,
1720        src1: Location,
1721        src2: Location,
1722        dst: Location,
1723    ) -> Result<(), CompileError> {
1724        match (sz, src1, src2, dst) {
1725            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1726                dynasm!(self ; orr X(dst), X(src1), X(src2));
1727            }
1728            (Size::S64, Location::GPR(src1), Location::Imm64(src2), Location::GPR(dst)) => {
1729                let src2 = src2 as u64;
1730
1731                if encode_logical_immediate_64bit(src2 as u64).is_none() {
1732                    codegen_error!("singlepass OR with incompatible imm {}", src2);
1733                }
1734                dynasm!(self ; orr XSP(dst), X(src1), src2);
1735            }
1736            (Size::S32, Location::GPR(src1), Location::Imm32(src2), Location::GPR(dst)) => {
1737                let src2 = src2 as u32;
1738
1739                if encode_logical_immediate_32bit(src2).is_none() {
1740                    codegen_error!("singlepass OR with incompatible imm {}", src2);
1741                }
1742                dynasm!(self ; orr WSP(dst), W(src1), src2);
1743            }
1744            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1745                dynasm!(self ; orr W(dst), W(src1), W(src2));
1746            }
1747            _ => codegen_error!(
1748                "singlepass can't emit OR {:?} {:?} {:?} {:?}",
1749                sz,
1750                src1,
1751                src2,
1752                dst
1753            ),
1754        }
1755        Ok(())
1756    }
1757    fn emit_and(
1758        &mut self,
1759        sz: Size,
1760        src1: Location,
1761        src2: Location,
1762        dst: Location,
1763    ) -> Result<(), CompileError> {
1764        match (sz, src1, src2, dst) {
1765            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1766                dynasm!(self ; and X(dst), X(src1), X(src2));
1767            }
1768            (Size::S64, Location::GPR(src1), Location::Imm64(src2), Location::GPR(dst)) => {
1769                let src2 = src2 as u64;
1770
1771                if encode_logical_immediate_64bit(src2 as u64).is_none() {
1772                    codegen_error!("singlepass AND with incompatible imm {}", src2);
1773                }
1774                dynasm!(self ; and XSP(dst), X(src1), src2);
1775            }
1776            (Size::S32, Location::GPR(src1), Location::Imm32(src2), Location::GPR(dst)) => {
1777                let src2 = src2 as u32;
1778
1779                if encode_logical_immediate_32bit(src2).is_none() {
1780                    codegen_error!("singlepass AND with incompatible imm {}", src2);
1781                }
1782                dynasm!(self ; and WSP(dst), W(src1), src2);
1783            }
1784            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1785                dynasm!(self ; and W(dst), W(src1), W(src2));
1786            }
1787            _ => codegen_error!(
1788                "singlepass can't emit AND {:?} {:?} {:?} {:?}",
1789                sz,
1790                src1,
1791                src2,
1792                dst
1793            ),
1794        }
1795        Ok(())
1796    }
1797    fn emit_eor(
1798        &mut self,
1799        sz: Size,
1800        src1: Location,
1801        src2: Location,
1802        dst: Location,
1803    ) -> Result<(), CompileError> {
1804        match (sz, src1, src2, dst) {
1805            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1806                dynasm!(self ; eor X(dst), X(src1), X(src2));
1807            }
1808            (Size::S64, Location::GPR(src1), Location::Imm64(src2), Location::GPR(dst)) => {
1809                let src2 = src2 as u64;
1810
1811                if encode_logical_immediate_64bit(src2 as u64).is_none() {
1812                    codegen_error!("singlepass EOR with incompatible imm {}", src2);
1813                }
1814                dynasm!(self ; eor XSP(dst), X(src1), src2);
1815            }
1816            (Size::S32, Location::GPR(src1), Location::Imm32(src2), Location::GPR(dst)) => {
1817                let src2 = src2 as u32;
1818
1819                if encode_logical_immediate_32bit(src2).is_none() {
1820                    codegen_error!("singlepass EOR with incompatible imm {}", src2);
1821                }
1822                dynasm!(self ; eor WSP(dst), W(src1), src2);
1823            }
1824            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1825                dynasm!(self ; eor W(dst), W(src1), W(src2));
1826            }
1827            _ => codegen_error!(
1828                "singlepass can't emit EOR {:?} {:?} {:?} {:?}",
1829                sz,
1830                src1,
1831                src2,
1832                dst
1833            ),
1834        }
1835        Ok(())
1836    }
1837
1838    fn emit_bfc(
1839        &mut self,
1840        sz: Size,
1841        lsb: u32,
1842        width: u32,
1843        dst: Location,
1844    ) -> Result<(), CompileError> {
1845        match (sz, dst) {
1846            (Size::S32, Location::GPR(dst)) => {
1847                dynasm!(self ; bfc W(dst), lsb, width);
1848            }
1849            (Size::S64, Location::GPR(dst)) => {
1850                dynasm!(self ; bfc X(dst), lsb, width);
1851            }
1852            _ => codegen_error!("singlepass can't emit BFC"),
1853        }
1854        Ok(())
1855    }
1856    fn emit_bfi(
1857        &mut self,
1858        sz: Size,
1859        src: Location,
1860        lsb: u32,
1861        width: u32,
1862        dst: Location,
1863    ) -> Result<(), CompileError> {
1864        match (sz, src, dst) {
1865            (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1866                dynasm!(self ; bfi W(dst), W(src), lsb, width);
1867            }
1868            (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1869                dynasm!(self ; bfi X(dst), X(src), lsb, width);
1870            }
1871            _ => codegen_error!("singlepass can't emit BFI"),
1872        }
1873        Ok(())
1874    }
1875
1876    fn emit_udiv(
1877        &mut self,
1878        sz: Size,
1879        src1: Location,
1880        src2: Location,
1881        dst: Location,
1882    ) -> Result<(), CompileError> {
1883        match (sz, src1, src2, dst) {
1884            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1885                dynasm!(self ; udiv W(dst), W(src1), W(src2));
1886            }
1887            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1888                dynasm!(self ; udiv X(dst), X(src1), X(src2));
1889            }
1890            _ => codegen_error!(
1891                "singlepass can't emit UDIV {:?} {:?} {:?} {:?}",
1892                sz,
1893                src1,
1894                src2,
1895                dst
1896            ),
1897        }
1898        Ok(())
1899    }
1900    fn emit_sdiv(
1901        &mut self,
1902        sz: Size,
1903        src1: Location,
1904        src2: Location,
1905        dst: Location,
1906    ) -> Result<(), CompileError> {
1907        match (sz, src1, src2, dst) {
1908            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1909                dynasm!(self ; sdiv W(dst), W(src1), W(src2));
1910            }
1911            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1912                dynasm!(self ; sdiv X(dst), X(src1), X(src2));
1913            }
1914            _ => codegen_error!(
1915                "singlepass can't emit SDIV {:?} {:?} {:?} {:?}",
1916                sz,
1917                src1,
1918                src2,
1919                dst
1920            ),
1921        }
1922        Ok(())
1923    }
1924
1925    /// msub : c - a*b -> dst
1926    fn emit_msub(
1927        &mut self,
1928        sz: Size,
1929        a: Location,
1930        b: Location,
1931        c: Location,
1932        dst: Location,
1933    ) -> Result<(), CompileError> {
1934        match (sz, a, b, c, dst) {
1935            (
1936                Size::S32,
1937                Location::GPR(a),
1938                Location::GPR(b),
1939                Location::GPR(c),
1940                Location::GPR(dst),
1941            ) => {
1942                dynasm!(self ; msub W(dst), W(a), W(b), W(c));
1943            }
1944            (
1945                Size::S64,
1946                Location::GPR(a),
1947                Location::GPR(b),
1948                Location::GPR(c),
1949                Location::GPR(dst),
1950            ) => {
1951                dynasm!(self ; msub X(dst), X(a), X(b), X(c));
1952            }
1953            _ => codegen_error!(
1954                "singlepass can't emit msub {:?} {:?} {:?} {:?} {:?}",
1955                sz,
1956                a,
1957                b,
1958                c,
1959                dst
1960            ),
1961        }
1962        Ok(())
1963    }
1964
1965    fn emit_sxtb(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1966        match (sz, src, dst) {
1967            (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1968                dynasm!(self ; sxtb W(dst), W(src));
1969            }
1970            (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1971                dynasm!(self ; sxtb X(dst), W(src));
1972            }
1973            _ => codegen_error!("singlepass can't emit SXTB {:?} {:?} {:?}", sz, src, dst),
1974        }
1975        Ok(())
1976    }
1977    fn emit_sxth(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1978        match (sz, src, dst) {
1979            (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1980                dynasm!(self ; sxth W(dst), W(src));
1981            }
1982            (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1983                dynasm!(self ; sxth X(dst), W(src));
1984            }
1985            _ => codegen_error!("singlepass can't emit SXTH {:?} {:?} {:?}", sz, src, dst),
1986        }
1987        Ok(())
1988    }
1989    fn emit_sxtw(&mut self, _sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1990        match (src, dst) {
1991            (Location::GPR(src), Location::GPR(dst)) => {
1992                dynasm!(self ; sxtw X(dst), W(src));
1993            }
1994            _ => codegen_error!("singlepass can't emit SXTW {:?} {:?}", src, dst),
1995        }
1996        Ok(())
1997    }
1998    fn emit_uxtb(&mut self, _sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1999        match (src, dst) {
2000            (Location::GPR(src), Location::GPR(dst)) => {
2001                dynasm!(self ; uxtb W(dst), W(src));
2002            }
2003            _ => codegen_error!("singlepass can't emit UXTB {:?} {:?}", src, dst),
2004        }
2005        Ok(())
2006    }
2007    fn emit_uxth(&mut self, _sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2008        match (src, dst) {
2009            (Location::GPR(src), Location::GPR(dst)) => {
2010                dynasm!(self ; uxth W(dst), W(src));
2011            }
2012            _ => codegen_error!("singlepass can't emit UXTH {:?} {:?}", src, dst),
2013        }
2014        Ok(())
2015    }
2016
2017    fn emit_cset(&mut self, sz: Size, dst: Location, cond: Condition) -> Result<(), CompileError> {
2018        match (sz, dst) {
2019            (Size::S32, Location::GPR(reg)) => match cond {
2020                Condition::Eq => dynasm!(self ; cset W(reg), eq),
2021                Condition::Ne => dynasm!(self ; cset W(reg), ne),
2022                Condition::Cs => dynasm!(self ; cset W(reg), cs),
2023                Condition::Cc => dynasm!(self ; cset W(reg), cc),
2024                Condition::Mi => dynasm!(self ; cset W(reg), mi),
2025                Condition::Pl => dynasm!(self ; cset W(reg), pl),
2026                Condition::Vs => dynasm!(self ; cset W(reg), vs),
2027                Condition::Vc => dynasm!(self ; cset W(reg), vc),
2028                Condition::Hi => dynasm!(self ; cset W(reg), hi),
2029                Condition::Ls => dynasm!(self ; cset W(reg), ls),
2030                Condition::Ge => dynasm!(self ; cset W(reg), ge),
2031                Condition::Lt => dynasm!(self ; cset W(reg), lt),
2032                Condition::Gt => dynasm!(self ; cset W(reg), gt),
2033                Condition::Le => dynasm!(self ; cset W(reg), le),
2034                Condition::Al => dynasm!(self ; cset W(reg), al),
2035            },
2036            (Size::S64, Location::GPR(reg)) => match cond {
2037                Condition::Eq => dynasm!(self ; cset X(reg), eq),
2038                Condition::Ne => dynasm!(self ; cset X(reg), ne),
2039                Condition::Cs => dynasm!(self ; cset X(reg), cs),
2040                Condition::Cc => dynasm!(self ; cset X(reg), cc),
2041                Condition::Mi => dynasm!(self ; cset X(reg), mi),
2042                Condition::Pl => dynasm!(self ; cset X(reg), pl),
2043                Condition::Vs => dynasm!(self ; cset X(reg), vs),
2044                Condition::Vc => dynasm!(self ; cset X(reg), vc),
2045                Condition::Hi => dynasm!(self ; cset X(reg), hi),
2046                Condition::Ls => dynasm!(self ; cset X(reg), ls),
2047                Condition::Ge => dynasm!(self ; cset X(reg), ge),
2048                Condition::Lt => dynasm!(self ; cset X(reg), lt),
2049                Condition::Gt => dynasm!(self ; cset X(reg), gt),
2050                Condition::Le => dynasm!(self ; cset X(reg), le),
2051                Condition::Al => dynasm!(self ; cset X(reg), al),
2052            },
2053            _ => codegen_error!("singlepass can't emit CSET {:?} {:?} {:?}", sz, dst, cond),
2054        }
2055        Ok(())
2056    }
2057    fn emit_csetm(&mut self, sz: Size, dst: Location, cond: Condition) -> Result<(), CompileError> {
2058        match (sz, dst) {
2059            (Size::S32, Location::GPR(reg)) => match cond {
2060                Condition::Eq => dynasm!(self ; csetm W(reg), eq),
2061                Condition::Ne => dynasm!(self ; csetm W(reg), ne),
2062                Condition::Cs => dynasm!(self ; csetm W(reg), cs),
2063                Condition::Cc => dynasm!(self ; csetm W(reg), cc),
2064                Condition::Mi => dynasm!(self ; csetm W(reg), mi),
2065                Condition::Pl => dynasm!(self ; csetm W(reg), pl),
2066                Condition::Vs => dynasm!(self ; csetm W(reg), vs),
2067                Condition::Vc => dynasm!(self ; csetm W(reg), vc),
2068                Condition::Hi => dynasm!(self ; csetm W(reg), hi),
2069                Condition::Ls => dynasm!(self ; csetm W(reg), ls),
2070                Condition::Ge => dynasm!(self ; csetm W(reg), ge),
2071                Condition::Lt => dynasm!(self ; csetm W(reg), lt),
2072                Condition::Gt => dynasm!(self ; csetm W(reg), gt),
2073                Condition::Le => dynasm!(self ; csetm W(reg), le),
2074                Condition::Al => dynasm!(self ; csetm W(reg), al),
2075            },
2076            (Size::S64, Location::GPR(reg)) => match cond {
2077                Condition::Eq => dynasm!(self ; csetm X(reg), eq),
2078                Condition::Ne => dynasm!(self ; csetm X(reg), ne),
2079                Condition::Cs => dynasm!(self ; csetm X(reg), cs),
2080                Condition::Cc => dynasm!(self ; csetm X(reg), cc),
2081                Condition::Mi => dynasm!(self ; csetm X(reg), mi),
2082                Condition::Pl => dynasm!(self ; csetm X(reg), pl),
2083                Condition::Vs => dynasm!(self ; csetm X(reg), vs),
2084                Condition::Vc => dynasm!(self ; csetm X(reg), vc),
2085                Condition::Hi => dynasm!(self ; csetm X(reg), hi),
2086                Condition::Ls => dynasm!(self ; csetm X(reg), ls),
2087                Condition::Ge => dynasm!(self ; csetm X(reg), ge),
2088                Condition::Lt => dynasm!(self ; csetm X(reg), lt),
2089                Condition::Gt => dynasm!(self ; csetm X(reg), gt),
2090                Condition::Le => dynasm!(self ; csetm X(reg), le),
2091                Condition::Al => dynasm!(self ; csetm X(reg), al),
2092            },
2093            _ => codegen_error!("singlepass can't emit CSETM {:?} {:?} {:?}", sz, dst, cond),
2094        }
2095        Ok(())
2096    }
2097    fn emit_cinc(
2098        &mut self,
2099        sz: Size,
2100        src: Location,
2101        dst: Location,
2102        cond: Condition,
2103    ) -> Result<(), CompileError> {
2104        match (sz, src, dst) {
2105            (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
2106                match cond {
2107                    Condition::Eq => dynasm!(self ; cinc W(dst), W(src), eq),
2108                    Condition::Ne => dynasm!(self ; cinc W(dst), W(src), ne),
2109                    Condition::Cs => dynasm!(self ; cinc W(dst), W(src), cs),
2110                    Condition::Cc => dynasm!(self ; cinc W(dst), W(src), cc),
2111                    Condition::Mi => dynasm!(self ; cinc W(dst), W(src), mi),
2112                    Condition::Pl => dynasm!(self ; cinc W(dst), W(src), pl),
2113                    Condition::Vs => dynasm!(self ; cinc W(dst), W(src), vs),
2114                    Condition::Vc => dynasm!(self ; cinc W(dst), W(src), vc),
2115                    Condition::Hi => dynasm!(self ; cinc W(dst), W(src), hi),
2116                    Condition::Ls => dynasm!(self ; cinc W(dst), W(src), ls),
2117                    Condition::Ge => dynasm!(self ; cinc W(dst), W(src), ge),
2118                    Condition::Lt => dynasm!(self ; cinc W(dst), W(src), lt),
2119                    Condition::Gt => dynasm!(self ; cinc W(dst), W(src), gt),
2120                    Condition::Le => dynasm!(self ; cinc W(dst), W(src), le),
2121                    Condition::Al => dynasm!(self ; cinc W(dst), W(src), al),
2122                };
2123            }
2124            (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
2125                match cond {
2126                    Condition::Eq => dynasm!(self ; cinc X(src), X(dst), eq),
2127                    Condition::Ne => dynasm!(self ; cinc X(src), X(dst), ne),
2128                    Condition::Cs => dynasm!(self ; cinc X(src), X(dst), cs),
2129                    Condition::Cc => dynasm!(self ; cinc X(src), X(dst), cc),
2130                    Condition::Mi => dynasm!(self ; cinc X(src), X(dst), mi),
2131                    Condition::Pl => dynasm!(self ; cinc X(src), X(dst), pl),
2132                    Condition::Vs => dynasm!(self ; cinc X(src), X(dst), vs),
2133                    Condition::Vc => dynasm!(self ; cinc X(src), X(dst), vc),
2134                    Condition::Hi => dynasm!(self ; cinc X(src), X(dst), hi),
2135                    Condition::Ls => dynasm!(self ; cinc X(src), X(dst), ls),
2136                    Condition::Ge => dynasm!(self ; cinc X(src), X(dst), ge),
2137                    Condition::Lt => dynasm!(self ; cinc X(src), X(dst), lt),
2138                    Condition::Gt => dynasm!(self ; cinc X(src), X(dst), gt),
2139                    Condition::Le => dynasm!(self ; cinc X(src), X(dst), le),
2140                    Condition::Al => dynasm!(self ; cinc X(src), X(dst), al),
2141                };
2142            }
2143            _ => codegen_error!("singlepass can't emit CINC"),
2144        }
2145        Ok(())
2146    }
2147
2148    fn emit_clz(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2149        match (sz, src, dst) {
2150            (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
2151                dynasm!(self ; clz X(dst), X(src));
2152            }
2153            (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
2154                dynasm!(self ; clz W(dst), W(src));
2155            }
2156            _ => codegen_error!("singlepass can't emit CLS {:?} {:?} {:?}", sz, src, dst),
2157        }
2158        Ok(())
2159    }
2160    fn emit_rbit(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2161        match (sz, src, dst) {
2162            (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
2163                dynasm!(self ; rbit X(dst), X(src));
2164            }
2165            (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
2166                dynasm!(self ; rbit W(dst), W(src));
2167            }
2168            _ => codegen_error!("singlepass can't emit CLS {:?} {:?} {:?}", sz, src, dst),
2169        }
2170        Ok(())
2171    }
2172
2173    fn emit_label(&mut self, label: Label) -> Result<(), CompileError> {
2174        dynasm!(self ; => label);
2175        Ok(())
2176    }
2177    fn emit_load_label(&mut self, reg: GPR, label: Label) -> Result<(), CompileError> {
2178        dynasm!(self ; adr X(reg), =>label);
2179        Ok(())
2180    }
2181
2182    fn emit_b_label(&mut self, label: Label) -> Result<(), CompileError> {
2183        dynasm!(self ; b =>label);
2184        Ok(())
2185    }
2186    fn emit_cbz_label(
2187        &mut self,
2188        sz: Size,
2189        reg: Location,
2190        label: Label,
2191    ) -> Result<(), CompileError> {
2192        match (sz, reg) {
2193            (Size::S32, Location::GPR(reg)) => {
2194                dynasm!(self ; cbz W(reg), =>label);
2195            }
2196            (Size::S64, Location::GPR(reg)) => {
2197                dynasm!(self ; cbz X(reg), =>label);
2198            }
2199            _ => codegen_error!("singlepass can't emit CBZ {:?} {:?} {:?}", sz, reg, label),
2200        }
2201        Ok(())
2202    }
2203    fn emit_cbnz_label(
2204        &mut self,
2205        sz: Size,
2206        reg: Location,
2207        label: Label,
2208    ) -> Result<(), CompileError> {
2209        match (sz, reg) {
2210            (Size::S32, Location::GPR(reg)) => {
2211                dynasm!(self ; cbnz W(reg), =>label);
2212            }
2213            (Size::S64, Location::GPR(reg)) => {
2214                dynasm!(self ; cbnz X(reg), =>label);
2215            }
2216            _ => codegen_error!("singlepass can't emit CBNZ {:?} {:?} {:?}", sz, reg, label),
2217        }
2218        Ok(())
2219    }
2220    fn emit_cbz_label_far(
2221        &mut self,
2222        sz: Size,
2223        reg: Location,
2224        label: Label,
2225    ) -> Result<(), CompileError> {
2226        let near_label: Label = self.get_label();
2227        let continue_label: Label = self.get_label();
2228
2229        match (sz, reg) {
2230            (Size::S32, Location::GPR(reg)) => {
2231                dynasm!(self ; cbz W(reg), => near_label);
2232                dynasm!(self ; b => continue_label);
2233            }
2234            (Size::S64, Location::GPR(reg)) => {
2235                dynasm!(self ; cbz X(reg), => near_label);
2236                dynasm!(self ; b => continue_label);
2237            }
2238            _ => codegen_error!("singlepass can't emit CBZ {:?} {:?} {:?}", sz, reg, label),
2239        }
2240        self.emit_label(near_label)?;
2241        dynasm!(self ; b => label );
2242
2243        self.emit_label(continue_label)?;
2244
2245        Ok(())
2246    }
2247    fn emit_tbz_label(
2248        &mut self,
2249        sz: Size,
2250        reg: Location,
2251        n: u32,
2252        label: Label,
2253    ) -> Result<(), CompileError> {
2254        match (sz, reg) {
2255            (Size::S32, Location::GPR(reg)) => {
2256                dynasm!(self ; tbz W(reg), n, =>label);
2257            }
2258            (Size::S64, Location::GPR(reg)) => {
2259                dynasm!(self ; tbz X(reg), n, =>label);
2260            }
2261            _ => codegen_error!(
2262                "singlepass can't emit TBZ {:?} {:?} {:?} {:?}",
2263                sz,
2264                reg,
2265                n,
2266                label
2267            ),
2268        }
2269        Ok(())
2270    }
2271    fn emit_tbnz_label(
2272        &mut self,
2273        sz: Size,
2274        reg: Location,
2275        n: u32,
2276        label: Label,
2277    ) -> Result<(), CompileError> {
2278        match (sz, reg) {
2279            (Size::S32, Location::GPR(reg)) => {
2280                dynasm!(self ; tbnz W(reg), n, =>label);
2281            }
2282            (Size::S64, Location::GPR(reg)) => {
2283                dynasm!(self ; tbnz X(reg), n, =>label);
2284            }
2285            _ => codegen_error!(
2286                "singlepass can't emit TBNZ {:?} {:?} {:?} {:?}",
2287                sz,
2288                reg,
2289                n,
2290                label
2291            ),
2292        }
2293        Ok(())
2294    }
2295    fn emit_bcond_label(&mut self, condition: Condition, label: Label) -> Result<(), CompileError> {
2296        match condition {
2297            Condition::Eq => dynasm!(self ; b.eq => label),
2298            Condition::Ne => dynasm!(self ; b.ne => label),
2299            Condition::Cs => dynasm!(self ; b.cs => label),
2300            Condition::Cc => dynasm!(self ; b.cc => label),
2301            Condition::Mi => dynasm!(self ; b.mi => label),
2302            Condition::Pl => dynasm!(self ; b.pl => label),
2303            Condition::Vs => dynasm!(self ; b.vs => label),
2304            Condition::Vc => dynasm!(self ; b.vc => label),
2305            Condition::Hi => dynasm!(self ; b.hi => label),
2306            Condition::Ls => dynasm!(self ; b.ls => label),
2307            Condition::Ge => dynasm!(self ; b.ge => label),
2308            Condition::Lt => dynasm!(self ; b.lt => label),
2309            Condition::Gt => dynasm!(self ; b.gt => label),
2310            Condition::Le => dynasm!(self ; b.le => label),
2311            Condition::Al => dynasm!(self ; b => label),
2312        }
2313        Ok(())
2314    }
2315    fn emit_bcond_label_far(
2316        &mut self,
2317        condition: Condition,
2318        label: Label,
2319    ) -> Result<(), CompileError> {
2320        let cont: Label = self.get_label();
2321        match condition {
2322            // if not condition than continue
2323            Condition::Eq => dynasm!(self ; b.ne => cont),
2324            Condition::Ne => dynasm!(self ; b.eq => cont),
2325            Condition::Cs => dynasm!(self ; b.cc => cont),
2326            Condition::Cc => dynasm!(self ; b.cs => cont),
2327            Condition::Mi => dynasm!(self ; b.pl => cont),
2328            Condition::Pl => dynasm!(self ; b.mi => cont),
2329            Condition::Vs => dynasm!(self ; b.vc => cont),
2330            Condition::Vc => dynasm!(self ; b.vs => cont),
2331            Condition::Hi => dynasm!(self ; b.ls => cont),
2332            Condition::Ls => dynasm!(self ; b.hi => cont),
2333            Condition::Ge => dynasm!(self ; b.lt => cont),
2334            Condition::Lt => dynasm!(self ; b.ge => cont),
2335            Condition::Gt => dynasm!(self ; b.le => cont),
2336            Condition::Le => dynasm!(self ; b.gt => cont),
2337            Condition::Al => { /*nothing*/ }
2338        }
2339        dynasm!(self ; b => label);
2340        self.emit_label(cont)?;
2341        Ok(())
2342    }
2343    fn emit_b_register(&mut self, reg: GPR) -> Result<(), CompileError> {
2344        dynasm!(self ; br X(reg));
2345        Ok(())
2346    }
2347    fn emit_call_label(&mut self, label: Label) -> Result<(), CompileError> {
2348        dynasm!(self ; bl =>label);
2349        Ok(())
2350    }
2351    fn emit_call_register(&mut self, reg: GPR) -> Result<(), CompileError> {
2352        dynasm!(self ; blr X(reg));
2353        Ok(())
2354    }
2355    fn emit_ret(&mut self) -> Result<(), CompileError> {
2356        dynasm!(self ; ret);
2357        Ok(())
2358    }
2359
2360    fn emit_udf(&mut self, payload: u16) -> Result<(), CompileError> {
2361        dynasm!(self ; udf payload as u32);
2362        Ok(())
2363    }
2364    fn emit_dmb(&mut self) -> Result<(), CompileError> {
2365        dynasm!(self ; dmb ish);
2366        Ok(())
2367    }
2368    fn emit_brk(&mut self) -> Result<(), CompileError> {
2369        dynasm!(self ; brk 0);
2370        Ok(())
2371    }
2372
2373    fn emit_fcmp(&mut self, sz: Size, src1: Location, src2: Location) -> Result<(), CompileError> {
2374        match (sz, src1, src2) {
2375            (Size::S32, Location::SIMD(src1), Location::SIMD(src2)) => {
2376                dynasm!(self ; fcmp S(src1), S(src2));
2377            }
2378            (Size::S64, Location::SIMD(src1), Location::SIMD(src2)) => {
2379                dynasm!(self ; fcmp D(src1), D(src2));
2380            }
2381            _ => codegen_error!("singlepass can't emit FCMP {:?} {:?} {:?}", sz, src1, src2),
2382        }
2383        Ok(())
2384    }
2385
2386    fn emit_fneg(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2387        match (sz, src, dst) {
2388            (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2389                dynasm!(self ; fneg S(dst), S(src));
2390            }
2391            (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2392                dynasm!(self ; fneg D(dst), D(src));
2393            }
2394            _ => codegen_error!("singlepass can't emit FNEG {:?} {:?} {:?}", sz, src, dst),
2395        }
2396        Ok(())
2397    }
2398    fn emit_fsqrt(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2399        match (sz, src, dst) {
2400            (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2401                dynasm!(self ; fsqrt S(dst), S(src));
2402            }
2403            (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2404                dynasm!(self ; fsqrt D(dst), D(src));
2405            }
2406            _ => codegen_error!("singlepass can't emit FSQRT {:?} {:?} {:?}", sz, src, dst),
2407        }
2408        Ok(())
2409    }
2410
2411    fn emit_fadd(
2412        &mut self,
2413        sz: Size,
2414        src1: Location,
2415        src2: Location,
2416        dst: Location,
2417    ) -> Result<(), CompileError> {
2418        match (sz, src1, src2, dst) {
2419            (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2420                dynasm!(self ; fadd S(dst), S(src1), S(src2));
2421            }
2422            (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2423                dynasm!(self ; fadd D(dst), D(src1), D(src2));
2424            }
2425            _ => codegen_error!(
2426                "singlepass can't emit FADD {:?} {:?} {:?} {:?}",
2427                sz,
2428                src1,
2429                src2,
2430                dst
2431            ),
2432        }
2433        Ok(())
2434    }
2435    fn emit_fsub(
2436        &mut self,
2437        sz: Size,
2438        src1: Location,
2439        src2: Location,
2440        dst: Location,
2441    ) -> Result<(), CompileError> {
2442        match (sz, src1, src2, dst) {
2443            (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2444                dynasm!(self ; fsub S(dst), S(src1), S(src2));
2445            }
2446            (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2447                dynasm!(self ; fsub D(dst), D(src1), D(src2));
2448            }
2449            _ => codegen_error!(
2450                "singlepass can't emit FSUB {:?} {:?} {:?} {:?}",
2451                sz,
2452                src1,
2453                src2,
2454                dst
2455            ),
2456        }
2457        Ok(())
2458    }
2459    fn emit_fmul(
2460        &mut self,
2461        sz: Size,
2462        src1: Location,
2463        src2: Location,
2464        dst: Location,
2465    ) -> Result<(), CompileError> {
2466        match (sz, src1, src2, dst) {
2467            (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2468                dynasm!(self ; fmul S(dst), S(src1), S(src2));
2469            }
2470            (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2471                dynasm!(self ; fmul D(dst), D(src1), D(src2));
2472            }
2473            _ => codegen_error!(
2474                "singlepass can't emit FMUL {:?} {:?} {:?} {:?}",
2475                sz,
2476                src1,
2477                src2,
2478                dst
2479            ),
2480        }
2481        Ok(())
2482    }
2483    fn emit_fdiv(
2484        &mut self,
2485        sz: Size,
2486        src1: Location,
2487        src2: Location,
2488        dst: Location,
2489    ) -> Result<(), CompileError> {
2490        match (sz, src1, src2, dst) {
2491            (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2492                dynasm!(self ; fdiv S(dst), S(src1), S(src2));
2493            }
2494            (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2495                dynasm!(self ; fdiv D(dst), D(src1), D(src2));
2496            }
2497            _ => codegen_error!(
2498                "singlepass can't emit FDIV {:?} {:?} {:?} {:?}",
2499                sz,
2500                src1,
2501                src2,
2502                dst
2503            ),
2504        }
2505        Ok(())
2506    }
2507
2508    fn emit_fmin(
2509        &mut self,
2510        sz: Size,
2511        src1: Location,
2512        src2: Location,
2513        dst: Location,
2514    ) -> Result<(), CompileError> {
2515        match (sz, src1, src2, dst) {
2516            (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2517                dynasm!(self ; fmin S(dst), S(src1), S(src2));
2518            }
2519            (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2520                dynasm!(self ; fmin D(dst), D(src1), D(src2));
2521            }
2522            _ => codegen_error!(
2523                "singlepass can't emit FMIN {:?} {:?} {:?} {:?}",
2524                sz,
2525                src1,
2526                src2,
2527                dst
2528            ),
2529        }
2530        Ok(())
2531    }
2532    fn emit_fmax(
2533        &mut self,
2534        sz: Size,
2535        src1: Location,
2536        src2: Location,
2537        dst: Location,
2538    ) -> Result<(), CompileError> {
2539        match (sz, src1, src2, dst) {
2540            (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2541                dynasm!(self ; fmax S(dst), S(src1), S(src2));
2542            }
2543            (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2544                dynasm!(self ; fmax D(dst), D(src1), D(src2));
2545            }
2546            _ => codegen_error!(
2547                "singlepass can't emit FMAX {:?} {:?} {:?} {:?}",
2548                sz,
2549                src1,
2550                src2,
2551                dst
2552            ),
2553        }
2554        Ok(())
2555    }
2556
2557    fn emit_frintz(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2558        match (sz, src, dst) {
2559            (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2560                dynasm!(self ; frintz S(dst), S(src));
2561            }
2562            (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2563                dynasm!(self ; frintz D(dst), D(src));
2564            }
2565            _ => codegen_error!("singlepass can't emit FRINTZ {:?} {:?} {:?}", sz, src, dst),
2566        }
2567        Ok(())
2568    }
2569    fn emit_frintn(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2570        match (sz, src, dst) {
2571            (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2572                dynasm!(self ; frintn S(dst), S(src));
2573            }
2574            (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2575                dynasm!(self ; frintn D(dst), D(src));
2576            }
2577            _ => codegen_error!("singlepass can't emit FRINTN {:?} {:?} {:?}", sz, src, dst),
2578        }
2579        Ok(())
2580    }
2581    fn emit_frintm(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2582        match (sz, src, dst) {
2583            (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2584                dynasm!(self ; frintm S(dst), S(src));
2585            }
2586            (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2587                dynasm!(self ; frintm D(dst), D(src));
2588            }
2589            _ => codegen_error!("singlepass can't emit FRINTM {:?} {:?} {:?}", sz, src, dst),
2590        }
2591        Ok(())
2592    }
2593    fn emit_frintp(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2594        match (sz, src, dst) {
2595            (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2596                dynasm!(self ; frintp S(dst), S(src));
2597            }
2598            (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2599                dynasm!(self ; frintp D(dst), D(src));
2600            }
2601            _ => codegen_error!("singlepass can't emit FRINTP {:?} {:?} {:?}", sz, src, dst),
2602        }
2603        Ok(())
2604    }
2605
2606    fn emit_scvtf(
2607        &mut self,
2608        sz_in: Size,
2609        src: Location,
2610        sz_out: Size,
2611        dst: Location,
2612    ) -> Result<(), CompileError> {
2613        match (sz_in, src, sz_out, dst) {
2614            (Size::S32, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
2615                dynasm!(self ; scvtf S(dst), W(src));
2616            }
2617            (Size::S64, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
2618                dynasm!(self ; scvtf S(dst), X(src));
2619            }
2620            (Size::S32, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
2621                dynasm!(self ; scvtf D(dst), W(src));
2622            }
2623            (Size::S64, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
2624                dynasm!(self ; scvtf D(dst), X(src));
2625            }
2626            _ => codegen_error!(
2627                "singlepass can't emit SCVTF {:?} {:?} {:?} {:?}",
2628                sz_in,
2629                src,
2630                sz_out,
2631                dst
2632            ),
2633        }
2634        Ok(())
2635    }
2636    fn emit_ucvtf(
2637        &mut self,
2638        sz_in: Size,
2639        src: Location,
2640        sz_out: Size,
2641        dst: Location,
2642    ) -> Result<(), CompileError> {
2643        match (sz_in, src, sz_out, dst) {
2644            (Size::S32, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
2645                dynasm!(self ; ucvtf S(dst), W(src));
2646            }
2647            (Size::S64, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
2648                dynasm!(self ; ucvtf S(dst), X(src));
2649            }
2650            (Size::S32, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
2651                dynasm!(self ; ucvtf D(dst), W(src));
2652            }
2653            (Size::S64, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
2654                dynasm!(self ; ucvtf D(dst), X(src));
2655            }
2656            _ => codegen_error!(
2657                "singlepass can't emit UCVTF {:?} {:?} {:?} {:?}",
2658                sz_in,
2659                src,
2660                sz_out,
2661                dst
2662            ),
2663        }
2664        Ok(())
2665    }
2666    fn emit_fcvt(&mut self, sz_in: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2667        match (sz_in, src, dst) {
2668            (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2669                dynasm!(self ; fcvt D(dst), S(src));
2670            }
2671            (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2672                dynasm!(self ; fcvt S(dst), D(src));
2673            }
2674            _ => codegen_error!(
2675                "singlepass can't emit UCVTF {:?} {:?} {:?}",
2676                sz_in,
2677                src,
2678                dst
2679            ),
2680        }
2681        Ok(())
2682    }
2683    fn emit_fcvtzs(
2684        &mut self,
2685        sz_in: Size,
2686        src: Location,
2687        sz_out: Size,
2688        dst: Location,
2689    ) -> Result<(), CompileError> {
2690        match (sz_in, src, sz_out, dst) {
2691            (Size::S32, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
2692                dynasm!(self ; fcvtzs W(dst), S(src));
2693            }
2694            (Size::S64, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
2695                dynasm!(self ; fcvtzs W(dst), D(src));
2696            }
2697            (Size::S32, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
2698                dynasm!(self ; fcvtzs X(dst), S(src));
2699            }
2700            (Size::S64, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
2701                dynasm!(self ; fcvtzs X(dst), D(src));
2702            }
2703            _ => codegen_error!(
2704                "singlepass can't emit FCVTZS {:?} {:?} {:?} {:?}",
2705                sz_in,
2706                src,
2707                sz_out,
2708                dst
2709            ),
2710        }
2711        Ok(())
2712    }
2713    fn emit_fcvtzu(
2714        &mut self,
2715        sz_in: Size,
2716        src: Location,
2717        sz_out: Size,
2718        dst: Location,
2719    ) -> Result<(), CompileError> {
2720        match (sz_in, src, sz_out, dst) {
2721            (Size::S32, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
2722                dynasm!(self ; fcvtzu W(dst), S(src));
2723            }
2724            (Size::S64, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
2725                dynasm!(self ; fcvtzu W(dst), D(src));
2726            }
2727            (Size::S32, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
2728                dynasm!(self ; fcvtzu X(dst), S(src));
2729            }
2730            (Size::S64, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
2731                dynasm!(self ; fcvtzu X(dst), D(src));
2732            }
2733            _ => codegen_error!(
2734                "singlepass can't emit FCVTZU {:?} {:?} {:?} {:?}",
2735                sz_in,
2736                src,
2737                sz_out,
2738                dst
2739            ),
2740        }
2741        Ok(())
2742    }
2743
2744    // 1 011 0100 0100 000 => fpcr
2745    fn emit_read_fpcr(&mut self, reg: GPR) -> Result<(), CompileError> {
2746        dynasm!(self ; mrs X(reg), 0b1_011_0100_0100_000);
2747        Ok(())
2748    }
2749    fn emit_write_fpcr(&mut self, reg: GPR) -> Result<(), CompileError> {
2750        dynasm!(self ; msr 0b1_011_0100_0100_000, X(reg));
2751        Ok(())
2752    }
2753    // 1 011 0100 0100 001 => fpsr
2754    fn emit_read_fpsr(&mut self, reg: GPR) -> Result<(), CompileError> {
2755        dynasm!(self ; mrs X(reg), 0b1_011_0100_0100_001);
2756        Ok(())
2757    }
2758    fn emit_write_fpsr(&mut self, reg: GPR) -> Result<(), CompileError> {
2759        dynasm!(self ; msr 0b1_011_0100_0100_001, X(reg));
2760        Ok(())
2761    }
2762
2763    fn emit_cnt(&mut self, src: NEON, dst: NEON) -> Result<(), CompileError> {
2764        dynasm!(self ; cnt V(dst).B8, V(src).B8);
2765        Ok(())
2766    }
2767
2768    fn emit_addv(&mut self, src: NEON, dst: NEON) -> Result<(), CompileError> {
2769        dynasm!(self ; addv B(dst), V(src).B8);
2770        Ok(())
2771    }
2772
2773    #[allow(clippy::unit_arg)]
2774    fn emit_fmov(
2775        &mut self,
2776        src_size: Size,
2777        src: Location,
2778        dst_size: Size,
2779        dst: Location,
2780    ) -> Result<(), CompileError> {
2781        let mut err = false;
2782        match (src, dst) {
2783            (AbstractLocation::GPR(src), AbstractLocation::SIMD(dst)) => {
2784                match (src_size, dst_size) {
2785                    (Size::S32, Size::S32) => dynasm!(self ; fmov S(dst), W(src)),
2786                    (Size::S64, Size::S64) => dynasm!(self ; fmov D(dst), X(src)),
2787                    _ => {
2788                        err = true;
2789                    }
2790                }
2791            }
2792            (AbstractLocation::SIMD(src), AbstractLocation::GPR(dst)) => {
2793                match (src_size, dst_size) {
2794                    (Size::S32, Size::S32) => dynasm!(self ; fmov W(dst), S(src)),
2795                    (Size::S64, Size::S64) => dynasm!(self ; fmov X(dst), D(src)),
2796                    _ => {
2797                        err = true;
2798                    }
2799                }
2800            }
2801            // (AbstractLocation::SIMD(src), AbstractLocation::SIMD(dst)) => todo!(),
2802            // (AbstractLocation::SIMD(src), AbstractLocation::Imm32(dst)) => todo!(),
2803            // (AbstractLocation::SIMD(src), AbstractLocation::Imm64(dst)) => todo!(),
2804            _ => {
2805                err = true;
2806            }
2807        };
2808
2809        if err {
2810            codegen_error!(
2811                "singlepass can't generate fmov instruction for src_size: {:?}, src: {:?}, dst_size: {:?}, dst: {:?}",
2812                src_size,
2813                src,
2814                dst_size,
2815                dst,
2816            )
2817        }
2818        Ok(())
2819    }
2820}
2821
2822pub fn gen_std_trampoline_arm64(
2823    sig: &FunctionType,
2824    calling_convention: CallingConvention,
2825) -> Result<FunctionBody, CompileError> {
2826    let mut a = Assembler::new(0);
2827
2828    let fptr = GPR::X27;
2829    let args = GPR::X28;
2830
2831    dynasm!(a
2832        ; sub sp, sp, 32
2833        ; stp x29, x30, [sp]
2834        ; stp X(fptr), X(args), [sp, 16]
2835        ; mov x29, sp
2836        ; mov X(fptr), x1
2837        ; mov X(args), x2
2838    );
2839
2840    let stack_args = sig.params().len().saturating_sub(7); //1st arg is ctx, not an actual arg
2841    let mut stack_offset = stack_args as u32 * 8;
2842    if stack_args > 0 {
2843        if stack_offset % 16 != 0 {
2844            stack_offset += 8;
2845            assert!(stack_offset % 16 == 0);
2846        }
2847        if stack_offset < 0x1000 {
2848            dynasm!(a ; sub sp, sp, stack_offset);
2849        } else {
2850            a.emit_mov_imm(Location::GPR(GPR::X26), stack_offset as u64)?;
2851            dynasm!(a ; sub sp, sp, x26);
2852        }
2853    }
2854
2855    // Move arguments to their locations.
2856    // `callee_vmctx` is already in the first argument register, so no need to move.
2857    let mut caller_stack_offset: i32 = 0;
2858    for (i, param) in sig.params().iter().enumerate() {
2859        let sz = match *param {
2860            Type::I32 | Type::F32 => Size::S32,
2861            Type::I64 | Type::F64 => Size::S64,
2862            Type::ExternRef => Size::S64,
2863            Type::FuncRef => Size::S64,
2864            _ => codegen_error!(
2865                "singlepass unsupported param type for trampoline {:?}",
2866                *param
2867            ),
2868        };
2869        match i {
2870            0..=6 => {
2871                a.emit_ldr(
2872                    sz,
2873                    Location::GPR(GPR::from_index(i + 1).unwrap()),
2874                    Location::Memory(args, (i * 16) as i32),
2875                )?;
2876            }
2877            _ => {
2878                #[allow(clippy::single_match)]
2879                match calling_convention {
2880                    CallingConvention::AppleAarch64 => {
2881                        let sz = 1
2882                            << match sz {
2883                                Size::S8 => 0,
2884                                Size::S16 => 1,
2885                                Size::S32 => 2,
2886                                Size::S64 => 3,
2887                            };
2888                        // align first
2889                        if sz > 1 && caller_stack_offset & (sz - 1) != 0 {
2890                            caller_stack_offset = (caller_stack_offset + (sz - 1)) & !(sz - 1);
2891                        }
2892                    }
2893                    _ => (),
2894                };
2895                // using X16 as scratch reg
2896                a.emit_ldr(
2897                    sz,
2898                    Location::GPR(GPR::X16),
2899                    Location::Memory(args, (i * 16) as i32),
2900                )?;
2901                a.emit_str(
2902                    sz,
2903                    Location::GPR(GPR::X16),
2904                    Location::Memory(GPR::XzrSp, caller_stack_offset),
2905                )?;
2906                match calling_convention {
2907                    CallingConvention::AppleAarch64 => {
2908                        caller_stack_offset += 1
2909                            << match sz {
2910                                Size::S8 => 0,
2911                                Size::S16 => 1,
2912                                Size::S32 => 2,
2913                                Size::S64 => 3,
2914                            };
2915                    }
2916                    _ => {
2917                        caller_stack_offset += 8;
2918                    }
2919                }
2920            }
2921        }
2922    }
2923
2924    dynasm!(a  ; blr X(fptr));
2925
2926    // Write return value.
2927    if !sig.results().is_empty() {
2928        a.emit_str(Size::S64, Location::GPR(GPR::X0), Location::Memory(args, 0))?;
2929    }
2930
2931    // Restore stack.
2932    dynasm!(a
2933        ; ldp X(fptr), X(args), [x29, 16]
2934        ; ldp x29, x30, [x29]
2935    );
2936    let restored_stack_offset = 32 + stack_offset;
2937    if restored_stack_offset < 0x1000 {
2938        dynasm!(a; add sp, sp, restored_stack_offset);
2939    } else {
2940        a.emit_mov_imm(Location::GPR(GPR::X26), restored_stack_offset as u64)?;
2941        dynasm!(a; add sp, sp, x26);
2942    }
2943    dynasm!(a; ret);
2944
2945    let mut body = a.finalize().unwrap();
2946    body.shrink_to_fit();
2947    Ok(FunctionBody {
2948        body,
2949        unwind_info: None,
2950    })
2951}
2952// Generates dynamic import function call trampoline for a function type.
2953pub fn gen_std_dynamic_import_trampoline_arm64(
2954    vmoffsets: &VMOffsets,
2955    sig: &FunctionType,
2956    calling_convention: CallingConvention,
2957) -> Result<FunctionBody, CompileError> {
2958    let mut a = Assembler::new(0);
2959    // Allocate argument array.
2960    let stack_offset: usize = 16 * std::cmp::max(sig.params().len(), sig.results().len());
2961    // Save LR and X26, as scratch register
2962    a.emit_stpdb(
2963        Size::S64,
2964        Location::GPR(GPR::X30),
2965        Location::GPR(GPR::X26),
2966        GPR::XzrSp,
2967        16,
2968    )?;
2969
2970    if stack_offset != 0 {
2971        if stack_offset < 0x1000 {
2972            a.emit_sub(
2973                Size::S64,
2974                Location::GPR(GPR::XzrSp),
2975                Location::Imm32(stack_offset as _),
2976                Location::GPR(GPR::XzrSp),
2977            )?;
2978        } else {
2979            a.emit_mov_imm(Location::GPR(GPR::X26), stack_offset as u64)?;
2980            a.emit_sub(
2981                Size::S64,
2982                Location::GPR(GPR::XzrSp),
2983                Location::GPR(GPR::X26),
2984                Location::GPR(GPR::XzrSp),
2985            )?;
2986        }
2987    }
2988
2989    // Copy arguments.
2990    if !sig.params().is_empty() {
2991        let mut argalloc = ArgumentRegisterAllocator::default();
2992        argalloc.next(Type::I64, calling_convention).unwrap(); // skip VMContext
2993
2994        let mut stack_param_count: usize = 0;
2995
2996        for (i, ty) in sig.params().iter().enumerate() {
2997            let source_loc = match argalloc.next(*ty, calling_convention)? {
2998                Some(ARM64Register::GPR(gpr)) => Location::GPR(gpr),
2999                Some(ARM64Register::NEON(neon)) => Location::SIMD(neon),
3000                None => {
3001                    let sz = match calling_convention {
3002                        CallingConvention::AppleAarch64 => match *ty {
3003                            Type::I32 | Type::F32 => Size::S32,
3004                            _ => {
3005                                if stack_param_count & 7 != 0 {
3006                                    stack_param_count = (stack_param_count + 7) & !7;
3007                                };
3008                                Size::S64
3009                            }
3010                        },
3011                        _ => Size::S64,
3012                    };
3013                    a.emit_ldr(
3014                        sz,
3015                        Location::GPR(GPR::X26),
3016                        Location::Memory(GPR::XzrSp, (stack_offset + 16 + stack_param_count) as _),
3017                    )?;
3018                    stack_param_count += match sz {
3019                        Size::S32 => 4,
3020                        Size::S64 => 8,
3021                        _ => codegen_error!(
3022                            "singlepass unreachable in gen_std_dynamic_import_trampoline_arm64"
3023                        ),
3024                    };
3025                    Location::GPR(GPR::X26)
3026                }
3027            };
3028            a.emit_str(
3029                Size::S64,
3030                source_loc,
3031                Location::Memory(GPR::XzrSp, (i * 16) as _),
3032            )?;
3033
3034            // Zero upper 64 bits.
3035            a.emit_str(
3036                Size::S64,
3037                Location::GPR(GPR::XzrSp),                       // XZR here
3038                Location::Memory(GPR::XzrSp, (i * 16 + 8) as _), // XSP here
3039            )?;
3040        }
3041    }
3042
3043    #[allow(clippy::match_single_binding)]
3044    match calling_convention {
3045        _ => {
3046            // Load target address.
3047            let offset = vmoffsets.vmdynamicfunction_import_context_address();
3048            a.emit_ldur(Size::S64, Location::GPR(GPR::X26), GPR::X0, offset as i32)?;
3049            // Load values array.
3050            a.emit_add(
3051                Size::S64,
3052                Location::GPR(GPR::XzrSp),
3053                Location::Imm8(0),
3054                Location::GPR(GPR::X1),
3055            )?;
3056        }
3057    };
3058
3059    // Call target.
3060    a.emit_call_register(GPR::X26)?;
3061
3062    // Fetch return value.
3063    if !sig.results().is_empty() {
3064        assert_eq!(sig.results().len(), 1);
3065        a.emit_ldr(
3066            Size::S64,
3067            Location::GPR(GPR::X0),
3068            Location::Memory(GPR::XzrSp, 0),
3069        )?;
3070    }
3071
3072    // Release values array.
3073    if stack_offset != 0 {
3074        if stack_offset < 0x1000 {
3075            a.emit_add(
3076                Size::S64,
3077                Location::GPR(GPR::XzrSp),
3078                Location::Imm32(stack_offset as _),
3079                Location::GPR(GPR::XzrSp),
3080            )?;
3081        } else {
3082            a.emit_mov_imm(Location::GPR(GPR::X26), stack_offset as u64)?;
3083            a.emit_add(
3084                Size::S64,
3085                Location::GPR(GPR::XzrSp),
3086                Location::GPR(GPR::X26),
3087                Location::GPR(GPR::XzrSp),
3088            )?;
3089        }
3090    }
3091    a.emit_ldpia(
3092        Size::S64,
3093        Location::GPR(GPR::X30),
3094        Location::GPR(GPR::X26),
3095        GPR::XzrSp,
3096        16,
3097    )?;
3098
3099    // Return.
3100    a.emit_ret()?;
3101
3102    let mut body = a.finalize().unwrap();
3103    body.shrink_to_fit();
3104    Ok(FunctionBody {
3105        body,
3106        unwind_info: None,
3107    })
3108}
3109// Singlepass calls import functions through a trampoline.
3110pub fn gen_import_call_trampoline_arm64(
3111    vmoffsets: &VMOffsets,
3112    index: FunctionIndex,
3113    sig: &FunctionType,
3114    calling_convention: CallingConvention,
3115) -> Result<CustomSection, CompileError> {
3116    let mut a = Assembler::new(0);
3117
3118    // Singlepass internally treats all arguments as integers
3119    // For the standard System V calling convention requires
3120    //  floating point arguments to be passed in NEON registers.
3121    //  Translation is expensive, so only do it if needed.
3122    if sig
3123        .params()
3124        .iter()
3125        .any(|&x| x == Type::F32 || x == Type::F64)
3126    {
3127        #[allow(clippy::match_single_binding)]
3128        match calling_convention {
3129            _ => {
3130                // Allocate stack space for arguments.
3131                let stack_offset: i32 = if sig.params().len() > 7 {
3132                    7 * 8
3133                } else {
3134                    (sig.params().len() as i32) * 8
3135                };
3136                let stack_offset = if stack_offset & 15 != 0 {
3137                    stack_offset + 8
3138                } else {
3139                    stack_offset
3140                };
3141                if stack_offset > 0 {
3142                    if stack_offset < 0x1000 {
3143                        a.emit_sub(
3144                            Size::S64,
3145                            Location::GPR(GPR::XzrSp),
3146                            Location::Imm32(stack_offset as u32),
3147                            Location::GPR(GPR::XzrSp),
3148                        )?;
3149                    } else {
3150                        a.emit_mov_imm(Location::GPR(GPR::X16), stack_offset as u64)?;
3151                        a.emit_sub(
3152                            Size::S64,
3153                            Location::GPR(GPR::XzrSp),
3154                            Location::GPR(GPR::X16),
3155                            Location::GPR(GPR::XzrSp),
3156                        )?;
3157                    }
3158                }
3159
3160                // Store all arguments to the stack to prevent overwrite.
3161                static PARAM_REGS: &[GPR] = &[
3162                    GPR::X1,
3163                    GPR::X2,
3164                    GPR::X3,
3165                    GPR::X4,
3166                    GPR::X5,
3167                    GPR::X6,
3168                    GPR::X7,
3169                ];
3170                let mut param_locations = vec![];
3171                for (i, param_reg) in PARAM_REGS.iter().enumerate().take(sig.params().len()) {
3172                    let loc = match i {
3173                        0..=6 => {
3174                            let loc = Location::Memory(GPR::XzrSp, (i * 8) as i32);
3175                            a.emit_str(Size::S64, Location::GPR(*param_reg), loc)?;
3176                            loc
3177                        }
3178                        _ => Location::Memory(GPR::XzrSp, stack_offset + ((i - 7) * 8) as i32),
3179                    };
3180                    param_locations.push(loc);
3181                }
3182
3183                // Copy arguments.
3184                let mut caller_stack_offset: i32 = 0;
3185                let mut argalloc = ArgumentRegisterAllocator::default();
3186                argalloc.next(Type::I64, calling_convention).unwrap(); // skip VMContext
3187                for (i, ty) in sig.params().iter().enumerate() {
3188                    let prev_loc = param_locations[i];
3189                    let targ = match argalloc.next(*ty, calling_convention)? {
3190                        Some(ARM64Register::GPR(gpr)) => Location::GPR(gpr),
3191                        Some(ARM64Register::NEON(neon)) => Location::SIMD(neon),
3192                        None => {
3193                            // No register can be allocated. Put this argument on the stack.
3194                            a.emit_ldr(Size::S64, Location::GPR(GPR::X16), prev_loc)?;
3195                            a.emit_str(
3196                                Size::S64,
3197                                Location::GPR(GPR::X16),
3198                                Location::Memory(GPR::XzrSp, stack_offset + caller_stack_offset),
3199                            )?;
3200                            caller_stack_offset += 8;
3201                            continue;
3202                        }
3203                    };
3204                    a.emit_ldr(Size::S64, targ, prev_loc)?;
3205                }
3206
3207                // Restore stack pointer.
3208                if stack_offset > 0 {
3209                    if stack_offset < 0x1000 {
3210                        a.emit_add(
3211                            Size::S64,
3212                            Location::GPR(GPR::XzrSp),
3213                            Location::Imm32(stack_offset as u32),
3214                            Location::GPR(GPR::XzrSp),
3215                        )?;
3216                    } else {
3217                        a.emit_mov_imm(Location::GPR(GPR::X16), stack_offset as u64)?;
3218                        a.emit_add(
3219                            Size::S64,
3220                            Location::GPR(GPR::XzrSp),
3221                            Location::GPR(GPR::X16),
3222                            Location::GPR(GPR::XzrSp),
3223                        )?;
3224                    }
3225                }
3226            }
3227        }
3228    }
3229
3230    // Emits a tail call trampoline that loads the address of the target import function
3231    // from Ctx and jumps to it.
3232
3233    let offset = vmoffsets.vmctx_vmfunction_import(index);
3234    // for ldr, offset needs to be a multiple of 8, wich often is not
3235    // so use ldur, but then offset is limited to -255 .. +255. It will be positive here
3236    let offset =
3237        if (offset > 0 && offset < 0xF8) || (offset > 0 && offset < 0x7FF8 && (offset & 7) == 0) {
3238            offset
3239        } else {
3240            a.emit_mov_imm(Location::GPR(GPR::X16), (offset as i64) as u64)?;
3241            a.emit_add(
3242                Size::S64,
3243                Location::GPR(GPR::X0),
3244                Location::GPR(GPR::X16),
3245                Location::GPR(GPR::X0),
3246            )?;
3247            0
3248        };
3249    #[allow(clippy::match_single_binding)]
3250    match calling_convention {
3251        _ => {
3252            if (offset & 7) == 0 {
3253                a.emit_ldr(
3254                    Size::S64,
3255                    Location::GPR(GPR::X16),
3256                    Location::Memory(GPR::X0, offset as i32), // function pointer
3257                )?;
3258                a.emit_ldr(
3259                    Size::S64,
3260                    Location::GPR(GPR::X0),
3261                    Location::Memory(GPR::X0, offset as i32 + 8), // target vmctx
3262                )?;
3263            } else {
3264                a.emit_ldur(
3265                    Size::S64,
3266                    Location::GPR(GPR::X16),
3267                    GPR::X0,
3268                    offset as i32, // function pointer
3269                )?;
3270                a.emit_ldur(
3271                    Size::S64,
3272                    Location::GPR(GPR::X0),
3273                    GPR::X0,
3274                    offset as i32 + 8, // target vmctx
3275                )?;
3276            }
3277        }
3278    }
3279    a.emit_b_register(GPR::X16)?;
3280
3281    let mut contents = a.finalize().unwrap();
3282    contents.shrink_to_fit();
3283    let section_body = SectionBody::new_with_vec(contents);
3284
3285    Ok(CustomSection {
3286        protection: CustomSectionProtection::ReadExecute,
3287        alignment: None,
3288        bytes: section_body,
3289        relocations: vec![],
3290    })
3291}