wasmer_compiler_singlepass/
emitter_arm64.rs

1pub use crate::{
2    arm64_decl::{ARM64Register, ArgumentRegisterAllocator, GPR, NEON},
3    location::{Multiplier, Reg},
4    machine::{Label, Offset},
5};
6use crate::{
7    codegen_error, common_decl::Size, location::Location as AbstractLocation,
8    machine_arm64::ARM64_RETURN_VALUE_REGISTERS, output_reporter::ChunkedOutputReporter,
9};
10pub use dynasmrt::aarch64::{encode_logical_immediate_32bit, encode_logical_immediate_64bit};
11use dynasmrt::{
12    AssemblyOffset, DynamicLabel, DynasmApi, DynasmLabelApi, VecAssembler,
13    aarch64::Aarch64Relocation,
14};
15use wasmer_compiler::types::{
16    function::FunctionBody,
17    section::{CustomSection, CustomSectionProtection, SectionBody},
18};
19use wasmer_types::{
20    CompilationProgressCallback, CompileError, FunctionIndex, FunctionType, Type, VMOffsets,
21    target::CallingConvention, vmctx_offset,
22};
23
24type Assembler = VecAssembler<Aarch64Relocation>;
25
26/// Force `dynasm!` to use the correct arch (aarch64) when cross-compiling.
27/// `dynasm!` proc-macro tries to auto-detect it by default by looking at the
28/// `target_arch`, but it sees the `target_arch` of the proc-macro itself, which
29/// is always equal to host, even when cross-compiling.
30macro_rules! dynasm {
31    ($a:expr ; $($tt:tt)*) => {
32        dynasm::dynasm!(
33            $a
34            ; .arch aarch64
35            ; $($tt)*
36        )
37    };
38}
39
40pub type Location = AbstractLocation<GPR, NEON>;
41
42#[derive(Copy, Clone, Debug, Eq, PartialEq)]
43#[allow(dead_code)]
44#[repr(u8)]
45pub enum Condition {
46    // meaning for cmp or sub
47    /// Equal
48    Eq = 0,
49    /// Not equal
50    Ne = 1,
51    /// Unsigned higher or same (or carry set)
52    Cs = 2,
53    /// Unsigned lower (or carry clear)
54    Cc = 3,
55    /// Negative. The mnemonic stands for "minus"
56    Mi = 4,
57    /// Positive or zero. The mnemonic stands for "plus"
58    Pl = 5,
59    /// Signed overflow. The mnemonic stands for "V set"
60    Vs = 6,
61    /// No signed overflow. The mnemonic stands for "V clear"
62    Vc = 7,
63    /// Unsigned higher
64    Hi = 8,
65    /// Unsigned lower or same
66    Ls = 9,
67    /// Signed greater than or equal
68    Ge = 10,
69    /// Signed less than
70    Lt = 11,
71    /// Signed greater than
72    Gt = 12,
73    /// Signed less than or equal
74    Le = 13,
75    /// Always executed
76    Al = 14,
77}
78
79#[derive(Copy, Clone, Debug, Eq, PartialEq)]
80#[allow(dead_code, clippy::upper_case_acronyms)]
81pub enum NeonOrMemory {
82    NEON(NEON),
83    Memory(GPR, i32),
84}
85
86#[derive(Copy, Clone, Debug)]
87#[allow(dead_code, clippy::upper_case_acronyms)]
88pub enum GPROrMemory {
89    GPR(GPR),
90    Memory(GPR, i32),
91}
92
93#[allow(unused)]
94pub trait EmitterARM64 {
95    fn get_label(&mut self) -> Label;
96    fn get_offset(&self) -> Offset;
97    fn get_jmp_instr_size(&self) -> u8;
98
99    fn finalize_function(&mut self);
100
101    fn emit_str(&mut self, sz: Size, reg: Location, addr: Location) -> Result<(), CompileError>;
102    fn emit_ldr(&mut self, sz: Size, reg: Location, addr: Location) -> Result<(), CompileError>;
103    fn emit_stur(
104        &mut self,
105        sz: Size,
106        reg: Location,
107        addr: GPR,
108        offset: i32,
109    ) -> Result<(), CompileError>;
110    fn emit_ldur(
111        &mut self,
112        sz: Size,
113        reg: Location,
114        addr: GPR,
115        offset: i32,
116    ) -> Result<(), CompileError>;
117    fn emit_strdb(
118        &mut self,
119        sz: Size,
120        reg: Location,
121        addr: GPR,
122        offset: u32,
123    ) -> Result<(), CompileError>;
124    fn emit_stria(
125        &mut self,
126        sz: Size,
127        reg: Location,
128        addr: GPR,
129        offset: u32,
130    ) -> Result<(), CompileError>;
131    fn emit_ldria(
132        &mut self,
133        sz: Size,
134        reg: Location,
135        addr: GPR,
136        offset: u32,
137    ) -> Result<(), CompileError>;
138    fn emit_stpdb(
139        &mut self,
140        sz: Size,
141        reg1: Location,
142        reg2: Location,
143        addr: GPR,
144        offset: u32,
145    ) -> Result<(), CompileError>;
146    fn emit_ldpia(
147        &mut self,
148        sz: Size,
149        reg1: Location,
150        reg2: Location,
151        addr: GPR,
152        offset: u32,
153    ) -> Result<(), CompileError>;
154
155    fn emit_ldrb(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
156    fn emit_ldrh(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
157    fn emit_ldrsb(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
158    fn emit_ldrsh(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
159    fn emit_ldrsw(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
160    fn emit_strb(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
161    fn emit_strh(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
162
163    fn emit_ldaxr(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
164    fn emit_ldaxrb(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
165    fn emit_ldaxrh(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
166    fn emit_stlxr(
167        &mut self,
168        sz: Size,
169        status: Location,
170        reg: Location,
171        dst: Location,
172    ) -> Result<(), CompileError>;
173    fn emit_stlxrb(
174        &mut self,
175        sz: Size,
176        status: Location,
177        reg: Location,
178        dst: Location,
179    ) -> Result<(), CompileError>;
180    fn emit_stlxrh(
181        &mut self,
182        sz: Size,
183        status: Location,
184        reg: Location,
185        dst: Location,
186    ) -> Result<(), CompileError>;
187
188    fn emit_mov(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
189
190    fn emit_movn(&mut self, sz: Size, reg: Location, val: u32) -> Result<(), CompileError>;
191    fn emit_movz(&mut self, reg: Location, val: u32) -> Result<(), CompileError>;
192    fn emit_movk(&mut self, reg: Location, val: u32, shift: u32) -> Result<(), CompileError>;
193
194    fn emit_mov_imm(&mut self, dst: Location, val: u64) -> Result<(), CompileError>;
195
196    fn emit_add(
197        &mut self,
198        sz: Size,
199        src1: Location,
200        src2: Location,
201        dst: Location,
202    ) -> Result<(), CompileError>;
203    fn emit_sub(
204        &mut self,
205        sz: Size,
206        src1: Location,
207        src2: Location,
208        dst: Location,
209    ) -> Result<(), CompileError>;
210    fn emit_mul(
211        &mut self,
212        sz: Size,
213        src1: Location,
214        src2: Location,
215        dst: Location,
216    ) -> Result<(), CompileError>;
217    fn emit_adds(
218        &mut self,
219        sz: Size,
220        src1: Location,
221        src2: Location,
222        dst: Location,
223    ) -> Result<(), CompileError>;
224    fn emit_subs(
225        &mut self,
226        sz: Size,
227        src1: Location,
228        src2: Location,
229        dst: Location,
230    ) -> Result<(), CompileError>;
231
232    fn emit_add_lsl(
233        &mut self,
234        sz: Size,
235        src1: Location,
236        src2: Location,
237        lsl: u32,
238        dst: Location,
239    ) -> Result<(), CompileError>;
240
241    fn emit_cmp(&mut self, sz: Size, left: Location, right: Location) -> Result<(), CompileError>;
242    fn emit_tst(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
243
244    fn emit_lsl(
245        &mut self,
246        sz: Size,
247        src1: Location,
248        src2: Location,
249        dst: Location,
250    ) -> Result<(), CompileError>;
251    fn emit_lsr(
252        &mut self,
253        sz: Size,
254        src1: Location,
255        src2: Location,
256        dst: Location,
257    ) -> Result<(), CompileError>;
258    fn emit_asr(
259        &mut self,
260        sz: Size,
261        src1: Location,
262        src2: Location,
263        dst: Location,
264    ) -> Result<(), CompileError>;
265    fn emit_ror(
266        &mut self,
267        sz: Size,
268        src1: Location,
269        src2: Location,
270        dst: Location,
271    ) -> Result<(), CompileError>;
272
273    fn emit_or(
274        &mut self,
275        sz: Size,
276        src1: Location,
277        src2: Location,
278        dst: Location,
279    ) -> Result<(), CompileError>;
280    fn emit_and(
281        &mut self,
282        sz: Size,
283        src1: Location,
284        src2: Location,
285        dst: Location,
286    ) -> Result<(), CompileError>;
287    fn emit_eor(
288        &mut self,
289        sz: Size,
290        src1: Location,
291        src2: Location,
292        dst: Location,
293    ) -> Result<(), CompileError>;
294
295    fn emit_bfc(
296        &mut self,
297        se: Size,
298        lsb: u32,
299        width: u32,
300        dst: Location,
301    ) -> Result<(), CompileError>;
302    fn emit_bfi(
303        &mut self,
304        se: Size,
305        src: Location,
306        lsb: u32,
307        width: u32,
308        dst: Location,
309    ) -> Result<(), CompileError>;
310
311    fn emit_udiv(
312        &mut self,
313        sz: Size,
314        src1: Location,
315        src2: Location,
316        dst: Location,
317    ) -> Result<(), CompileError>;
318    fn emit_sdiv(
319        &mut self,
320        sz: Size,
321        src1: Location,
322        src2: Location,
323        dst: Location,
324    ) -> Result<(), CompileError>;
325    /// msub : c - a*b -> dst
326    fn emit_msub(
327        &mut self,
328        sz: Size,
329        a: Location,
330        b: Location,
331        c: Location,
332        dst: Location,
333    ) -> Result<(), CompileError>;
334
335    fn emit_sxtb(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
336    fn emit_sxth(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
337    fn emit_sxtw(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
338    fn emit_uxtb(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
339    fn emit_uxth(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
340
341    fn emit_cset(&mut self, sz: Size, dst: Location, cond: Condition) -> Result<(), CompileError>;
342    fn emit_csetm(&mut self, sz: Size, dst: Location, cond: Condition) -> Result<(), CompileError>;
343    fn emit_cinc(
344        &mut self,
345        sz: Size,
346        src: Location,
347        dst: Location,
348        cond: Condition,
349    ) -> Result<(), CompileError>;
350    fn emit_clz(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
351    fn emit_rbit(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
352
353    fn emit_label(&mut self, label: Label) -> Result<(), CompileError>;
354    fn emit_load_label(&mut self, reg: GPR, label: Label) -> Result<(), CompileError>;
355    fn emit_b_label(&mut self, label: Label) -> Result<(), CompileError>;
356    fn emit_cbz_label(&mut self, sz: Size, reg: Location, label: Label)
357    -> Result<(), CompileError>;
358    fn emit_cbnz_label(
359        &mut self,
360        sz: Size,
361        reg: Location,
362        label: Label,
363    ) -> Result<(), CompileError>;
364    fn emit_cbz_label_far(
365        &mut self,
366        sz: Size,
367        reg: Location,
368        label: Label,
369    ) -> Result<(), CompileError>;
370    fn emit_tbz_label(
371        &mut self,
372        sz: Size,
373        reg: Location,
374        n: u32,
375        label: Label,
376    ) -> Result<(), CompileError>;
377    fn emit_tbnz_label(
378        &mut self,
379        sz: Size,
380        reg: Location,
381        n: u32,
382        label: Label,
383    ) -> Result<(), CompileError>;
384    fn emit_bcond_label(&mut self, condition: Condition, label: Label) -> Result<(), CompileError>;
385    fn emit_bcond_label_far(
386        &mut self,
387        condition: Condition,
388        label: Label,
389    ) -> Result<(), CompileError>;
390    fn emit_b_register(&mut self, reg: GPR) -> Result<(), CompileError>;
391    fn emit_call_label(&mut self, label: Label) -> Result<(), CompileError>;
392    fn emit_call_register(&mut self, reg: GPR) -> Result<(), CompileError>;
393    fn emit_ret(&mut self) -> Result<(), CompileError>;
394
395    fn emit_udf(&mut self, payload: u16) -> Result<(), CompileError>;
396    fn emit_dmb(&mut self) -> Result<(), CompileError>;
397    fn emit_brk(&mut self) -> Result<(), CompileError>;
398
399    fn emit_fcmp(&mut self, sz: Size, src1: Location, src2: Location) -> Result<(), CompileError>;
400    fn emit_fneg(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
401    fn emit_fsqrt(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
402
403    fn emit_fadd(
404        &mut self,
405        sz: Size,
406        src1: Location,
407        src2: Location,
408        dst: Location,
409    ) -> Result<(), CompileError>;
410    fn emit_fsub(
411        &mut self,
412        sz: Size,
413        src1: Location,
414        src2: Location,
415        dst: Location,
416    ) -> Result<(), CompileError>;
417    fn emit_fmul(
418        &mut self,
419        sz: Size,
420        src1: Location,
421        src2: Location,
422        dst: Location,
423    ) -> Result<(), CompileError>;
424    fn emit_fdiv(
425        &mut self,
426        sz: Size,
427        src1: Location,
428        src2: Location,
429        dst: Location,
430    ) -> Result<(), CompileError>;
431
432    fn emit_fmin(
433        &mut self,
434        sz: Size,
435        src1: Location,
436        src2: Location,
437        dst: Location,
438    ) -> Result<(), CompileError>;
439    fn emit_fmax(
440        &mut self,
441        sz: Size,
442        src1: Location,
443        src2: Location,
444        dst: Location,
445    ) -> Result<(), CompileError>;
446
447    fn emit_frintz(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
448    fn emit_frintn(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
449    fn emit_frintm(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
450    fn emit_frintp(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
451
452    fn emit_scvtf(
453        &mut self,
454        sz_in: Size,
455        src: Location,
456        sz_out: Size,
457        dst: Location,
458    ) -> Result<(), CompileError>;
459    fn emit_ucvtf(
460        &mut self,
461        sz_in: Size,
462        src: Location,
463        sz_out: Size,
464        dst: Location,
465    ) -> Result<(), CompileError>;
466    fn emit_fcvt(&mut self, sz_in: Size, src: Location, dst: Location) -> Result<(), CompileError>;
467    fn emit_fcvtzs(
468        &mut self,
469        sz_in: Size,
470        src: Location,
471        sz_out: Size,
472        dst: Location,
473    ) -> Result<(), CompileError>;
474    fn emit_fcvtzu(
475        &mut self,
476        sz_in: Size,
477        src: Location,
478        sz_out: Size,
479        dst: Location,
480    ) -> Result<(), CompileError>;
481
482    fn emit_fmov(
483        &mut self,
484        src_size: Size,
485        src: Location,
486        dst_size: Size,
487        dst: Location,
488    ) -> Result<(), CompileError>;
489    fn emit_cnt(&mut self, src: NEON, dst: NEON) -> Result<(), CompileError>;
490    fn emit_addv(&mut self, src: NEON, dst: NEON) -> Result<(), CompileError>;
491    fn emit_read_fpcr(&mut self, reg: GPR) -> Result<(), CompileError>;
492    fn emit_write_fpcr(&mut self, reg: GPR) -> Result<(), CompileError>;
493    fn emit_read_fpsr(&mut self, reg: GPR) -> Result<(), CompileError>;
494    fn emit_write_fpsr(&mut self, reg: GPR) -> Result<(), CompileError>;
495
496    fn arch_emit_indirect_call_with_trampoline(
497        &mut self,
498        _loc: Location,
499    ) -> Result<(), CompileError> {
500        codegen_error!("singlepass arch_emit_indirect_call_with_trampoline unimplemented")
501    }
502}
503
504impl EmitterARM64 for Assembler {
505    fn get_label(&mut self) -> DynamicLabel {
506        self.new_dynamic_label()
507    }
508
509    fn get_offset(&self) -> AssemblyOffset {
510        self.offset()
511    }
512
513    fn get_jmp_instr_size(&self) -> u8 {
514        4 // relative jump, not full 32bits capable
515    }
516
517    fn finalize_function(&mut self) {}
518
519    fn emit_str(&mut self, sz: Size, reg: Location, addr: Location) -> Result<(), CompileError> {
520        match (sz, reg, addr) {
521            (Size::S64, Location::GPR(reg), Location::Memory(addr, disp)) => {
522                let disp = disp as u32;
523                assert!(disp.is_multiple_of(8) && (disp < 0x8000));
524                dynasm!(self ; str X(reg), [X(addr), disp]);
525            }
526            (Size::S32, Location::GPR(reg), Location::Memory(addr, disp)) => {
527                let disp = disp as u32;
528                assert!(disp.is_multiple_of(4) && (disp < 0x4000));
529                dynasm!(self ; str W(reg), [X(addr), disp]);
530            }
531            (Size::S16, Location::GPR(reg), Location::Memory(addr, disp)) => {
532                let disp = disp as u32;
533                assert!(disp.is_multiple_of(2) && (disp < 0x2000));
534                dynasm!(self ; strh W(reg), [X(addr), disp]);
535            }
536            (Size::S8, Location::GPR(reg), Location::Memory(addr, disp)) => {
537                let disp = disp as u32;
538                assert!(disp < 0x1000);
539                dynasm!(self ; strb W(reg), [X(addr), disp]);
540            }
541            (Size::S64, Location::SIMD(reg), Location::Memory(addr, disp)) => {
542                let disp = disp as u32;
543                assert!(disp.is_multiple_of(8) && (disp < 0x8000));
544                dynasm!(self ; str D(reg), [X(addr), disp]);
545            }
546            (Size::S32, Location::SIMD(reg), Location::Memory(addr, disp)) => {
547                let disp = disp as u32;
548                assert!(disp.is_multiple_of(4) && (disp < 0x4000));
549                dynasm!(self ; str S(reg), [X(addr), disp]);
550            }
551            (Size::S64, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
552                assert!(offs == 0);
553                let mult = mult as u32;
554                match mult {
555                    0 => dynasm!(self ; str X(reg), [X(addr)]),
556                    1 => dynasm!(self ; str X(reg), [X(addr), X(r2)]),
557                    _ => dynasm!(self ; str X(reg), [X(addr), X(r2), LSL mult]),
558                };
559            }
560            (Size::S32, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
561                assert!(offs == 0);
562                let mult = mult as u32;
563                match mult {
564                    0 => dynasm!(self ; str W(reg), [X(addr)]),
565                    1 => dynasm!(self ; str W(reg), [X(addr), X(r2)]),
566                    _ => dynasm!(self ; str W(reg), [X(addr), X(r2), LSL mult]),
567                };
568            }
569            _ => codegen_error!("singlepass can't emit STR {:?}, {:?}, {:?}", sz, reg, addr),
570        }
571        Ok(())
572    }
573    fn emit_ldr(&mut self, sz: Size, reg: Location, addr: Location) -> Result<(), CompileError> {
574        match (sz, reg, addr) {
575            (Size::S64, Location::GPR(reg), Location::Memory(addr, disp)) => {
576                assert!((disp & 0x7) == 0 && (disp < 0x8000));
577                let disp = disp as u32;
578                dynasm!(self ; ldr X(reg), [X(addr), disp]);
579            }
580            (Size::S32, Location::GPR(reg), Location::Memory(addr, disp)) => {
581                assert!((disp & 0x3) == 0 && (disp < 0x4000));
582                let disp = disp as u32;
583                dynasm!(self ; ldr W(reg), [X(addr), disp]);
584            }
585            (Size::S16, Location::GPR(reg), Location::Memory(addr, disp)) => {
586                assert!((disp & 0x1 == 0) && (disp < 0x2000));
587                let disp = disp as u32;
588                dynasm!(self ; ldrh W(reg), [X(addr), disp]);
589            }
590            (Size::S8, Location::GPR(reg), Location::Memory(addr, disp)) => {
591                assert!(disp < 0x1000);
592                let disp = disp as u32;
593                dynasm!(self ; ldrb W(reg), [X(addr), disp]);
594            }
595            (Size::S64, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
596                assert!(offs == 0);
597                let mult = mult as u32;
598                match mult {
599                    0 => dynasm!(self ; ldr X(reg), [X(addr)]),
600                    1 => dynasm!(self ; ldr X(reg), [X(addr), X(r2)]),
601                    _ => dynasm!(self ; ldr X(reg), [X(addr), X(r2), LSL mult]),
602                };
603            }
604            (Size::S32, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
605                assert!(offs == 0);
606                let mult = mult as u32;
607                match mult {
608                    0 => dynasm!(self ; ldr W(reg), [X(addr)]),
609                    1 => dynasm!(self ; ldr W(reg), [X(addr), X(r2)]),
610                    _ => dynasm!(self ; ldr W(reg), [X(addr), X(r2), LSL mult]),
611                };
612            }
613            (Size::S64, Location::SIMD(reg), Location::Memory(addr, disp)) => {
614                let disp = disp as u32;
615                assert!((disp & 0x7) == 0 && (disp < 0x8000));
616                dynasm!(self ; ldr D(reg), [X(addr), disp]);
617            }
618            (Size::S32, Location::SIMD(reg), Location::Memory(addr, disp)) => {
619                let disp = disp as u32;
620                assert!((disp & 0x3) == 0 && (disp < 0x4000));
621                dynasm!(self ; ldr S(reg), [X(addr), disp]);
622            }
623            (Size::S64, Location::SIMD(reg), Location::Memory2(addr, r2, mult, offs)) => {
624                assert!(offs == 0);
625                let mult = mult as u32;
626                match mult {
627                    0 => dynasm!(self ; ldr D(reg), [X(addr)]),
628                    1 => dynasm!(self ; ldr D(reg), [X(addr), X(r2)]),
629                    _ => dynasm!(self ; ldr D(reg), [X(addr), X(r2), LSL mult]),
630                };
631            }
632            (Size::S32, Location::SIMD(reg), Location::Memory2(addr, r2, mult, offs)) => {
633                assert!(offs == 0);
634                let mult = mult as u32;
635                match mult {
636                    0 => dynasm!(self ; ldr S(reg), [X(addr)]),
637                    1 => dynasm!(self ; ldr S(reg), [X(addr), X(r2)]),
638                    _ => dynasm!(self ; ldr S(reg), [X(addr), X(r2), LSL mult]),
639                };
640            }
641            _ => codegen_error!("singlepass can't emit LDR {:?}, {:?}, {:?}", sz, reg, addr),
642        }
643        Ok(())
644    }
645    fn emit_stur(
646        &mut self,
647        sz: Size,
648        reg: Location,
649        addr: GPR,
650        offset: i32,
651    ) -> Result<(), CompileError> {
652        assert!((-255..=255).contains(&offset));
653        match (sz, reg) {
654            (Size::S64, Location::GPR(reg)) => {
655                dynasm!(self ; stur X(reg), [X(addr), offset]);
656            }
657            (Size::S32, Location::GPR(reg)) => {
658                dynasm!(self ; stur W(reg), [X(addr), offset]);
659            }
660            (Size::S64, Location::SIMD(reg)) => {
661                dynasm!(self ; stur D(reg), [X(addr), offset]);
662            }
663            (Size::S32, Location::SIMD(reg)) => {
664                dynasm!(self ; stur S(reg), [X(addr), offset]);
665            }
666            _ => codegen_error!(
667                "singlepass can't emit STUR {:?}, {:?}, {:?}, {:?}",
668                sz,
669                reg,
670                addr,
671                offset
672            ),
673        }
674        Ok(())
675    }
676    fn emit_ldur(
677        &mut self,
678        sz: Size,
679        reg: Location,
680        addr: GPR,
681        offset: i32,
682    ) -> Result<(), CompileError> {
683        assert!((-255..=255).contains(&offset));
684        match (sz, reg) {
685            (Size::S64, Location::GPR(reg)) => {
686                dynasm!(self ; ldur X(reg), [X(addr), offset]);
687            }
688            (Size::S32, Location::GPR(reg)) => {
689                dynasm!(self ; ldur W(reg), [X(addr), offset]);
690            }
691            (Size::S64, Location::SIMD(reg)) => {
692                dynasm!(self ; ldur D(reg), [X(addr), offset]);
693            }
694            (Size::S32, Location::SIMD(reg)) => {
695                dynasm!(self ; ldur S(reg), [X(addr), offset]);
696            }
697            _ => codegen_error!(
698                "singlepass can't emit LDUR {:?}, {:?}, {:?}, {:?}",
699                sz,
700                reg,
701                addr,
702                offset
703            ),
704        }
705        Ok(())
706    }
707
708    fn emit_strdb(
709        &mut self,
710        sz: Size,
711        reg: Location,
712        addr: GPR,
713        offset: u32,
714    ) -> Result<(), CompileError> {
715        assert!(offset <= 255);
716        match (sz, reg) {
717            (Size::S64, Location::GPR(reg)) => {
718                dynasm!(self ; str X(reg), [X(addr), -(offset as i32)]!);
719            }
720            (Size::S64, Location::SIMD(reg)) => {
721                dynasm!(self ; str D(reg), [X(addr), -(offset as i32)]!);
722            }
723            _ => codegen_error!("singlepass can't emit STRDB"),
724        }
725        Ok(())
726    }
727    fn emit_stria(
728        &mut self,
729        sz: Size,
730        reg: Location,
731        addr: GPR,
732        offset: u32,
733    ) -> Result<(), CompileError> {
734        assert!(offset <= 255);
735        match (sz, reg) {
736            (Size::S64, Location::GPR(reg)) => {
737                dynasm!(self ; str X(reg), [X(addr)], offset as i32);
738            }
739            (Size::S64, Location::SIMD(reg)) => {
740                dynasm!(self ; str D(reg), [X(addr)], offset as i32);
741            }
742            _ => codegen_error!("singlepass can't emit STRIA"),
743        }
744        Ok(())
745    }
746    fn emit_ldria(
747        &mut self,
748        sz: Size,
749        reg: Location,
750        addr: GPR,
751        offset: u32,
752    ) -> Result<(), CompileError> {
753        assert!(offset <= 255);
754        match (sz, reg) {
755            (Size::S64, Location::GPR(reg)) => {
756                dynasm!(self ; ldr X(reg), [X(addr)], offset as _);
757            }
758            (Size::S64, Location::SIMD(reg)) => {
759                dynasm!(self ; ldr D(reg), [X(addr)], offset as _);
760            }
761            _ => codegen_error!("singlepass can't emit LDRIA"),
762        }
763        Ok(())
764    }
765
766    fn emit_stpdb(
767        &mut self,
768        sz: Size,
769        reg1: Location,
770        reg2: Location,
771        addr: GPR,
772        offset: u32,
773    ) -> Result<(), CompileError> {
774        assert!(offset <= 255);
775        match (sz, reg1, reg2) {
776            (Size::S64, Location::GPR(reg1), Location::GPR(reg2)) => {
777                dynasm!(self ; stp X(reg1), X(reg2), [X(addr), -(offset as i32)]!);
778            }
779            _ => codegen_error!("singlepass can't emit STPDB"),
780        }
781        Ok(())
782    }
783    fn emit_ldpia(
784        &mut self,
785        sz: Size,
786        reg1: Location,
787        reg2: Location,
788        addr: GPR,
789        offset: u32,
790    ) -> Result<(), CompileError> {
791        assert!(offset <= 255);
792        match (sz, reg1, reg2) {
793            (Size::S64, Location::GPR(reg1), Location::GPR(reg2)) => {
794                dynasm!(self ; ldp X(reg1), X(reg2), [X(addr)], offset as _);
795            }
796            _ => codegen_error!("singlepass can't emit LDPIA"),
797        }
798        Ok(())
799    }
800
801    fn emit_ldrb(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
802        match (reg, dst) {
803            (Location::GPR(reg), Location::Memory(addr, offset)) => {
804                let offset = offset as u32;
805                assert!(offset < 0x1000);
806                dynasm!(self ; ldrb W(reg), [X(addr), offset]);
807            }
808            (Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
809                assert!(offs == 0);
810                let mult = mult as u32;
811                match mult {
812                    0 => dynasm!(self ; ldrb W(reg), [X(addr)]),
813                    1 => dynasm!(self ; ldrb W(reg), [X(addr), X(r2)]),
814                    _ => dynasm!(self ; ldrb W(reg), [X(addr), X(r2), LSL mult]),
815                };
816            }
817            _ => codegen_error!("singlepass can't emit LDRB {:?}, {:?}", reg, dst),
818        }
819        Ok(())
820    }
821    fn emit_ldrh(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
822        match (reg, dst) {
823            (Location::GPR(reg), Location::Memory(addr, offset)) => {
824                let offset = offset as u32;
825                assert!((offset & 1 == 0) && (offset < 0x2000));
826                dynasm!(self ; ldrh W(reg), [X(addr), offset]);
827            }
828            (Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
829                assert!(offs == 0);
830                let mult = mult as u32;
831                match mult {
832                    0 => dynasm!(self ; ldrh W(reg), [X(addr)]),
833                    1 => dynasm!(self ; ldrh W(reg), [X(addr), X(r2)]),
834                    _ => dynasm!(self ; ldrh W(reg), [X(addr), X(r2), LSL mult]),
835                };
836            }
837            _ => codegen_error!("singlepass can't emit LDRH {:?}, {:?}", reg, dst),
838        }
839        Ok(())
840    }
841    fn emit_ldrsb(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
842        match (sz, reg, dst) {
843            (Size::S64, Location::GPR(reg), Location::Memory(addr, offset)) => {
844                let offset = offset as u32;
845                assert!(offset < 0x1000);
846                dynasm!(self ; ldrsb X(reg), [X(addr), offset]);
847            }
848            (Size::S32, Location::GPR(reg), Location::Memory(addr, offset)) => {
849                let offset = offset as u32;
850                assert!(offset < 0x1000);
851                dynasm!(self ; ldrsb W(reg), [X(addr), offset]);
852            }
853            (Size::S64, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
854                assert!(offs == 0);
855                let mult = mult as u32;
856                match mult {
857                    0 => dynasm!(self ; ldrsb X(reg), [X(addr)]),
858                    1 => dynasm!(self ; ldrsb X(reg), [X(addr), X(r2)]),
859                    _ => dynasm!(self ; ldrsb X(reg), [X(addr), X(r2), LSL mult]),
860                };
861            }
862            (Size::S32, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
863                assert!(offs == 0);
864                let mult = mult as u32;
865                match mult {
866                    0 => dynasm!(self ; ldrsb W(reg), [X(addr)]),
867                    1 => dynasm!(self ; ldrsb W(reg), [X(addr), X(r2)]),
868                    _ => dynasm!(self ; ldrsb W(reg), [X(addr), X(r2), LSL mult]),
869                };
870            }
871            _ => codegen_error!("singlepass can't emit LDRSB {:?}, {:?}, {:?}", sz, reg, dst),
872        }
873        Ok(())
874    }
875    fn emit_ldrsh(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
876        match (sz, reg, dst) {
877            (Size::S64, Location::GPR(reg), Location::Memory(addr, offset)) => {
878                let offset = offset as u32;
879                assert!((offset & 1 == 0) && (offset < 0x2000));
880                dynasm!(self ; ldrsh X(reg), [X(addr), offset]);
881            }
882            (Size::S32, Location::GPR(reg), Location::Memory(addr, offset)) => {
883                let offset = offset as u32;
884                assert!((offset & 1 == 0) && (offset < 0x2000));
885                dynasm!(self ; ldrsh W(reg), [X(addr), offset]);
886            }
887            (Size::S64, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
888                assert!(offs == 0);
889                let mult = mult as u32;
890                match mult {
891                    0 => dynasm!(self ; ldrsh X(reg), [X(addr)]),
892                    1 => dynasm!(self ; ldrsh X(reg), [X(addr), X(r2)]),
893                    _ => dynasm!(self ; ldrsh X(reg), [X(addr), X(r2), LSL mult]),
894                };
895            }
896            (Size::S32, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
897                assert!(offs == 0);
898                let mult = mult as u32;
899                match mult {
900                    0 => dynasm!(self ; ldrsh W(reg), [X(addr)]),
901                    1 => dynasm!(self ; ldrsh W(reg), [X(addr), X(r2)]),
902                    _ => dynasm!(self ; ldrsh W(reg), [X(addr), X(r2), LSL mult]),
903                };
904            }
905            _ => codegen_error!("singlepass can't emit LDRSH {:?}, {:?}, {:?}", sz, reg, dst),
906        }
907        Ok(())
908    }
909    fn emit_ldrsw(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
910        match (sz, reg, dst) {
911            (Size::S64, Location::GPR(reg), Location::Memory(addr, offset)) => {
912                let offset = offset as u32;
913                assert!((offset & 3 == 0) && (offset < 0x4000));
914                dynasm!(self ; ldrsw X(reg), [X(addr), offset]);
915            }
916            (Size::S64, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
917                assert!(offs == 0);
918                let mult = mult as u32;
919                match mult {
920                    0 => dynasm!(self ; ldrsw X(reg), [X(addr)]),
921                    1 => dynasm!(self ; ldrsw X(reg), [X(addr), X(r2)]),
922                    _ => dynasm!(self ; ldrsw X(reg), [X(addr), X(r2), LSL mult]),
923                };
924            }
925            _ => codegen_error!("singlepass can't emit LDRSW {:?}, {:?}, {:?}", sz, reg, dst),
926        }
927        Ok(())
928    }
929    fn emit_strb(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
930        match (reg, dst) {
931            (Location::GPR(reg), Location::Memory(addr, offset)) => {
932                let offset = offset as u32;
933                assert!(offset < 0x1000);
934                dynasm!(self ; strb W(reg), [X(addr), offset]);
935            }
936            (Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
937                assert!(offs == 0);
938                let mult = mult as u32;
939                match mult {
940                    0 => dynasm!(self ; strb W(reg), [X(addr)]),
941                    1 => dynasm!(self ; strb W(reg), [X(addr), X(r2)]),
942                    _ => dynasm!(self ; strb W(reg), [X(addr), X(r2), LSL mult]),
943                };
944            }
945            _ => codegen_error!("singlepass can't emit STRB {:?}, {:?}", reg, dst),
946        }
947        Ok(())
948    }
949    fn emit_strh(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
950        match (reg, dst) {
951            (Location::GPR(reg), Location::Memory(addr, offset)) => {
952                let offset = offset as u32;
953                assert!((offset & 1 == 0) && (offset < 0x2000));
954                dynasm!(self ; strh W(reg), [X(addr), offset]);
955            }
956            (Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
957                assert!(offs == 0);
958                let mult = mult as u32;
959                match mult {
960                    0 => dynasm!(self ; strh W(reg), [X(addr)]),
961                    1 => dynasm!(self ; strh W(reg), [X(addr), X(r2)]),
962                    _ => dynasm!(self ; strh W(reg), [X(addr), X(r2), LSL mult]),
963                };
964            }
965            _ => codegen_error!("singlepass can't emit STRH {:?}, {:?}", reg, dst),
966        }
967        Ok(())
968    }
969
970    fn emit_ldaxr(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
971        match (sz, reg, dst) {
972            (Size::S32, Location::GPR(reg), Location::GPR(dst)) => {
973                dynasm!(self ; ldaxr W(reg), [X(dst)]);
974            }
975            (Size::S64, Location::GPR(reg), Location::GPR(dst)) => {
976                dynasm!(self ; ldaxr X(reg), [X(dst)]);
977            }
978            _ => codegen_error!("singlepass can't emit LDAXR {:?}, {:?}", reg, dst),
979        }
980        Ok(())
981    }
982    fn emit_ldaxrb(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
983        match (reg, dst) {
984            (Location::GPR(reg), Location::GPR(dst)) => {
985                dynasm!(self ; ldaxrb W(reg), [X(dst)]);
986            }
987            _ => codegen_error!("singlepass can't emit LDAXRB {:?}, {:?}", reg, dst),
988        }
989        Ok(())
990    }
991    fn emit_ldaxrh(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
992        match (reg, dst) {
993            (Location::GPR(reg), Location::GPR(dst)) => {
994                dynasm!(self ; ldaxrh W(reg), [X(dst)]);
995            }
996            _ => codegen_error!("singlepass can't emit LDAXRH {:?}, {:?}", reg, dst),
997        }
998        Ok(())
999    }
1000    fn emit_stlxr(
1001        &mut self,
1002        sz: Size,
1003        status: Location,
1004        reg: Location,
1005        dst: Location,
1006    ) -> Result<(), CompileError> {
1007        match (sz, status, reg, dst) {
1008            (Size::S32, Location::GPR(status), Location::GPR(reg), Location::GPR(dst)) => {
1009                dynasm!(self ; stlxr W(status), W(reg), [X(dst)]);
1010            }
1011            (Size::S64, Location::GPR(status), Location::GPR(reg), Location::GPR(dst)) => {
1012                dynasm!(self ; stlxr W(status), X(reg), [X(dst)]);
1013            }
1014            _ => codegen_error!("singlepass can't emit STLXR {:?}, {:?}", reg, dst),
1015        }
1016        Ok(())
1017    }
1018    fn emit_stlxrb(
1019        &mut self,
1020        _sz: Size,
1021        status: Location,
1022        reg: Location,
1023        dst: Location,
1024    ) -> Result<(), CompileError> {
1025        match (status, reg, dst) {
1026            (Location::GPR(status), Location::GPR(reg), Location::GPR(dst)) => {
1027                dynasm!(self ; stlxrb W(status), W(reg), [X(dst)]);
1028            }
1029            _ => codegen_error!("singlepass can't emit STLXRB {:?}, {:?}", reg, dst),
1030        }
1031        Ok(())
1032    }
1033    fn emit_stlxrh(
1034        &mut self,
1035        _sz: Size,
1036        status: Location,
1037        reg: Location,
1038        dst: Location,
1039    ) -> Result<(), CompileError> {
1040        match (status, reg, dst) {
1041            (Location::GPR(status), Location::GPR(reg), Location::GPR(dst)) => {
1042                dynasm!(self ; stlxrh W(status), W(reg), [X(dst)]);
1043            }
1044            _ => codegen_error!("singlepass can't emit STLXRH {:?}, {:?}", reg, dst),
1045        }
1046        Ok(())
1047    }
1048
1049    fn emit_mov(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1050        match (sz, src, dst) {
1051            (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1052                dynasm!(self ; mov X(dst), X(src));
1053            }
1054            (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1055                dynasm!(self ; mov W(dst), W(src));
1056            }
1057            (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
1058                dynasm!(self ; mov V(dst).D[0], V(src).D[0]);
1059            }
1060            (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
1061                dynasm!(self ; mov V(dst).S[0], V(src).S[0]);
1062            }
1063            (Size::S64, Location::GPR(src), Location::SIMD(dst)) => {
1064                dynasm!(self ; mov V(dst).D[0], X(src));
1065            }
1066            (Size::S32, Location::GPR(src), Location::SIMD(dst)) => {
1067                dynasm!(self ; mov V(dst).S[0], W(src));
1068            }
1069            (Size::S64, Location::SIMD(src), Location::GPR(dst)) => {
1070                dynasm!(self ; mov X(dst), V(src).D[0]);
1071            }
1072            (Size::S32, Location::SIMD(src), Location::GPR(dst)) => {
1073                dynasm!(self ; mov W(dst), V(src).S[0]);
1074            }
1075            (Size::S32, Location::Imm32(val), Location::GPR(dst)) => {
1076                if val < 0x1000 || encode_logical_immediate_32bit(val as _).is_some() {
1077                    dynasm!(self ; mov W(dst), val);
1078                } else {
1079                    dynasm!(self
1080                        ; movz W(dst), val
1081                        ; movk W(dst), val, lsl #16);
1082                }
1083            }
1084            (Size::S64, Location::Imm32(val), Location::GPR(dst)) => {
1085                if val < 0x1000 {
1086                    dynasm!(self ; mov W(dst), val);
1087                } else if encode_logical_immediate_64bit(val as _).is_some() {
1088                    dynasm!(self ; mov X(dst), val as _);
1089                } else {
1090                    dynasm!(self
1091                        ; movz X(dst), val
1092                        ; movk X(dst), val, lsl #16);
1093                }
1094            }
1095            (Size::S64, Location::Imm64(val), Location::GPR(dst)) => {
1096                if val < 0x1000 || encode_logical_immediate_64bit(val as _).is_some() {
1097                    dynasm!(self ; mov X(dst), val as _);
1098                } else {
1099                    dynasm!(self
1100                        ; movz X(dst), val as _
1101                        ; movk X(dst), val as _, lsl #16
1102                        ; movk X(dst), val as _, lsl #32
1103                        ; movk X(dst), val as _, lsl #48);
1104                }
1105            }
1106            _ => codegen_error!("singlepass can't emit MOV {:?}, {:?}, {:?}", sz, src, dst),
1107        }
1108        Ok(())
1109    }
1110
1111    fn emit_movn(&mut self, sz: Size, reg: Location, val: u32) -> Result<(), CompileError> {
1112        match (sz, reg) {
1113            (Size::S32, Location::GPR(reg)) => {
1114                dynasm!(self ; movn W(reg), val);
1115            }
1116            (Size::S64, Location::GPR(reg)) => {
1117                dynasm!(self ; movn X(reg), val);
1118            }
1119            _ => codegen_error!("singlepass can't emit MOVN"),
1120        }
1121        Ok(())
1122    }
1123    fn emit_movz(&mut self, reg: Location, val: u32) -> Result<(), CompileError> {
1124        match reg {
1125            Location::GPR(reg) => {
1126                dynasm!(self ; movz W(reg), val);
1127            }
1128            _ => codegen_error!("singlepass can't emit MOVZ"),
1129        }
1130        Ok(())
1131    }
1132    fn emit_movk(&mut self, reg: Location, val: u32, shift: u32) -> Result<(), CompileError> {
1133        match reg {
1134            Location::GPR(reg) => {
1135                dynasm!(self ; movk X(reg), val, LSL shift);
1136            }
1137            _ => codegen_error!("singlepass can't emit MOVK"),
1138        }
1139        Ok(())
1140    }
1141
1142    fn emit_mov_imm(&mut self, dst: Location, val: u64) -> Result<(), CompileError> {
1143        match dst {
1144            Location::GPR(dst) => {
1145                let offset = val.trailing_zeros() & 48;
1146                let masked = 0xffff & (val >> offset);
1147                if (masked << offset) == val {
1148                    dynasm!(self ; movz X(dst), masked as u32, LSL offset);
1149                } else if val >> 16 == 0xffff_ffff_ffff {
1150                    let val: u16 = !((val & 0xffff) as u16);
1151                    dynasm!(self ; movn X(dst), val as u32);
1152                } else if val >> 16 == 0xffff {
1153                    let val: u16 = !((val & 0xffff) as u16);
1154                    dynasm!(self ; movn W(dst), val as u32);
1155                } else {
1156                    dynasm!(self ; movz W(dst), (val&0xffff) as u32);
1157                    let val = val >> 16;
1158                    if val != 0 {
1159                        dynasm!(self ; movk X(dst), (val&0xffff) as u32, LSL 16);
1160                        let val = val >> 16;
1161                        if val != 0 {
1162                            dynasm!(self ; movk X(dst), (val&0xffff) as u32, LSL 32);
1163                            let val = val >> 16;
1164                            if val != 0 {
1165                                dynasm!(self ; movk X(dst), (val&0xffff) as u32, LSL 48);
1166                            }
1167                        }
1168                    }
1169                }
1170            }
1171            _ => codegen_error!("singlepass can't emit MOVW {:?}", dst),
1172        }
1173        Ok(())
1174    }
1175
1176    fn emit_add(
1177        &mut self,
1178        sz: Size,
1179        src1: Location,
1180        src2: Location,
1181        dst: Location,
1182    ) -> Result<(), CompileError> {
1183        match (sz, src1, src2, dst) {
1184            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1185                dynasm!(self ; add XSP(dst), XSP(src1), X(src2), UXTX);
1186            }
1187            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1188                dynasm!(self ; add WSP(dst), WSP(src1), W(src2), UXTX);
1189            }
1190            (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1191            | (Size::S64, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1192                dynasm!(self ; add XSP(dst), XSP(src1), imm as _);
1193            }
1194            (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
1195            | (Size::S64, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
1196                if imm >= 0x1000 {
1197                    unreachable!();
1198                }
1199                dynasm!(self ; add XSP(dst), XSP(src1), imm);
1200            }
1201            (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst))
1202            | (Size::S64, Location::Imm64(imm), Location::GPR(src1), Location::GPR(dst)) => {
1203                if imm >= 0x1000 {
1204                    unreachable!();
1205                }
1206                let imm = imm as u32;
1207                dynasm!(self ; add XSP(dst), XSP(src1), imm);
1208            }
1209            (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1210            | (Size::S32, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1211                dynasm!(self ; add WSP(dst), WSP(src1), imm as u32);
1212            }
1213            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
1214            | (Size::S32, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
1215                if imm >= 0x1000 {
1216                    unreachable!();
1217                }
1218                dynasm!(self ; add WSP(dst), WSP(src1), imm);
1219            }
1220            _ => codegen_error!(
1221                "singlepass can't emit ADD {:?} {:?} {:?} {:?}",
1222                sz,
1223                src1,
1224                src2,
1225                dst
1226            ),
1227        }
1228        Ok(())
1229    }
1230    fn emit_sub(
1231        &mut self,
1232        sz: Size,
1233        src1: Location,
1234        src2: Location,
1235        dst: Location,
1236    ) -> Result<(), CompileError> {
1237        match (sz, src1, src2, dst) {
1238            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1239                dynasm!(self; sub XSP(dst), XSP(src1), X(src2), UXTX 0);
1240            }
1241            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1242                dynasm!(self ; sub WSP(dst), WSP(src1), W(src2), UXTX 0);
1243            }
1244            (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1245                dynasm!(self ; sub XSP(dst), XSP(src1), imm as u32);
1246            }
1247            (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1248                dynasm!(self ; sub WSP(dst), WSP(src1), imm as u32);
1249            }
1250            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1251                if imm >= 0x1000 {
1252                    unreachable!();
1253                }
1254                dynasm!(self ; sub WSP(dst), WSP(src1), imm);
1255            }
1256            (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1257                if imm >= 0x1000 {
1258                    unreachable!();
1259                }
1260                dynasm!(self ; sub XSP(dst), XSP(src1), imm);
1261            }
1262            (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
1263                if imm >= 0x1000 {
1264                    unreachable!();
1265                }
1266                dynasm!(self ; sub XSP(dst), XSP(src1), imm as u32);
1267            }
1268            _ => codegen_error!(
1269                "singlepass can't emit SUB {:?} {:?} {:?} {:?}",
1270                sz,
1271                src1,
1272                src2,
1273                dst
1274            ),
1275        }
1276        Ok(())
1277    }
1278    fn emit_mul(
1279        &mut self,
1280        sz: Size,
1281        src1: Location,
1282        src2: Location,
1283        dst: Location,
1284    ) -> Result<(), CompileError> {
1285        match (sz, src1, src2, dst) {
1286            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1287                dynasm!(self ; mul X(dst), X(src1), X(src2));
1288            }
1289            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1290                dynasm!(self ; mul W(dst), W(src1), W(src2));
1291            }
1292            _ => codegen_error!(
1293                "singlepass can't emit MUL {:?} {:?} {:?} {:?}",
1294                sz,
1295                src1,
1296                src2,
1297                dst
1298            ),
1299        }
1300        Ok(())
1301    }
1302    fn emit_adds(
1303        &mut self,
1304        sz: Size,
1305        src1: Location,
1306        src2: Location,
1307        dst: Location,
1308    ) -> Result<(), CompileError> {
1309        match (sz, src1, src2, dst) {
1310            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1311                dynasm!(self ; adds X(dst), X(src1), X(src2));
1312            }
1313            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1314                dynasm!(self ; adds W(dst), W(src1), W(src2));
1315            }
1316            (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1317            | (Size::S64, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1318                dynasm!(self ; adds X(dst), XSP(src1), imm as u32);
1319            }
1320            (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
1321            | (Size::S64, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
1322                if imm >= 0x1000 {
1323                    codegen_error!("singlepass ADD.S with imm too large {}", imm);
1324                }
1325                dynasm!(self ; adds X(dst), XSP(src1), imm);
1326            }
1327            (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1328            | (Size::S32, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1329                dynasm!(self ; adds W(dst), WSP(src1), imm as u32);
1330            }
1331            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
1332            | (Size::S32, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
1333                if imm >= 0x1000 {
1334                    codegen_error!("singlepass ADD.S with imm too large {}", imm);
1335                }
1336                dynasm!(self ; adds W(dst), WSP(src1), imm);
1337            }
1338            _ => codegen_error!(
1339                "singlepass can't emit ADD.S {:?} {:?} {:?} {:?}",
1340                sz,
1341                src1,
1342                src2,
1343                dst
1344            ),
1345        }
1346        Ok(())
1347    }
1348    fn emit_subs(
1349        &mut self,
1350        sz: Size,
1351        src1: Location,
1352        src2: Location,
1353        dst: Location,
1354    ) -> Result<(), CompileError> {
1355        match (sz, src1, src2, dst) {
1356            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1357                dynasm!(self ; subs X(dst), X(src1), X(src2));
1358            }
1359            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1360                dynasm!(self ; subs W(dst), W(src1), W(src2));
1361            }
1362            (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1363                dynasm!(self ; subs X(dst), XSP(src1), imm as u32);
1364            }
1365            (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1366                dynasm!(self ; subs W(dst), WSP(src1), imm as u32);
1367            }
1368            _ => codegen_error!(
1369                "singlepass can't emit SUB.S {:?} {:?} {:?} {:?}",
1370                sz,
1371                src1,
1372                src2,
1373                dst
1374            ),
1375        }
1376        Ok(())
1377    }
1378    fn emit_add_lsl(
1379        &mut self,
1380        sz: Size,
1381        src1: Location,
1382        src2: Location,
1383        lsl: u32,
1384        dst: Location,
1385    ) -> Result<(), CompileError> {
1386        match (sz, src1, src2, dst) {
1387            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1388                dynasm!(self ; add X(dst), X(src1), X(src2), LSL lsl);
1389            }
1390            _ => codegen_error!(
1391                "singlepass can't emit LSL {:?} {:?} {:?} {:?} LSL {:?}",
1392                sz,
1393                src1,
1394                src2,
1395                dst,
1396                lsl
1397            ),
1398        }
1399        Ok(())
1400    }
1401
1402    /// Emit a CMP instruction that compares `left` against `right`.
1403    ///
1404    /// Note: callers sometimes pass operands in the opposite order compared
1405    /// to other binary operators. This function performs the comparison as
1406    /// provided (i.e. it emits `cmp left, right` semantics).
1407    fn emit_cmp(&mut self, sz: Size, left: Location, right: Location) -> Result<(), CompileError> {
1408        match (sz, left, right) {
1409            (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1410                dynasm!(self ; cmp X(dst), X(src));
1411            }
1412            (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1413                dynasm!(self ; cmp W(dst), W(src));
1414            }
1415            (Size::S64, Location::Imm8(imm), Location::GPR(dst)) => {
1416                dynasm!(self ; cmp XSP(dst), imm as u32);
1417            }
1418            (Size::S64, Location::Imm32(imm), Location::GPR(dst)) => {
1419                if imm >= 0x1000 {
1420                    codegen_error!("singlepass CMP with imm too large {}", imm);
1421                }
1422                dynasm!(self ; cmp XSP(dst), imm as u32);
1423            }
1424            (Size::S64, Location::Imm64(imm), Location::GPR(dst)) => {
1425                if imm >= 0x1000 {
1426                    codegen_error!("singlepass CMP with imm too large {}", imm);
1427                }
1428                dynasm!(self ; cmp XSP(dst), imm as u32);
1429            }
1430            (Size::S32, Location::Imm8(imm), Location::GPR(dst)) => {
1431                dynasm!(self ; cmp WSP(dst), imm as u32);
1432            }
1433            (Size::S32, Location::Imm32(imm), Location::GPR(dst)) => {
1434                if imm >= 0x1000 {
1435                    codegen_error!("singlepass CMP with imm too large {}", imm);
1436                }
1437                dynasm!(self ; cmp WSP(dst), imm as u32);
1438            }
1439            _ => codegen_error!("singlepass can't emit CMP {:?} {:?} {:?}", sz, left, right),
1440        }
1441        Ok(())
1442    }
1443
1444    fn emit_tst(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1445        match (sz, src, dst) {
1446            (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1447                dynasm!(self ; tst X(dst), X(src));
1448            }
1449            (Size::S64, Location::Imm32(imm), Location::GPR(dst)) => {
1450                if encode_logical_immediate_64bit(imm as u64).is_none() {
1451                    codegen_error!("singlepass TST with incompatible imm {}", imm);
1452                }
1453                dynasm!(self ; tst X(dst), imm as u64);
1454            }
1455            (Size::S64, Location::Imm64(imm), Location::GPR(dst)) => {
1456                if encode_logical_immediate_64bit(imm as u64).is_none() {
1457                    codegen_error!("singlepass TST with incompatible imm {}", imm);
1458                }
1459                dynasm!(self ; tst X(dst), imm as u64);
1460            }
1461            (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1462                dynasm!(self ; tst W(dst), W(src));
1463            }
1464            (Size::S32, Location::Imm32(imm), Location::GPR(dst)) => {
1465                if encode_logical_immediate_64bit(imm as u64).is_none() {
1466                    codegen_error!("singlepass TST with incompatible imm {}", imm);
1467                }
1468                dynasm!(self ; tst W(dst), imm);
1469            }
1470            _ => codegen_error!("singlepass can't emit TST"),
1471        }
1472        Ok(())
1473    }
1474
1475    fn emit_lsl(
1476        &mut self,
1477        sz: Size,
1478        src1: Location,
1479        src2: Location,
1480        dst: Location,
1481    ) -> Result<(), CompileError> {
1482        match (sz, src1, src2, dst) {
1483            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1484                dynasm!(self ; lsl X(dst), X(src1), X(src2));
1485            }
1486            (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1487                if imm > 63 {
1488                    codegen_error!("singlepass LSL with incompatible imm {}", imm);
1489                }
1490                let imm = imm as u32;
1491
1492                dynasm!(self ; lsl X(dst), X(src1), imm);
1493            }
1494            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1495                dynasm!(self ; lsl W(dst), W(src1), W(src2));
1496            }
1497            (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1498                if imm > 63 {
1499                    codegen_error!("singlepass LSL with incompatible imm {}", imm);
1500                }
1501                dynasm!(self ; lsl X(dst), X(src1), imm as u32);
1502            }
1503            (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
1504                if imm > 63 {
1505                    codegen_error!("singlepass LSL with incompatible imm {}", imm);
1506                }
1507                dynasm!(self ; lsl X(dst), X(src1), imm as u32);
1508            }
1509            (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1510                if imm > 31 {
1511                    codegen_error!("singlepass LSL with incompatible imm {}", imm);
1512                }
1513                dynasm!(self ; lsl W(dst), W(src1), imm as u32);
1514            }
1515            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1516                if imm > 31 {
1517                    codegen_error!("singlepass LSL with incompatible imm {}", imm);
1518                }
1519                dynasm!(self ; lsl W(dst), W(src1), imm as u32);
1520            }
1521            _ => codegen_error!(
1522                "singlepass can't emit LSL {:?} {:?} {:?} {:?}",
1523                sz,
1524                src1,
1525                src2,
1526                dst
1527            ),
1528        }
1529        Ok(())
1530    }
1531    fn emit_asr(
1532        &mut self,
1533        sz: Size,
1534        src1: Location,
1535        src2: Location,
1536        dst: Location,
1537    ) -> Result<(), CompileError> {
1538        match (sz, src1, src2, dst) {
1539            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1540                dynasm!(self ; asr X(dst), X(src1), X(src2));
1541            }
1542            (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1543                let imm = imm as u32;
1544
1545                if imm == 0 || imm > 63 {
1546                    codegen_error!("singlepass ASR with incompatible imm {}", imm);
1547                }
1548                dynasm!(self ; asr X(dst), X(src1), imm);
1549            }
1550            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1551                dynasm!(self ; asr W(dst), W(src1), W(src2));
1552            }
1553            (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1554                if imm == 0 || imm > 63 {
1555                    codegen_error!("singlepass ASR with incompatible imm {}", imm);
1556                }
1557                dynasm!(self ; asr X(dst), X(src1), imm as u32);
1558            }
1559            (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
1560                if imm == 0 || imm > 63 {
1561                    codegen_error!("singlepass ASR with incompatible imm {}", imm);
1562                }
1563                dynasm!(self ; asr X(dst), X(src1), imm as u32);
1564            }
1565            (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1566                if imm == 0 || imm > 31 {
1567                    codegen_error!("singlepass ASR with incompatible imm {}", imm);
1568                }
1569                dynasm!(self ; asr W(dst), W(src1), imm as u32);
1570            }
1571            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1572                if imm == 0 || imm > 31 {
1573                    codegen_error!("singlepass ASR with incompatible imm {}", imm);
1574                }
1575                dynasm!(self ; asr W(dst), W(src1), imm as u32);
1576            }
1577            _ => codegen_error!(
1578                "singlepass can't emit ASR {:?} {:?} {:?} {:?}",
1579                sz,
1580                src1,
1581                src2,
1582                dst
1583            ),
1584        }
1585        Ok(())
1586    }
1587    fn emit_lsr(
1588        &mut self,
1589        sz: Size,
1590        src1: Location,
1591        src2: Location,
1592        dst: Location,
1593    ) -> Result<(), CompileError> {
1594        match (sz, src1, src2, dst) {
1595            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1596                dynasm!(self ; lsr X(dst), X(src1), X(src2));
1597            }
1598            (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1599                let imm = imm as u32;
1600
1601                if imm == 0 || imm > 63 {
1602                    codegen_error!("singlepass LSR with incompatible imm {}", imm);
1603                }
1604                dynasm!(self ; lsr X(dst), X(src1), imm);
1605            }
1606            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1607                dynasm!(self ; lsr W(dst), W(src1), W(src2));
1608            }
1609            (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1610                if imm == 0 || imm > 63 {
1611                    codegen_error!("singlepass LSR with incompatible imm {}", imm);
1612                }
1613                dynasm!(self ; lsr X(dst), X(src1), imm as u32);
1614            }
1615            (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
1616                if imm == 0 || imm > 63 {
1617                    codegen_error!("singlepass LSR with incompatible imm {}", imm);
1618                }
1619                dynasm!(self ; lsr X(dst), X(src1), imm as u32);
1620            }
1621            (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1622                if imm == 0 || imm > 31 {
1623                    codegen_error!("singlepass LSR with incompatible imm {}", imm);
1624                }
1625                dynasm!(self ; lsr W(dst), W(src1), imm as u32);
1626            }
1627            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1628                if imm == 0 || imm > 31 {
1629                    codegen_error!("singlepass LSR with incompatible imm {}", imm);
1630                }
1631                dynasm!(self ; lsr W(dst), W(src1), imm as u32);
1632            }
1633            _ => codegen_error!(
1634                "singlepass can't emit LSR {:?} {:?} {:?} {:?}",
1635                sz,
1636                src1,
1637                src2,
1638                dst
1639            ),
1640        }
1641        Ok(())
1642    }
1643    fn emit_ror(
1644        &mut self,
1645        sz: Size,
1646        src1: Location,
1647        src2: Location,
1648        dst: Location,
1649    ) -> Result<(), CompileError> {
1650        match (sz, src1, src2, dst) {
1651            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1652                dynasm!(self ; ror X(dst), X(src1), X(src2));
1653            }
1654            (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1655                let imm = imm as u32;
1656
1657                if imm == 0 || imm > 63 {
1658                    codegen_error!("singlepass ROR with incompatible imm {}", imm);
1659                }
1660                dynasm!(self ; ror X(dst), X(src1), imm);
1661            }
1662            (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
1663                let imm = imm as u32;
1664
1665                if imm == 0 || imm > 63 {
1666                    codegen_error!("singlepass ROR with incompatible imm {}", imm);
1667                }
1668                dynasm!(self ; ror X(dst), X(src1), imm);
1669            }
1670            (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1671                if imm == 0 || imm > 31 {
1672                    codegen_error!("singlepass ROR with incompatible imm {}", imm);
1673                }
1674                dynasm!(self ; ror W(dst), W(src1), imm as u32);
1675            }
1676            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1677                dynasm!(self ; ror W(dst), W(src1), W(src2));
1678            }
1679            (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1680                if imm == 0 || imm > 63 {
1681                    codegen_error!("singlepass ROR with incompatible imm {}", imm);
1682                }
1683                dynasm!(self ; ror X(dst), X(src1), imm as u32);
1684            }
1685            (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1686                if imm == 0 || imm > 31 {
1687                    codegen_error!("singlepass ROR with incompatible imm {}", imm);
1688                }
1689                dynasm!(self ; ror W(dst), W(src1), imm as u32);
1690            }
1691            _ => codegen_error!(
1692                "singlepass can't emit ROR {:?} {:?} {:?} {:?}",
1693                sz,
1694                src1,
1695                src2,
1696                dst
1697            ),
1698        }
1699        Ok(())
1700    }
1701
1702    fn emit_or(
1703        &mut self,
1704        sz: Size,
1705        src1: Location,
1706        src2: Location,
1707        dst: Location,
1708    ) -> Result<(), CompileError> {
1709        match (sz, src1, src2, dst) {
1710            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1711                dynasm!(self ; orr X(dst), X(src1), X(src2));
1712            }
1713            (Size::S64, Location::GPR(src1), Location::Imm64(src2), Location::GPR(dst)) => {
1714                let src2 = src2 as u64;
1715
1716                if encode_logical_immediate_64bit(src2 as u64).is_none() {
1717                    codegen_error!("singlepass OR with incompatible imm {}", src2);
1718                }
1719                dynasm!(self ; orr XSP(dst), X(src1), src2);
1720            }
1721            (Size::S32, Location::GPR(src1), Location::Imm32(src2), Location::GPR(dst)) => {
1722                let src2 = src2 as u32;
1723
1724                if encode_logical_immediate_32bit(src2).is_none() {
1725                    codegen_error!("singlepass OR with incompatible imm {}", src2);
1726                }
1727                dynasm!(self ; orr WSP(dst), W(src1), src2);
1728            }
1729            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1730                dynasm!(self ; orr W(dst), W(src1), W(src2));
1731            }
1732            _ => codegen_error!(
1733                "singlepass can't emit OR {:?} {:?} {:?} {:?}",
1734                sz,
1735                src1,
1736                src2,
1737                dst
1738            ),
1739        }
1740        Ok(())
1741    }
1742    fn emit_and(
1743        &mut self,
1744        sz: Size,
1745        src1: Location,
1746        src2: Location,
1747        dst: Location,
1748    ) -> Result<(), CompileError> {
1749        match (sz, src1, src2, dst) {
1750            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1751                dynasm!(self ; and X(dst), X(src1), X(src2));
1752            }
1753            (Size::S64, Location::GPR(src1), Location::Imm64(src2), Location::GPR(dst)) => {
1754                let src2 = src2 as u64;
1755
1756                if encode_logical_immediate_64bit(src2 as u64).is_none() {
1757                    codegen_error!("singlepass AND with incompatible imm {}", src2);
1758                }
1759                dynasm!(self ; and XSP(dst), X(src1), src2);
1760            }
1761            (Size::S32, Location::GPR(src1), Location::Imm32(src2), Location::GPR(dst)) => {
1762                let src2 = src2 as u32;
1763
1764                if encode_logical_immediate_32bit(src2).is_none() {
1765                    codegen_error!("singlepass AND with incompatible imm {}", src2);
1766                }
1767                dynasm!(self ; and WSP(dst), W(src1), src2);
1768            }
1769            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1770                dynasm!(self ; and W(dst), W(src1), W(src2));
1771            }
1772            _ => codegen_error!(
1773                "singlepass can't emit AND {:?} {:?} {:?} {:?}",
1774                sz,
1775                src1,
1776                src2,
1777                dst
1778            ),
1779        }
1780        Ok(())
1781    }
1782    fn emit_eor(
1783        &mut self,
1784        sz: Size,
1785        src1: Location,
1786        src2: Location,
1787        dst: Location,
1788    ) -> Result<(), CompileError> {
1789        match (sz, src1, src2, dst) {
1790            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1791                dynasm!(self ; eor X(dst), X(src1), X(src2));
1792            }
1793            (Size::S64, Location::GPR(src1), Location::Imm64(src2), Location::GPR(dst)) => {
1794                let src2 = src2 as u64;
1795
1796                if encode_logical_immediate_64bit(src2 as u64).is_none() {
1797                    codegen_error!("singlepass EOR with incompatible imm {}", src2);
1798                }
1799                dynasm!(self ; eor XSP(dst), X(src1), src2);
1800            }
1801            (Size::S32, Location::GPR(src1), Location::Imm32(src2), Location::GPR(dst)) => {
1802                let src2 = src2 as u32;
1803
1804                if encode_logical_immediate_32bit(src2).is_none() {
1805                    codegen_error!("singlepass EOR with incompatible imm {}", src2);
1806                }
1807                dynasm!(self ; eor WSP(dst), W(src1), src2);
1808            }
1809            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1810                dynasm!(self ; eor W(dst), W(src1), W(src2));
1811            }
1812            _ => codegen_error!(
1813                "singlepass can't emit EOR {:?} {:?} {:?} {:?}",
1814                sz,
1815                src1,
1816                src2,
1817                dst
1818            ),
1819        }
1820        Ok(())
1821    }
1822
1823    fn emit_bfc(
1824        &mut self,
1825        sz: Size,
1826        lsb: u32,
1827        width: u32,
1828        dst: Location,
1829    ) -> Result<(), CompileError> {
1830        match (sz, dst) {
1831            (Size::S32, Location::GPR(dst)) => {
1832                dynasm!(self ; bfc W(dst), lsb, width);
1833            }
1834            (Size::S64, Location::GPR(dst)) => {
1835                dynasm!(self ; bfc X(dst), lsb, width);
1836            }
1837            _ => codegen_error!("singlepass can't emit BFC"),
1838        }
1839        Ok(())
1840    }
1841    fn emit_bfi(
1842        &mut self,
1843        sz: Size,
1844        src: Location,
1845        lsb: u32,
1846        width: u32,
1847        dst: Location,
1848    ) -> Result<(), CompileError> {
1849        match (sz, src, dst) {
1850            (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1851                dynasm!(self ; bfi W(dst), W(src), lsb, width);
1852            }
1853            (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1854                dynasm!(self ; bfi X(dst), X(src), lsb, width);
1855            }
1856            _ => codegen_error!("singlepass can't emit BFI"),
1857        }
1858        Ok(())
1859    }
1860
1861    fn emit_udiv(
1862        &mut self,
1863        sz: Size,
1864        src1: Location,
1865        src2: Location,
1866        dst: Location,
1867    ) -> Result<(), CompileError> {
1868        match (sz, src1, src2, dst) {
1869            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1870                dynasm!(self ; udiv W(dst), W(src1), W(src2));
1871            }
1872            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1873                dynasm!(self ; udiv X(dst), X(src1), X(src2));
1874            }
1875            _ => codegen_error!(
1876                "singlepass can't emit UDIV {:?} {:?} {:?} {:?}",
1877                sz,
1878                src1,
1879                src2,
1880                dst
1881            ),
1882        }
1883        Ok(())
1884    }
1885    fn emit_sdiv(
1886        &mut self,
1887        sz: Size,
1888        src1: Location,
1889        src2: Location,
1890        dst: Location,
1891    ) -> Result<(), CompileError> {
1892        match (sz, src1, src2, dst) {
1893            (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1894                dynasm!(self ; sdiv W(dst), W(src1), W(src2));
1895            }
1896            (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1897                dynasm!(self ; sdiv X(dst), X(src1), X(src2));
1898            }
1899            _ => codegen_error!(
1900                "singlepass can't emit SDIV {:?} {:?} {:?} {:?}",
1901                sz,
1902                src1,
1903                src2,
1904                dst
1905            ),
1906        }
1907        Ok(())
1908    }
1909
1910    /// msub : c - a*b -> dst
1911    fn emit_msub(
1912        &mut self,
1913        sz: Size,
1914        a: Location,
1915        b: Location,
1916        c: Location,
1917        dst: Location,
1918    ) -> Result<(), CompileError> {
1919        match (sz, a, b, c, dst) {
1920            (
1921                Size::S32,
1922                Location::GPR(a),
1923                Location::GPR(b),
1924                Location::GPR(c),
1925                Location::GPR(dst),
1926            ) => {
1927                dynasm!(self ; msub W(dst), W(a), W(b), W(c));
1928            }
1929            (
1930                Size::S64,
1931                Location::GPR(a),
1932                Location::GPR(b),
1933                Location::GPR(c),
1934                Location::GPR(dst),
1935            ) => {
1936                dynasm!(self ; msub X(dst), X(a), X(b), X(c));
1937            }
1938            _ => codegen_error!(
1939                "singlepass can't emit msub {:?} {:?} {:?} {:?} {:?}",
1940                sz,
1941                a,
1942                b,
1943                c,
1944                dst
1945            ),
1946        }
1947        Ok(())
1948    }
1949
1950    fn emit_sxtb(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1951        match (sz, src, dst) {
1952            (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1953                dynasm!(self ; sxtb W(dst), W(src));
1954            }
1955            (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1956                dynasm!(self ; sxtb X(dst), W(src));
1957            }
1958            _ => codegen_error!("singlepass can't emit SXTB {:?} {:?} {:?}", sz, src, dst),
1959        }
1960        Ok(())
1961    }
1962    fn emit_sxth(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1963        match (sz, src, dst) {
1964            (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1965                dynasm!(self ; sxth W(dst), W(src));
1966            }
1967            (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1968                dynasm!(self ; sxth X(dst), W(src));
1969            }
1970            _ => codegen_error!("singlepass can't emit SXTH {:?} {:?} {:?}", sz, src, dst),
1971        }
1972        Ok(())
1973    }
1974    fn emit_sxtw(&mut self, _sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1975        match (src, dst) {
1976            (Location::GPR(src), Location::GPR(dst)) => {
1977                dynasm!(self ; sxtw X(dst), W(src));
1978            }
1979            _ => codegen_error!("singlepass can't emit SXTW {:?} {:?}", src, dst),
1980        }
1981        Ok(())
1982    }
1983    fn emit_uxtb(&mut self, _sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1984        match (src, dst) {
1985            (Location::GPR(src), Location::GPR(dst)) => {
1986                dynasm!(self ; uxtb W(dst), W(src));
1987            }
1988            _ => codegen_error!("singlepass can't emit UXTB {:?} {:?}", src, dst),
1989        }
1990        Ok(())
1991    }
1992    fn emit_uxth(&mut self, _sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1993        match (src, dst) {
1994            (Location::GPR(src), Location::GPR(dst)) => {
1995                dynasm!(self ; uxth W(dst), W(src));
1996            }
1997            _ => codegen_error!("singlepass can't emit UXTH {:?} {:?}", src, dst),
1998        }
1999        Ok(())
2000    }
2001
2002    fn emit_cset(&mut self, sz: Size, dst: Location, cond: Condition) -> Result<(), CompileError> {
2003        match (sz, dst) {
2004            (Size::S32, Location::GPR(reg)) => match cond {
2005                Condition::Eq => dynasm!(self ; cset W(reg), eq),
2006                Condition::Ne => dynasm!(self ; cset W(reg), ne),
2007                Condition::Cs => dynasm!(self ; cset W(reg), cs),
2008                Condition::Cc => dynasm!(self ; cset W(reg), cc),
2009                Condition::Mi => dynasm!(self ; cset W(reg), mi),
2010                Condition::Pl => dynasm!(self ; cset W(reg), pl),
2011                Condition::Vs => dynasm!(self ; cset W(reg), vs),
2012                Condition::Vc => dynasm!(self ; cset W(reg), vc),
2013                Condition::Hi => dynasm!(self ; cset W(reg), hi),
2014                Condition::Ls => dynasm!(self ; cset W(reg), ls),
2015                Condition::Ge => dynasm!(self ; cset W(reg), ge),
2016                Condition::Lt => dynasm!(self ; cset W(reg), lt),
2017                Condition::Gt => dynasm!(self ; cset W(reg), gt),
2018                Condition::Le => dynasm!(self ; cset W(reg), le),
2019                Condition::Al => dynasm!(self ; cset W(reg), al),
2020            },
2021            (Size::S64, Location::GPR(reg)) => match cond {
2022                Condition::Eq => dynasm!(self ; cset X(reg), eq),
2023                Condition::Ne => dynasm!(self ; cset X(reg), ne),
2024                Condition::Cs => dynasm!(self ; cset X(reg), cs),
2025                Condition::Cc => dynasm!(self ; cset X(reg), cc),
2026                Condition::Mi => dynasm!(self ; cset X(reg), mi),
2027                Condition::Pl => dynasm!(self ; cset X(reg), pl),
2028                Condition::Vs => dynasm!(self ; cset X(reg), vs),
2029                Condition::Vc => dynasm!(self ; cset X(reg), vc),
2030                Condition::Hi => dynasm!(self ; cset X(reg), hi),
2031                Condition::Ls => dynasm!(self ; cset X(reg), ls),
2032                Condition::Ge => dynasm!(self ; cset X(reg), ge),
2033                Condition::Lt => dynasm!(self ; cset X(reg), lt),
2034                Condition::Gt => dynasm!(self ; cset X(reg), gt),
2035                Condition::Le => dynasm!(self ; cset X(reg), le),
2036                Condition::Al => dynasm!(self ; cset X(reg), al),
2037            },
2038            _ => codegen_error!("singlepass can't emit CSET {:?} {:?} {:?}", sz, dst, cond),
2039        }
2040        Ok(())
2041    }
2042    fn emit_csetm(&mut self, sz: Size, dst: Location, cond: Condition) -> Result<(), CompileError> {
2043        match (sz, dst) {
2044            (Size::S32, Location::GPR(reg)) => match cond {
2045                Condition::Eq => dynasm!(self ; csetm W(reg), eq),
2046                Condition::Ne => dynasm!(self ; csetm W(reg), ne),
2047                Condition::Cs => dynasm!(self ; csetm W(reg), cs),
2048                Condition::Cc => dynasm!(self ; csetm W(reg), cc),
2049                Condition::Mi => dynasm!(self ; csetm W(reg), mi),
2050                Condition::Pl => dynasm!(self ; csetm W(reg), pl),
2051                Condition::Vs => dynasm!(self ; csetm W(reg), vs),
2052                Condition::Vc => dynasm!(self ; csetm W(reg), vc),
2053                Condition::Hi => dynasm!(self ; csetm W(reg), hi),
2054                Condition::Ls => dynasm!(self ; csetm W(reg), ls),
2055                Condition::Ge => dynasm!(self ; csetm W(reg), ge),
2056                Condition::Lt => dynasm!(self ; csetm W(reg), lt),
2057                Condition::Gt => dynasm!(self ; csetm W(reg), gt),
2058                Condition::Le => dynasm!(self ; csetm W(reg), le),
2059                Condition::Al => dynasm!(self ; csetm W(reg), al),
2060            },
2061            (Size::S64, Location::GPR(reg)) => match cond {
2062                Condition::Eq => dynasm!(self ; csetm X(reg), eq),
2063                Condition::Ne => dynasm!(self ; csetm X(reg), ne),
2064                Condition::Cs => dynasm!(self ; csetm X(reg), cs),
2065                Condition::Cc => dynasm!(self ; csetm X(reg), cc),
2066                Condition::Mi => dynasm!(self ; csetm X(reg), mi),
2067                Condition::Pl => dynasm!(self ; csetm X(reg), pl),
2068                Condition::Vs => dynasm!(self ; csetm X(reg), vs),
2069                Condition::Vc => dynasm!(self ; csetm X(reg), vc),
2070                Condition::Hi => dynasm!(self ; csetm X(reg), hi),
2071                Condition::Ls => dynasm!(self ; csetm X(reg), ls),
2072                Condition::Ge => dynasm!(self ; csetm X(reg), ge),
2073                Condition::Lt => dynasm!(self ; csetm X(reg), lt),
2074                Condition::Gt => dynasm!(self ; csetm X(reg), gt),
2075                Condition::Le => dynasm!(self ; csetm X(reg), le),
2076                Condition::Al => dynasm!(self ; csetm X(reg), al),
2077            },
2078            _ => codegen_error!("singlepass can't emit CSETM {:?} {:?} {:?}", sz, dst, cond),
2079        }
2080        Ok(())
2081    }
2082    fn emit_cinc(
2083        &mut self,
2084        sz: Size,
2085        src: Location,
2086        dst: Location,
2087        cond: Condition,
2088    ) -> Result<(), CompileError> {
2089        match (sz, src, dst) {
2090            (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
2091                match cond {
2092                    Condition::Eq => dynasm!(self ; cinc W(dst), W(src), eq),
2093                    Condition::Ne => dynasm!(self ; cinc W(dst), W(src), ne),
2094                    Condition::Cs => dynasm!(self ; cinc W(dst), W(src), cs),
2095                    Condition::Cc => dynasm!(self ; cinc W(dst), W(src), cc),
2096                    Condition::Mi => dynasm!(self ; cinc W(dst), W(src), mi),
2097                    Condition::Pl => dynasm!(self ; cinc W(dst), W(src), pl),
2098                    Condition::Vs => dynasm!(self ; cinc W(dst), W(src), vs),
2099                    Condition::Vc => dynasm!(self ; cinc W(dst), W(src), vc),
2100                    Condition::Hi => dynasm!(self ; cinc W(dst), W(src), hi),
2101                    Condition::Ls => dynasm!(self ; cinc W(dst), W(src), ls),
2102                    Condition::Ge => dynasm!(self ; cinc W(dst), W(src), ge),
2103                    Condition::Lt => dynasm!(self ; cinc W(dst), W(src), lt),
2104                    Condition::Gt => dynasm!(self ; cinc W(dst), W(src), gt),
2105                    Condition::Le => dynasm!(self ; cinc W(dst), W(src), le),
2106                    Condition::Al => dynasm!(self ; cinc W(dst), W(src), al),
2107                };
2108            }
2109            (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
2110                match cond {
2111                    Condition::Eq => dynasm!(self ; cinc X(src), X(dst), eq),
2112                    Condition::Ne => dynasm!(self ; cinc X(src), X(dst), ne),
2113                    Condition::Cs => dynasm!(self ; cinc X(src), X(dst), cs),
2114                    Condition::Cc => dynasm!(self ; cinc X(src), X(dst), cc),
2115                    Condition::Mi => dynasm!(self ; cinc X(src), X(dst), mi),
2116                    Condition::Pl => dynasm!(self ; cinc X(src), X(dst), pl),
2117                    Condition::Vs => dynasm!(self ; cinc X(src), X(dst), vs),
2118                    Condition::Vc => dynasm!(self ; cinc X(src), X(dst), vc),
2119                    Condition::Hi => dynasm!(self ; cinc X(src), X(dst), hi),
2120                    Condition::Ls => dynasm!(self ; cinc X(src), X(dst), ls),
2121                    Condition::Ge => dynasm!(self ; cinc X(src), X(dst), ge),
2122                    Condition::Lt => dynasm!(self ; cinc X(src), X(dst), lt),
2123                    Condition::Gt => dynasm!(self ; cinc X(src), X(dst), gt),
2124                    Condition::Le => dynasm!(self ; cinc X(src), X(dst), le),
2125                    Condition::Al => dynasm!(self ; cinc X(src), X(dst), al),
2126                };
2127            }
2128            _ => codegen_error!("singlepass can't emit CINC"),
2129        }
2130        Ok(())
2131    }
2132
2133    fn emit_clz(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2134        match (sz, src, dst) {
2135            (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
2136                dynasm!(self ; clz X(dst), X(src));
2137            }
2138            (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
2139                dynasm!(self ; clz W(dst), W(src));
2140            }
2141            _ => codegen_error!("singlepass can't emit CLS {:?} {:?} {:?}", sz, src, dst),
2142        }
2143        Ok(())
2144    }
2145    fn emit_rbit(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2146        match (sz, src, dst) {
2147            (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
2148                dynasm!(self ; rbit X(dst), X(src));
2149            }
2150            (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
2151                dynasm!(self ; rbit W(dst), W(src));
2152            }
2153            _ => codegen_error!("singlepass can't emit CLS {:?} {:?} {:?}", sz, src, dst),
2154        }
2155        Ok(())
2156    }
2157
2158    fn emit_label(&mut self, label: Label) -> Result<(), CompileError> {
2159        dynasm!(self ; => label);
2160        Ok(())
2161    }
2162    fn emit_load_label(&mut self, reg: GPR, label: Label) -> Result<(), CompileError> {
2163        dynasm!(self ; adr X(reg), =>label);
2164        Ok(())
2165    }
2166
2167    fn emit_b_label(&mut self, label: Label) -> Result<(), CompileError> {
2168        dynasm!(self ; b =>label);
2169        Ok(())
2170    }
2171    fn emit_cbz_label(
2172        &mut self,
2173        sz: Size,
2174        reg: Location,
2175        label: Label,
2176    ) -> Result<(), CompileError> {
2177        match (sz, reg) {
2178            (Size::S32, Location::GPR(reg)) => {
2179                dynasm!(self ; cbz W(reg), =>label);
2180            }
2181            (Size::S64, Location::GPR(reg)) => {
2182                dynasm!(self ; cbz X(reg), =>label);
2183            }
2184            _ => codegen_error!("singlepass can't emit CBZ {:?} {:?} {:?}", sz, reg, label),
2185        }
2186        Ok(())
2187    }
2188    fn emit_cbnz_label(
2189        &mut self,
2190        sz: Size,
2191        reg: Location,
2192        label: Label,
2193    ) -> Result<(), CompileError> {
2194        match (sz, reg) {
2195            (Size::S32, Location::GPR(reg)) => {
2196                dynasm!(self ; cbnz W(reg), =>label);
2197            }
2198            (Size::S64, Location::GPR(reg)) => {
2199                dynasm!(self ; cbnz X(reg), =>label);
2200            }
2201            _ => codegen_error!("singlepass can't emit CBNZ {:?} {:?} {:?}", sz, reg, label),
2202        }
2203        Ok(())
2204    }
2205    fn emit_cbz_label_far(
2206        &mut self,
2207        sz: Size,
2208        reg: Location,
2209        label: Label,
2210    ) -> Result<(), CompileError> {
2211        let near_label: Label = self.get_label();
2212        let continue_label: Label = self.get_label();
2213
2214        match (sz, reg) {
2215            (Size::S32, Location::GPR(reg)) => {
2216                dynasm!(self ; cbz W(reg), => near_label);
2217                dynasm!(self ; b => continue_label);
2218            }
2219            (Size::S64, Location::GPR(reg)) => {
2220                dynasm!(self ; cbz X(reg), => near_label);
2221                dynasm!(self ; b => continue_label);
2222            }
2223            _ => codegen_error!("singlepass can't emit CBZ {:?} {:?} {:?}", sz, reg, label),
2224        }
2225        self.emit_label(near_label)?;
2226        dynasm!(self ; b => label );
2227
2228        self.emit_label(continue_label)?;
2229
2230        Ok(())
2231    }
2232    fn emit_tbz_label(
2233        &mut self,
2234        sz: Size,
2235        reg: Location,
2236        n: u32,
2237        label: Label,
2238    ) -> Result<(), CompileError> {
2239        match (sz, reg) {
2240            (Size::S32, Location::GPR(reg)) => {
2241                dynasm!(self ; tbz W(reg), n, =>label);
2242            }
2243            (Size::S64, Location::GPR(reg)) => {
2244                dynasm!(self ; tbz X(reg), n, =>label);
2245            }
2246            _ => codegen_error!(
2247                "singlepass can't emit TBZ {:?} {:?} {:?} {:?}",
2248                sz,
2249                reg,
2250                n,
2251                label
2252            ),
2253        }
2254        Ok(())
2255    }
2256    fn emit_tbnz_label(
2257        &mut self,
2258        sz: Size,
2259        reg: Location,
2260        n: u32,
2261        label: Label,
2262    ) -> Result<(), CompileError> {
2263        match (sz, reg) {
2264            (Size::S32, Location::GPR(reg)) => {
2265                dynasm!(self ; tbnz W(reg), n, =>label);
2266            }
2267            (Size::S64, Location::GPR(reg)) => {
2268                dynasm!(self ; tbnz X(reg), n, =>label);
2269            }
2270            _ => codegen_error!(
2271                "singlepass can't emit TBNZ {:?} {:?} {:?} {:?}",
2272                sz,
2273                reg,
2274                n,
2275                label
2276            ),
2277        }
2278        Ok(())
2279    }
2280    fn emit_bcond_label(&mut self, condition: Condition, label: Label) -> Result<(), CompileError> {
2281        match condition {
2282            Condition::Eq => dynasm!(self ; b.eq => label),
2283            Condition::Ne => dynasm!(self ; b.ne => label),
2284            Condition::Cs => dynasm!(self ; b.cs => label),
2285            Condition::Cc => dynasm!(self ; b.cc => label),
2286            Condition::Mi => dynasm!(self ; b.mi => label),
2287            Condition::Pl => dynasm!(self ; b.pl => label),
2288            Condition::Vs => dynasm!(self ; b.vs => label),
2289            Condition::Vc => dynasm!(self ; b.vc => label),
2290            Condition::Hi => dynasm!(self ; b.hi => label),
2291            Condition::Ls => dynasm!(self ; b.ls => label),
2292            Condition::Ge => dynasm!(self ; b.ge => label),
2293            Condition::Lt => dynasm!(self ; b.lt => label),
2294            Condition::Gt => dynasm!(self ; b.gt => label),
2295            Condition::Le => dynasm!(self ; b.le => label),
2296            Condition::Al => dynasm!(self ; b => label),
2297        }
2298        Ok(())
2299    }
2300    fn emit_bcond_label_far(
2301        &mut self,
2302        condition: Condition,
2303        label: Label,
2304    ) -> Result<(), CompileError> {
2305        let cont: Label = self.get_label();
2306        match condition {
2307            // if not condition than continue
2308            Condition::Eq => dynasm!(self ; b.ne => cont),
2309            Condition::Ne => dynasm!(self ; b.eq => cont),
2310            Condition::Cs => dynasm!(self ; b.cc => cont),
2311            Condition::Cc => dynasm!(self ; b.cs => cont),
2312            Condition::Mi => dynasm!(self ; b.pl => cont),
2313            Condition::Pl => dynasm!(self ; b.mi => cont),
2314            Condition::Vs => dynasm!(self ; b.vc => cont),
2315            Condition::Vc => dynasm!(self ; b.vs => cont),
2316            Condition::Hi => dynasm!(self ; b.ls => cont),
2317            Condition::Ls => dynasm!(self ; b.hi => cont),
2318            Condition::Ge => dynasm!(self ; b.lt => cont),
2319            Condition::Lt => dynasm!(self ; b.ge => cont),
2320            Condition::Gt => dynasm!(self ; b.le => cont),
2321            Condition::Le => dynasm!(self ; b.gt => cont),
2322            Condition::Al => { /*nothing*/ }
2323        }
2324        dynasm!(self ; b => label);
2325        self.emit_label(cont)?;
2326        Ok(())
2327    }
2328    fn emit_b_register(&mut self, reg: GPR) -> Result<(), CompileError> {
2329        dynasm!(self ; br X(reg));
2330        Ok(())
2331    }
2332    fn emit_call_label(&mut self, label: Label) -> Result<(), CompileError> {
2333        dynasm!(self ; bl =>label);
2334        Ok(())
2335    }
2336    fn emit_call_register(&mut self, reg: GPR) -> Result<(), CompileError> {
2337        dynasm!(self ; blr X(reg));
2338        Ok(())
2339    }
2340    fn emit_ret(&mut self) -> Result<(), CompileError> {
2341        dynasm!(self ; ret);
2342        Ok(())
2343    }
2344
2345    fn emit_udf(&mut self, payload: u16) -> Result<(), CompileError> {
2346        dynasm!(self ; udf payload as u32);
2347        Ok(())
2348    }
2349    fn emit_dmb(&mut self) -> Result<(), CompileError> {
2350        dynasm!(self ; dmb ish);
2351        Ok(())
2352    }
2353    fn emit_brk(&mut self) -> Result<(), CompileError> {
2354        dynasm!(self ; brk 0);
2355        Ok(())
2356    }
2357
2358    fn emit_fcmp(&mut self, sz: Size, src1: Location, src2: Location) -> Result<(), CompileError> {
2359        match (sz, src1, src2) {
2360            (Size::S32, Location::SIMD(src1), Location::SIMD(src2)) => {
2361                dynasm!(self ; fcmp S(src1), S(src2));
2362            }
2363            (Size::S64, Location::SIMD(src1), Location::SIMD(src2)) => {
2364                dynasm!(self ; fcmp D(src1), D(src2));
2365            }
2366            _ => codegen_error!("singlepass can't emit FCMP {:?} {:?} {:?}", sz, src1, src2),
2367        }
2368        Ok(())
2369    }
2370
2371    fn emit_fneg(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2372        match (sz, src, dst) {
2373            (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2374                dynasm!(self ; fneg S(dst), S(src));
2375            }
2376            (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2377                dynasm!(self ; fneg D(dst), D(src));
2378            }
2379            _ => codegen_error!("singlepass can't emit FNEG {:?} {:?} {:?}", sz, src, dst),
2380        }
2381        Ok(())
2382    }
2383    fn emit_fsqrt(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2384        match (sz, src, dst) {
2385            (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2386                dynasm!(self ; fsqrt S(dst), S(src));
2387            }
2388            (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2389                dynasm!(self ; fsqrt D(dst), D(src));
2390            }
2391            _ => codegen_error!("singlepass can't emit FSQRT {:?} {:?} {:?}", sz, src, dst),
2392        }
2393        Ok(())
2394    }
2395
2396    fn emit_fadd(
2397        &mut self,
2398        sz: Size,
2399        src1: Location,
2400        src2: Location,
2401        dst: Location,
2402    ) -> Result<(), CompileError> {
2403        match (sz, src1, src2, dst) {
2404            (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2405                dynasm!(self ; fadd S(dst), S(src1), S(src2));
2406            }
2407            (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2408                dynasm!(self ; fadd D(dst), D(src1), D(src2));
2409            }
2410            _ => codegen_error!(
2411                "singlepass can't emit FADD {:?} {:?} {:?} {:?}",
2412                sz,
2413                src1,
2414                src2,
2415                dst
2416            ),
2417        }
2418        Ok(())
2419    }
2420    fn emit_fsub(
2421        &mut self,
2422        sz: Size,
2423        src1: Location,
2424        src2: Location,
2425        dst: Location,
2426    ) -> Result<(), CompileError> {
2427        match (sz, src1, src2, dst) {
2428            (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2429                dynasm!(self ; fsub S(dst), S(src1), S(src2));
2430            }
2431            (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2432                dynasm!(self ; fsub D(dst), D(src1), D(src2));
2433            }
2434            _ => codegen_error!(
2435                "singlepass can't emit FSUB {:?} {:?} {:?} {:?}",
2436                sz,
2437                src1,
2438                src2,
2439                dst
2440            ),
2441        }
2442        Ok(())
2443    }
2444    fn emit_fmul(
2445        &mut self,
2446        sz: Size,
2447        src1: Location,
2448        src2: Location,
2449        dst: Location,
2450    ) -> Result<(), CompileError> {
2451        match (sz, src1, src2, dst) {
2452            (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2453                dynasm!(self ; fmul S(dst), S(src1), S(src2));
2454            }
2455            (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2456                dynasm!(self ; fmul D(dst), D(src1), D(src2));
2457            }
2458            _ => codegen_error!(
2459                "singlepass can't emit FMUL {:?} {:?} {:?} {:?}",
2460                sz,
2461                src1,
2462                src2,
2463                dst
2464            ),
2465        }
2466        Ok(())
2467    }
2468    fn emit_fdiv(
2469        &mut self,
2470        sz: Size,
2471        src1: Location,
2472        src2: Location,
2473        dst: Location,
2474    ) -> Result<(), CompileError> {
2475        match (sz, src1, src2, dst) {
2476            (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2477                dynasm!(self ; fdiv S(dst), S(src1), S(src2));
2478            }
2479            (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2480                dynasm!(self ; fdiv D(dst), D(src1), D(src2));
2481            }
2482            _ => codegen_error!(
2483                "singlepass can't emit FDIV {:?} {:?} {:?} {:?}",
2484                sz,
2485                src1,
2486                src2,
2487                dst
2488            ),
2489        }
2490        Ok(())
2491    }
2492
2493    fn emit_fmin(
2494        &mut self,
2495        sz: Size,
2496        src1: Location,
2497        src2: Location,
2498        dst: Location,
2499    ) -> Result<(), CompileError> {
2500        match (sz, src1, src2, dst) {
2501            (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2502                dynasm!(self ; fmin S(dst), S(src1), S(src2));
2503            }
2504            (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2505                dynasm!(self ; fmin D(dst), D(src1), D(src2));
2506            }
2507            _ => codegen_error!(
2508                "singlepass can't emit FMIN {:?} {:?} {:?} {:?}",
2509                sz,
2510                src1,
2511                src2,
2512                dst
2513            ),
2514        }
2515        Ok(())
2516    }
2517    fn emit_fmax(
2518        &mut self,
2519        sz: Size,
2520        src1: Location,
2521        src2: Location,
2522        dst: Location,
2523    ) -> Result<(), CompileError> {
2524        match (sz, src1, src2, dst) {
2525            (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2526                dynasm!(self ; fmax S(dst), S(src1), S(src2));
2527            }
2528            (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2529                dynasm!(self ; fmax D(dst), D(src1), D(src2));
2530            }
2531            _ => codegen_error!(
2532                "singlepass can't emit FMAX {:?} {:?} {:?} {:?}",
2533                sz,
2534                src1,
2535                src2,
2536                dst
2537            ),
2538        }
2539        Ok(())
2540    }
2541
2542    fn emit_frintz(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2543        match (sz, src, dst) {
2544            (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2545                dynasm!(self ; frintz S(dst), S(src));
2546            }
2547            (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2548                dynasm!(self ; frintz D(dst), D(src));
2549            }
2550            _ => codegen_error!("singlepass can't emit FRINTZ {:?} {:?} {:?}", sz, src, dst),
2551        }
2552        Ok(())
2553    }
2554    fn emit_frintn(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2555        match (sz, src, dst) {
2556            (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2557                dynasm!(self ; frintn S(dst), S(src));
2558            }
2559            (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2560                dynasm!(self ; frintn D(dst), D(src));
2561            }
2562            _ => codegen_error!("singlepass can't emit FRINTN {:?} {:?} {:?}", sz, src, dst),
2563        }
2564        Ok(())
2565    }
2566    fn emit_frintm(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2567        match (sz, src, dst) {
2568            (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2569                dynasm!(self ; frintm S(dst), S(src));
2570            }
2571            (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2572                dynasm!(self ; frintm D(dst), D(src));
2573            }
2574            _ => codegen_error!("singlepass can't emit FRINTM {:?} {:?} {:?}", sz, src, dst),
2575        }
2576        Ok(())
2577    }
2578    fn emit_frintp(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2579        match (sz, src, dst) {
2580            (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2581                dynasm!(self ; frintp S(dst), S(src));
2582            }
2583            (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2584                dynasm!(self ; frintp D(dst), D(src));
2585            }
2586            _ => codegen_error!("singlepass can't emit FRINTP {:?} {:?} {:?}", sz, src, dst),
2587        }
2588        Ok(())
2589    }
2590
2591    fn emit_scvtf(
2592        &mut self,
2593        sz_in: Size,
2594        src: Location,
2595        sz_out: Size,
2596        dst: Location,
2597    ) -> Result<(), CompileError> {
2598        match (sz_in, src, sz_out, dst) {
2599            (Size::S32, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
2600                dynasm!(self ; scvtf S(dst), W(src));
2601            }
2602            (Size::S64, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
2603                dynasm!(self ; scvtf S(dst), X(src));
2604            }
2605            (Size::S32, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
2606                dynasm!(self ; scvtf D(dst), W(src));
2607            }
2608            (Size::S64, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
2609                dynasm!(self ; scvtf D(dst), X(src));
2610            }
2611            _ => codegen_error!(
2612                "singlepass can't emit SCVTF {:?} {:?} {:?} {:?}",
2613                sz_in,
2614                src,
2615                sz_out,
2616                dst
2617            ),
2618        }
2619        Ok(())
2620    }
2621    fn emit_ucvtf(
2622        &mut self,
2623        sz_in: Size,
2624        src: Location,
2625        sz_out: Size,
2626        dst: Location,
2627    ) -> Result<(), CompileError> {
2628        match (sz_in, src, sz_out, dst) {
2629            (Size::S32, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
2630                dynasm!(self ; ucvtf S(dst), W(src));
2631            }
2632            (Size::S64, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
2633                dynasm!(self ; ucvtf S(dst), X(src));
2634            }
2635            (Size::S32, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
2636                dynasm!(self ; ucvtf D(dst), W(src));
2637            }
2638            (Size::S64, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
2639                dynasm!(self ; ucvtf D(dst), X(src));
2640            }
2641            _ => codegen_error!(
2642                "singlepass can't emit UCVTF {:?} {:?} {:?} {:?}",
2643                sz_in,
2644                src,
2645                sz_out,
2646                dst
2647            ),
2648        }
2649        Ok(())
2650    }
2651    fn emit_fcvt(&mut self, sz_in: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2652        match (sz_in, src, dst) {
2653            (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2654                dynasm!(self ; fcvt D(dst), S(src));
2655            }
2656            (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2657                dynasm!(self ; fcvt S(dst), D(src));
2658            }
2659            _ => codegen_error!(
2660                "singlepass can't emit UCVTF {:?} {:?} {:?}",
2661                sz_in,
2662                src,
2663                dst
2664            ),
2665        }
2666        Ok(())
2667    }
2668    fn emit_fcvtzs(
2669        &mut self,
2670        sz_in: Size,
2671        src: Location,
2672        sz_out: Size,
2673        dst: Location,
2674    ) -> Result<(), CompileError> {
2675        match (sz_in, src, sz_out, dst) {
2676            (Size::S32, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
2677                dynasm!(self ; fcvtzs W(dst), S(src));
2678            }
2679            (Size::S64, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
2680                dynasm!(self ; fcvtzs W(dst), D(src));
2681            }
2682            (Size::S32, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
2683                dynasm!(self ; fcvtzs X(dst), S(src));
2684            }
2685            (Size::S64, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
2686                dynasm!(self ; fcvtzs X(dst), D(src));
2687            }
2688            _ => codegen_error!(
2689                "singlepass can't emit FCVTZS {:?} {:?} {:?} {:?}",
2690                sz_in,
2691                src,
2692                sz_out,
2693                dst
2694            ),
2695        }
2696        Ok(())
2697    }
2698    fn emit_fcvtzu(
2699        &mut self,
2700        sz_in: Size,
2701        src: Location,
2702        sz_out: Size,
2703        dst: Location,
2704    ) -> Result<(), CompileError> {
2705        match (sz_in, src, sz_out, dst) {
2706            (Size::S32, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
2707                dynasm!(self ; fcvtzu W(dst), S(src));
2708            }
2709            (Size::S64, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
2710                dynasm!(self ; fcvtzu W(dst), D(src));
2711            }
2712            (Size::S32, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
2713                dynasm!(self ; fcvtzu X(dst), S(src));
2714            }
2715            (Size::S64, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
2716                dynasm!(self ; fcvtzu X(dst), D(src));
2717            }
2718            _ => codegen_error!(
2719                "singlepass can't emit FCVTZU {:?} {:?} {:?} {:?}",
2720                sz_in,
2721                src,
2722                sz_out,
2723                dst
2724            ),
2725        }
2726        Ok(())
2727    }
2728
2729    // 1 011 0100 0100 000 => fpcr
2730    fn emit_read_fpcr(&mut self, reg: GPR) -> Result<(), CompileError> {
2731        dynasm!(self ; mrs X(reg), 0b1_011_0100_0100_000);
2732        Ok(())
2733    }
2734    fn emit_write_fpcr(&mut self, reg: GPR) -> Result<(), CompileError> {
2735        dynasm!(self ; msr 0b1_011_0100_0100_000, X(reg));
2736        Ok(())
2737    }
2738    // 1 011 0100 0100 001 => fpsr
2739    fn emit_read_fpsr(&mut self, reg: GPR) -> Result<(), CompileError> {
2740        dynasm!(self ; mrs X(reg), 0b1_011_0100_0100_001);
2741        Ok(())
2742    }
2743    fn emit_write_fpsr(&mut self, reg: GPR) -> Result<(), CompileError> {
2744        dynasm!(self ; msr 0b1_011_0100_0100_001, X(reg));
2745        Ok(())
2746    }
2747
2748    fn emit_cnt(&mut self, src: NEON, dst: NEON) -> Result<(), CompileError> {
2749        dynasm!(self ; cnt V(dst).B8, V(src).B8);
2750        Ok(())
2751    }
2752
2753    fn emit_addv(&mut self, src: NEON, dst: NEON) -> Result<(), CompileError> {
2754        dynasm!(self ; addv B(dst), V(src).B8);
2755        Ok(())
2756    }
2757
2758    #[allow(clippy::unit_arg)]
2759    fn emit_fmov(
2760        &mut self,
2761        src_size: Size,
2762        src: Location,
2763        dst_size: Size,
2764        dst: Location,
2765    ) -> Result<(), CompileError> {
2766        let mut err = false;
2767        match (src, dst) {
2768            (AbstractLocation::GPR(src), AbstractLocation::SIMD(dst)) => {
2769                match (src_size, dst_size) {
2770                    (Size::S32, Size::S32) => dynasm!(self ; fmov S(dst), W(src)),
2771                    (Size::S64, Size::S64) => dynasm!(self ; fmov D(dst), X(src)),
2772                    _ => {
2773                        err = true;
2774                    }
2775                }
2776            }
2777            (AbstractLocation::SIMD(src), AbstractLocation::GPR(dst)) => {
2778                match (src_size, dst_size) {
2779                    (Size::S32, Size::S32) => dynasm!(self ; fmov W(dst), S(src)),
2780                    (Size::S64, Size::S64) => dynasm!(self ; fmov X(dst), D(src)),
2781                    _ => {
2782                        err = true;
2783                    }
2784                }
2785            }
2786            // (AbstractLocation::SIMD(src), AbstractLocation::SIMD(dst)) => todo!(),
2787            // (AbstractLocation::SIMD(src), AbstractLocation::Imm32(dst)) => todo!(),
2788            // (AbstractLocation::SIMD(src), AbstractLocation::Imm64(dst)) => todo!(),
2789            _ => {
2790                err = true;
2791            }
2792        };
2793
2794        if err {
2795            codegen_error!(
2796                "singlepass can't generate fmov instruction for src_size: {:?}, src: {:?}, dst_size: {:?}, dst: {:?}",
2797                src_size,
2798                src,
2799                dst_size,
2800                dst,
2801            )
2802        }
2803        Ok(())
2804    }
2805}
2806
2807pub fn gen_std_trampoline_arm64(
2808    sig: &FunctionType,
2809    calling_convention: CallingConvention,
2810    progress_callback: Option<&CompilationProgressCallback>,
2811) -> Result<FunctionBody, CompileError> {
2812    let mut a = Assembler::new(0);
2813    let mut output_reporter = ChunkedOutputReporter::new(progress_callback);
2814
2815    let fptr = GPR::X27;
2816    let args = GPR::X28;
2817
2818    dynasm!(a
2819        ; sub sp, sp, 32
2820        ; stp x29, x30, [sp]
2821        ; stp X(fptr), X(args), [sp, 16]
2822        ; mov x29, sp
2823        ; mov X(fptr), x1
2824        ; mov X(args), x2
2825    );
2826
2827    let stack_args = sig.params().len().saturating_sub(7); //1st arg is ctx, not an actual arg
2828    let stack_return_slots = sig
2829        .results()
2830        .len()
2831        .saturating_sub(ARM64_RETURN_VALUE_REGISTERS.len());
2832    let mut stack_offset = (stack_args + stack_return_slots) as u32 * 8;
2833    if stack_offset > 0 {
2834        if !stack_offset.is_multiple_of(16) {
2835            stack_offset += 8;
2836            assert!(stack_offset.is_multiple_of(16));
2837        }
2838        if stack_offset < 0x1000 {
2839            dynasm!(a ; sub sp, sp, stack_offset);
2840        } else {
2841            a.emit_mov_imm(Location::GPR(GPR::X26), stack_offset as u64)?;
2842            dynasm!(a ; sub sp, sp, x26);
2843        }
2844    }
2845
2846    // Move arguments to their locations.
2847    // `callee_vmctx` is already in the first argument register, so no need to move.
2848    let mut caller_stack_offset = stack_return_slots as u32 * 8;
2849    for (i, param) in sig.params().iter().enumerate() {
2850        let sz = match *param {
2851            Type::I32 | Type::F32 => Size::S32,
2852            Type::I64 | Type::F64 => Size::S64,
2853            Type::ExternRef => Size::S64,
2854            Type::FuncRef => Size::S64,
2855            _ => codegen_error!(
2856                "singlepass unsupported param type for trampoline {:?}",
2857                *param
2858            ),
2859        };
2860        match i {
2861            0..=6 => {
2862                a.emit_ldr(
2863                    sz,
2864                    Location::GPR(GPR::from_index(i + 1).unwrap()),
2865                    Location::Memory(args, (i * 16) as i32),
2866                )?;
2867            }
2868            _ => {
2869                #[allow(clippy::single_match)]
2870                match calling_convention {
2871                    CallingConvention::AppleAarch64 => {
2872                        // align first
2873                        caller_stack_offset = caller_stack_offset.next_multiple_of(sz.bytes());
2874                    }
2875                    _ => (),
2876                };
2877                // using X16 as scratch reg
2878                a.emit_ldr(
2879                    sz,
2880                    Location::GPR(GPR::X16),
2881                    Location::Memory(args, (i * 16) as i32),
2882                )?;
2883                a.emit_str(
2884                    sz,
2885                    Location::GPR(GPR::X16),
2886                    Location::Memory(GPR::XzrSp, caller_stack_offset as _),
2887                )?;
2888                match calling_convention {
2889                    CallingConvention::AppleAarch64 => {
2890                        caller_stack_offset += sz.bytes();
2891                    }
2892                    _ => {
2893                        caller_stack_offset += 8;
2894                    }
2895                }
2896            }
2897        }
2898        output_reporter.check(a.offset().0)?;
2899    }
2900
2901    dynasm!(a  ; blr X(fptr));
2902
2903    // Write return values.
2904    let mut n_stack_return_slots: usize = 0;
2905    for i in 0..sig.results().len() {
2906        let src = if let Some(&reg) = ARM64_RETURN_VALUE_REGISTERS.get(i) {
2907            reg
2908        } else {
2909            let loc = GPR::X16;
2910            a.emit_ldr(
2911                Size::S64,
2912                Location::GPR(GPR::X16),
2913                Location::Memory(GPR::XzrSp, (n_stack_return_slots as u32 * 8) as _),
2914            )?;
2915            n_stack_return_slots += 1;
2916            loc
2917        };
2918        a.emit_str(
2919            Size::S64,
2920            Location::GPR(src),
2921            Location::Memory(args, (i * 16) as _),
2922        )?;
2923        output_reporter.check(a.offset().0)?;
2924    }
2925
2926    // Restore stack.
2927    dynasm!(a
2928        ; ldp X(fptr), X(args), [x29, 16]
2929        ; ldp x29, x30, [x29]
2930    );
2931    let restored_stack_offset = 32 + stack_offset;
2932    if restored_stack_offset < 0x1000 {
2933        dynasm!(a; add sp, sp, restored_stack_offset);
2934    } else {
2935        a.emit_mov_imm(Location::GPR(GPR::X26), restored_stack_offset as u64)?;
2936        dynasm!(a; add sp, sp, x26);
2937    }
2938    dynasm!(a; ret);
2939
2940    let mut body = a.finalize().unwrap();
2941    body.shrink_to_fit();
2942    output_reporter.finish(body.len())?;
2943    Ok(FunctionBody {
2944        body,
2945        unwind_info: None,
2946    })
2947}
2948// Generates dynamic import function call trampoline for a function type.
2949pub fn gen_std_dynamic_import_trampoline_arm64(
2950    vmoffsets: &VMOffsets,
2951    sig: &FunctionType,
2952    calling_convention: CallingConvention,
2953    progress_callback: Option<&CompilationProgressCallback>,
2954) -> Result<FunctionBody, CompileError> {
2955    let mut a = Assembler::new(0);
2956    let mut output_reporter = ChunkedOutputReporter::new(progress_callback);
2957    // Allocate argument array.
2958    let stack_offset: usize = 16 * std::cmp::max(sig.params().len(), sig.results().len());
2959    // Save LR and X26, as scratch register
2960    a.emit_stpdb(
2961        Size::S64,
2962        Location::GPR(GPR::X30),
2963        Location::GPR(GPR::X26),
2964        GPR::XzrSp,
2965        16,
2966    )?;
2967
2968    if stack_offset != 0 {
2969        if stack_offset < 0x1000 {
2970            a.emit_sub(
2971                Size::S64,
2972                Location::GPR(GPR::XzrSp),
2973                Location::Imm32(stack_offset as _),
2974                Location::GPR(GPR::XzrSp),
2975            )?;
2976        } else {
2977            a.emit_mov_imm(Location::GPR(GPR::X26), stack_offset as u64)?;
2978            a.emit_sub(
2979                Size::S64,
2980                Location::GPR(GPR::XzrSp),
2981                Location::GPR(GPR::X26),
2982                Location::GPR(GPR::XzrSp),
2983            )?;
2984        }
2985    }
2986
2987    // Copy arguments.
2988    if !sig.params().is_empty() {
2989        let mut argalloc = ArgumentRegisterAllocator::default();
2990        argalloc.next(Type::I64, calling_convention).unwrap(); // skip VMContext
2991
2992        let mut stack_param_count: usize = 0;
2993
2994        for (i, ty) in sig.params().iter().enumerate() {
2995            let source_loc = match argalloc.next(*ty, calling_convention)? {
2996                Some(ARM64Register::GPR(gpr)) => Location::GPR(gpr),
2997                Some(ARM64Register::NEON(neon)) => Location::SIMD(neon),
2998                None => {
2999                    let sz = match calling_convention {
3000                        CallingConvention::AppleAarch64 => match *ty {
3001                            Type::I32 | Type::F32 => Size::S32,
3002                            _ => {
3003                                stack_param_count = stack_param_count.next_multiple_of(8);
3004                                Size::S64
3005                            }
3006                        },
3007                        _ => Size::S64,
3008                    };
3009                    a.emit_ldr(
3010                        sz,
3011                        Location::GPR(GPR::X26),
3012                        Location::Memory(GPR::XzrSp, (stack_offset + 16 + stack_param_count) as _),
3013                    )?;
3014                    stack_param_count += match sz {
3015                        Size::S32 => 4,
3016                        Size::S64 => 8,
3017                        _ => codegen_error!(
3018                            "singlepass unreachable in gen_std_dynamic_import_trampoline_arm64"
3019                        ),
3020                    };
3021                    Location::GPR(GPR::X26)
3022                }
3023            };
3024            a.emit_str(
3025                Size::S64,
3026                source_loc,
3027                Location::Memory(GPR::XzrSp, (i * 16) as _),
3028            )?;
3029
3030            // Zero upper 64 bits.
3031            a.emit_str(
3032                Size::S64,
3033                Location::GPR(GPR::XzrSp),                       // XZR here
3034                Location::Memory(GPR::XzrSp, (i * 16 + 8) as _), // XSP here
3035            )?;
3036            output_reporter.check(a.offset().0)?;
3037        }
3038    }
3039
3040    #[allow(clippy::match_single_binding)]
3041    match calling_convention {
3042        _ => {
3043            // Load target address.
3044            let offset = vmoffsets.vmdynamicfunction_import_context_address();
3045            a.emit_ldur(Size::S64, Location::GPR(GPR::X26), GPR::X0, offset as i32)?;
3046            // Load values array.
3047            a.emit_add(
3048                Size::S64,
3049                Location::GPR(GPR::XzrSp),
3050                Location::Imm8(0),
3051                Location::GPR(GPR::X1),
3052            )?;
3053        }
3054    };
3055
3056    // Call target.
3057    a.emit_call_register(GPR::X26)?;
3058
3059    // Fetch return value.
3060    if !sig.results().is_empty() {
3061        assert_eq!(sig.results().len(), 1);
3062        a.emit_ldr(
3063            Size::S64,
3064            Location::GPR(GPR::X0),
3065            Location::Memory(GPR::XzrSp, 0),
3066        )?;
3067    }
3068
3069    // Release values array.
3070    if stack_offset != 0 {
3071        if stack_offset < 0x1000 {
3072            a.emit_add(
3073                Size::S64,
3074                Location::GPR(GPR::XzrSp),
3075                Location::Imm32(stack_offset as _),
3076                Location::GPR(GPR::XzrSp),
3077            )?;
3078        } else {
3079            a.emit_mov_imm(Location::GPR(GPR::X26), stack_offset as u64)?;
3080            a.emit_add(
3081                Size::S64,
3082                Location::GPR(GPR::XzrSp),
3083                Location::GPR(GPR::X26),
3084                Location::GPR(GPR::XzrSp),
3085            )?;
3086        }
3087    }
3088    a.emit_ldpia(
3089        Size::S64,
3090        Location::GPR(GPR::X30),
3091        Location::GPR(GPR::X26),
3092        GPR::XzrSp,
3093        16,
3094    )?;
3095
3096    // Return.
3097    a.emit_ret()?;
3098
3099    let mut body = a.finalize().unwrap();
3100    body.shrink_to_fit();
3101    output_reporter.finish(body.len())?;
3102    Ok(FunctionBody {
3103        body,
3104        unwind_info: None,
3105    })
3106}
3107// Singlepass calls import functions through a trampoline.
3108pub fn gen_import_call_trampoline_arm64(
3109    vmoffsets: &VMOffsets,
3110    index: FunctionIndex,
3111    sig: &FunctionType,
3112    calling_convention: CallingConvention,
3113    progress_callback: Option<&CompilationProgressCallback>,
3114) -> Result<CustomSection, CompileError> {
3115    let mut a = Assembler::new(0);
3116    let mut output_reporter = ChunkedOutputReporter::new(progress_callback);
3117
3118    // Singlepass internally treats all arguments as integers
3119    // For the standard System V calling convention requires
3120    //  floating point arguments to be passed in NEON registers.
3121    //  Translation is expensive, so only do it if needed.
3122    if sig
3123        .params()
3124        .iter()
3125        .any(|&x| x == Type::F32 || x == Type::F64)
3126    {
3127        #[allow(clippy::match_single_binding)]
3128        match calling_convention {
3129            _ => {
3130                // Allocate stack space for arguments.
3131                let mut stack_offset: u32 = if sig.params().len() > 7 {
3132                    7 * 8
3133                } else {
3134                    (sig.params().len() as u32) * 8
3135                };
3136                stack_offset = stack_offset.next_multiple_of(16);
3137                if stack_offset != 0 {
3138                    if stack_offset < 0x1000 {
3139                        a.emit_sub(
3140                            Size::S64,
3141                            Location::GPR(GPR::XzrSp),
3142                            Location::Imm32(stack_offset as u32),
3143                            Location::GPR(GPR::XzrSp),
3144                        )?;
3145                    } else {
3146                        a.emit_mov_imm(Location::GPR(GPR::X16), stack_offset as u64)?;
3147                        a.emit_sub(
3148                            Size::S64,
3149                            Location::GPR(GPR::XzrSp),
3150                            Location::GPR(GPR::X16),
3151                            Location::GPR(GPR::XzrSp),
3152                        )?;
3153                    }
3154                }
3155
3156                // Store all arguments to the stack to prevent overwrite.
3157                static PARAM_REGS: &[GPR] = &[
3158                    GPR::X1,
3159                    GPR::X2,
3160                    GPR::X3,
3161                    GPR::X4,
3162                    GPR::X5,
3163                    GPR::X6,
3164                    GPR::X7,
3165                ];
3166                let mut param_locations = vec![];
3167                for (i, param_reg) in PARAM_REGS.iter().enumerate().take(sig.params().len()) {
3168                    let loc = match i {
3169                        0..=6 => {
3170                            let loc = Location::Memory(GPR::XzrSp, (i * 8) as i32);
3171                            a.emit_str(Size::S64, Location::GPR(*param_reg), loc)?;
3172                            loc
3173                        }
3174                        _ => {
3175                            Location::Memory(GPR::XzrSp, (stack_offset + (i as u32 - 7) * 8) as i32)
3176                        }
3177                    };
3178                    param_locations.push(loc);
3179                    output_reporter.check(a.offset().0)?;
3180                }
3181
3182                // Copy arguments.
3183                let mut caller_stack_offset: u32 = 0;
3184                let mut argalloc = ArgumentRegisterAllocator::default();
3185                argalloc.next(Type::I64, calling_convention).unwrap(); // skip VMContext
3186                for (i, ty) in sig.params().iter().enumerate() {
3187                    let prev_loc = param_locations[i];
3188                    let targ = match argalloc.next(*ty, calling_convention)? {
3189                        Some(ARM64Register::GPR(gpr)) => Location::GPR(gpr),
3190                        Some(ARM64Register::NEON(neon)) => Location::SIMD(neon),
3191                        None => {
3192                            // No register can be allocated. Put this argument on the stack.
3193                            a.emit_ldr(Size::S64, Location::GPR(GPR::X16), prev_loc)?;
3194                            a.emit_str(
3195                                Size::S64,
3196                                Location::GPR(GPR::X16),
3197                                Location::Memory(
3198                                    GPR::XzrSp,
3199                                    (stack_offset + caller_stack_offset) as i32,
3200                                ),
3201                            )?;
3202                            caller_stack_offset += 8;
3203                            output_reporter.check(a.offset().0)?;
3204                            continue;
3205                        }
3206                    };
3207                    a.emit_ldr(Size::S64, targ, prev_loc)?;
3208                    output_reporter.check(a.offset().0)?;
3209                }
3210
3211                // Restore stack pointer.
3212                if stack_offset > 0 {
3213                    if stack_offset < 0x1000 {
3214                        a.emit_add(
3215                            Size::S64,
3216                            Location::GPR(GPR::XzrSp),
3217                            Location::Imm32(stack_offset as u32),
3218                            Location::GPR(GPR::XzrSp),
3219                        )?;
3220                    } else {
3221                        a.emit_mov_imm(Location::GPR(GPR::X16), stack_offset as u64)?;
3222                        a.emit_add(
3223                            Size::S64,
3224                            Location::GPR(GPR::XzrSp),
3225                            Location::GPR(GPR::X16),
3226                            Location::GPR(GPR::XzrSp),
3227                        )?;
3228                    }
3229                }
3230            }
3231        }
3232    }
3233
3234    // Emits a tail call trampoline that loads the address of the target import function
3235    // from Ctx and jumps to it.
3236
3237    let offset = vmoffsets.vmctx_vmfunction_import(index);
3238    // for ldr, offset needs to be a multiple of 8, which often is not
3239    // so use ldur, but then offset is limited to -255 .. +255. It will be positive here
3240    let offset =
3241        if (offset > 0) && ((offset < 0xF8) || (offset < 0x7FF8 && offset.is_multiple_of(8))) {
3242            offset
3243        } else {
3244            a.emit_mov_imm(Location::GPR(GPR::X16), (offset as i64) as u64)?;
3245            a.emit_add(
3246                Size::S64,
3247                Location::GPR(GPR::X0),
3248                Location::GPR(GPR::X16),
3249                Location::GPR(GPR::X0),
3250            )?;
3251            0
3252        };
3253    let ptr_arg_offset = vmctx_offset(offset)?;
3254    let vmctx_arg_offset = vmctx_offset(offset.saturating_add(8))?;
3255
3256    #[allow(clippy::match_single_binding)]
3257    match calling_convention {
3258        _ => {
3259            if (ptr_arg_offset as u32).is_multiple_of(8) {
3260                a.emit_ldr(
3261                    Size::S64,
3262                    Location::GPR(GPR::X16),
3263                    Location::Memory(GPR::X0, ptr_arg_offset), // function pointer
3264                )?;
3265                a.emit_ldr(
3266                    Size::S64,
3267                    Location::GPR(GPR::X0),
3268                    Location::Memory(GPR::X0, vmctx_arg_offset), // target vmctx
3269                )?;
3270            } else {
3271                a.emit_ldur(
3272                    Size::S64,
3273                    Location::GPR(GPR::X16),
3274                    GPR::X0,
3275                    ptr_arg_offset, // function pointer
3276                )?;
3277                a.emit_ldur(
3278                    Size::S64,
3279                    Location::GPR(GPR::X0),
3280                    GPR::X0,
3281                    vmctx_arg_offset, // target vmctx
3282                )?;
3283            }
3284        }
3285    }
3286    a.emit_b_register(GPR::X16)?;
3287
3288    let mut contents = a.finalize().unwrap();
3289    contents.shrink_to_fit();
3290    output_reporter.finish(contents.len())?;
3291    let section_body = SectionBody::new_with_vec(contents);
3292
3293    Ok(CustomSection {
3294        protection: CustomSectionProtection::ReadExecute,
3295        alignment: None,
3296        bytes: section_body,
3297        relocations: vec![],
3298    })
3299}
wasmer_compiler_singlepass/emitter_arm64.rs

wasmer_compiler_singlepass/
emitter_arm64.rs
