1pub use crate::{
2 arm64_decl::{ARM64Register, ArgumentRegisterAllocator, GPR, NEON},
3 location::{Multiplier, Reg},
4 machine::{Label, Offset},
5};
6use crate::{codegen_error, common_decl::Size, location::Location as AbstractLocation};
7pub use dynasmrt::aarch64::{encode_logical_immediate_32bit, encode_logical_immediate_64bit};
8use dynasmrt::{
9 AssemblyOffset, DynamicLabel, DynasmApi, DynasmLabelApi, VecAssembler,
10 aarch64::Aarch64Relocation,
11};
12use wasmer_compiler::types::{
13 function::FunctionBody,
14 section::{CustomSection, CustomSectionProtection, SectionBody},
15};
16use wasmer_types::{
17 CompileError, FunctionIndex, FunctionType, Type, VMOffsets, target::CallingConvention,
18};
19
20type Assembler = VecAssembler<Aarch64Relocation>;
21
22macro_rules! dynasm {
27 ($a:expr_2021 ; $($tt:tt)*) => {
28 dynasm::dynasm!(
29 $a
30 ; .arch aarch64
31 ; $($tt)*
32 )
33 };
34}
35
36pub type Location = AbstractLocation<GPR, NEON>;
37
38#[derive(Copy, Clone, Debug, Eq, PartialEq)]
39#[allow(dead_code)]
40#[repr(u8)]
41pub enum Condition {
42 Eq = 0,
45 Ne = 1,
47 Cs = 2,
49 Cc = 3,
51 Mi = 4,
53 Pl = 5,
55 Vs = 6,
57 Vc = 7,
59 Hi = 8,
61 Ls = 9,
63 Ge = 10,
65 Lt = 11,
67 Gt = 12,
69 Le = 13,
71 Al = 14,
73}
74
75#[derive(Copy, Clone, Debug, Eq, PartialEq)]
76#[allow(dead_code, clippy::upper_case_acronyms)]
77pub enum NeonOrMemory {
78 NEON(NEON),
79 Memory(GPR, i32),
80}
81
82#[derive(Copy, Clone, Debug)]
83#[allow(dead_code, clippy::upper_case_acronyms)]
84pub enum GPROrMemory {
85 GPR(GPR),
86 Memory(GPR, i32),
87}
88
89#[allow(unused)]
90pub trait EmitterARM64 {
91 fn get_label(&mut self) -> Label;
92 fn get_offset(&self) -> Offset;
93 fn get_jmp_instr_size(&self) -> u8;
94
95 fn finalize_function(&mut self);
96
97 fn emit_str(&mut self, sz: Size, reg: Location, addr: Location) -> Result<(), CompileError>;
98 fn emit_ldr(&mut self, sz: Size, reg: Location, addr: Location) -> Result<(), CompileError>;
99 fn emit_stur(
100 &mut self,
101 sz: Size,
102 reg: Location,
103 addr: GPR,
104 offset: i32,
105 ) -> Result<(), CompileError>;
106 fn emit_ldur(
107 &mut self,
108 sz: Size,
109 reg: Location,
110 addr: GPR,
111 offset: i32,
112 ) -> Result<(), CompileError>;
113 fn emit_strdb(
114 &mut self,
115 sz: Size,
116 reg: Location,
117 addr: GPR,
118 offset: u32,
119 ) -> Result<(), CompileError>;
120 fn emit_stria(
121 &mut self,
122 sz: Size,
123 reg: Location,
124 addr: GPR,
125 offset: u32,
126 ) -> Result<(), CompileError>;
127 fn emit_ldria(
128 &mut self,
129 sz: Size,
130 reg: Location,
131 addr: GPR,
132 offset: u32,
133 ) -> Result<(), CompileError>;
134 fn emit_stpdb(
135 &mut self,
136 sz: Size,
137 reg1: Location,
138 reg2: Location,
139 addr: GPR,
140 offset: u32,
141 ) -> Result<(), CompileError>;
142 fn emit_ldpia(
143 &mut self,
144 sz: Size,
145 reg1: Location,
146 reg2: Location,
147 addr: GPR,
148 offset: u32,
149 ) -> Result<(), CompileError>;
150
151 fn emit_ldrb(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
152 fn emit_ldrh(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
153 fn emit_ldrsb(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
154 fn emit_ldrsh(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
155 fn emit_ldrsw(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
156 fn emit_strb(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
157 fn emit_strh(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
158
159 fn emit_ldaxr(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
160 fn emit_ldaxrb(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
161 fn emit_ldaxrh(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
162 fn emit_stlxr(
163 &mut self,
164 sz: Size,
165 status: Location,
166 reg: Location,
167 dst: Location,
168 ) -> Result<(), CompileError>;
169 fn emit_stlxrb(
170 &mut self,
171 sz: Size,
172 status: Location,
173 reg: Location,
174 dst: Location,
175 ) -> Result<(), CompileError>;
176 fn emit_stlxrh(
177 &mut self,
178 sz: Size,
179 status: Location,
180 reg: Location,
181 dst: Location,
182 ) -> Result<(), CompileError>;
183
184 fn emit_mov(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
185
186 fn emit_movn(&mut self, sz: Size, reg: Location, val: u32) -> Result<(), CompileError>;
187 fn emit_movz(&mut self, reg: Location, val: u32) -> Result<(), CompileError>;
188 fn emit_movk(&mut self, reg: Location, val: u32, shift: u32) -> Result<(), CompileError>;
189
190 fn emit_mov_imm(&mut self, dst: Location, val: u64) -> Result<(), CompileError>;
191
192 fn emit_add(
193 &mut self,
194 sz: Size,
195 src1: Location,
196 src2: Location,
197 dst: Location,
198 ) -> Result<(), CompileError>;
199 fn emit_sub(
200 &mut self,
201 sz: Size,
202 src1: Location,
203 src2: Location,
204 dst: Location,
205 ) -> Result<(), CompileError>;
206 fn emit_mul(
207 &mut self,
208 sz: Size,
209 src1: Location,
210 src2: Location,
211 dst: Location,
212 ) -> Result<(), CompileError>;
213 fn emit_adds(
214 &mut self,
215 sz: Size,
216 src1: Location,
217 src2: Location,
218 dst: Location,
219 ) -> Result<(), CompileError>;
220 fn emit_subs(
221 &mut self,
222 sz: Size,
223 src1: Location,
224 src2: Location,
225 dst: Location,
226 ) -> Result<(), CompileError>;
227
228 fn emit_add_lsl(
229 &mut self,
230 sz: Size,
231 src1: Location,
232 src2: Location,
233 lsl: u32,
234 dst: Location,
235 ) -> Result<(), CompileError>;
236
237 fn emit_cmp(&mut self, sz: Size, left: Location, right: Location) -> Result<(), CompileError>;
238 fn emit_tst(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
239
240 fn emit_lsl(
241 &mut self,
242 sz: Size,
243 src1: Location,
244 src2: Location,
245 dst: Location,
246 ) -> Result<(), CompileError>;
247 fn emit_lsr(
248 &mut self,
249 sz: Size,
250 src1: Location,
251 src2: Location,
252 dst: Location,
253 ) -> Result<(), CompileError>;
254 fn emit_asr(
255 &mut self,
256 sz: Size,
257 src1: Location,
258 src2: Location,
259 dst: Location,
260 ) -> Result<(), CompileError>;
261 fn emit_ror(
262 &mut self,
263 sz: Size,
264 src1: Location,
265 src2: Location,
266 dst: Location,
267 ) -> Result<(), CompileError>;
268
269 fn emit_or(
270 &mut self,
271 sz: Size,
272 src1: Location,
273 src2: Location,
274 dst: Location,
275 ) -> Result<(), CompileError>;
276 fn emit_and(
277 &mut self,
278 sz: Size,
279 src1: Location,
280 src2: Location,
281 dst: Location,
282 ) -> Result<(), CompileError>;
283 fn emit_eor(
284 &mut self,
285 sz: Size,
286 src1: Location,
287 src2: Location,
288 dst: Location,
289 ) -> Result<(), CompileError>;
290
291 fn emit_bfc(
292 &mut self,
293 se: Size,
294 lsb: u32,
295 width: u32,
296 dst: Location,
297 ) -> Result<(), CompileError>;
298 fn emit_bfi(
299 &mut self,
300 se: Size,
301 src: Location,
302 lsb: u32,
303 width: u32,
304 dst: Location,
305 ) -> Result<(), CompileError>;
306
307 fn emit_udiv(
308 &mut self,
309 sz: Size,
310 src1: Location,
311 src2: Location,
312 dst: Location,
313 ) -> Result<(), CompileError>;
314 fn emit_sdiv(
315 &mut self,
316 sz: Size,
317 src1: Location,
318 src2: Location,
319 dst: Location,
320 ) -> Result<(), CompileError>;
321 fn emit_msub(
323 &mut self,
324 sz: Size,
325 a: Location,
326 b: Location,
327 c: Location,
328 dst: Location,
329 ) -> Result<(), CompileError>;
330
331 fn emit_sxtb(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
332 fn emit_sxth(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
333 fn emit_sxtw(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
334 fn emit_uxtb(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
335 fn emit_uxth(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
336
337 fn emit_cset(&mut self, sz: Size, dst: Location, cond: Condition) -> Result<(), CompileError>;
338 fn emit_csetm(&mut self, sz: Size, dst: Location, cond: Condition) -> Result<(), CompileError>;
339 fn emit_cinc(
340 &mut self,
341 sz: Size,
342 src: Location,
343 dst: Location,
344 cond: Condition,
345 ) -> Result<(), CompileError>;
346 fn emit_clz(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
347 fn emit_rbit(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
348
349 fn emit_label(&mut self, label: Label) -> Result<(), CompileError>;
350 fn emit_load_label(&mut self, reg: GPR, label: Label) -> Result<(), CompileError>;
351 fn emit_b_label(&mut self, label: Label) -> Result<(), CompileError>;
352 fn emit_cbz_label(&mut self, sz: Size, reg: Location, label: Label)
353 -> Result<(), CompileError>;
354 fn emit_cbnz_label(
355 &mut self,
356 sz: Size,
357 reg: Location,
358 label: Label,
359 ) -> Result<(), CompileError>;
360 fn emit_cbz_label_far(
361 &mut self,
362 sz: Size,
363 reg: Location,
364 label: Label,
365 ) -> Result<(), CompileError>;
366 fn emit_tbz_label(
367 &mut self,
368 sz: Size,
369 reg: Location,
370 n: u32,
371 label: Label,
372 ) -> Result<(), CompileError>;
373 fn emit_tbnz_label(
374 &mut self,
375 sz: Size,
376 reg: Location,
377 n: u32,
378 label: Label,
379 ) -> Result<(), CompileError>;
380 fn emit_bcond_label(&mut self, condition: Condition, label: Label) -> Result<(), CompileError>;
381 fn emit_bcond_label_far(
382 &mut self,
383 condition: Condition,
384 label: Label,
385 ) -> Result<(), CompileError>;
386 fn emit_b_register(&mut self, reg: GPR) -> Result<(), CompileError>;
387 fn emit_call_label(&mut self, label: Label) -> Result<(), CompileError>;
388 fn emit_call_register(&mut self, reg: GPR) -> Result<(), CompileError>;
389 fn emit_ret(&mut self) -> Result<(), CompileError>;
390
391 fn emit_udf(&mut self, payload: u16) -> Result<(), CompileError>;
392 fn emit_dmb(&mut self) -> Result<(), CompileError>;
393 fn emit_brk(&mut self) -> Result<(), CompileError>;
394
395 fn emit_fcmp(&mut self, sz: Size, src1: Location, src2: Location) -> Result<(), CompileError>;
396 fn emit_fneg(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
397 fn emit_fsqrt(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
398
399 fn emit_fadd(
400 &mut self,
401 sz: Size,
402 src1: Location,
403 src2: Location,
404 dst: Location,
405 ) -> Result<(), CompileError>;
406 fn emit_fsub(
407 &mut self,
408 sz: Size,
409 src1: Location,
410 src2: Location,
411 dst: Location,
412 ) -> Result<(), CompileError>;
413 fn emit_fmul(
414 &mut self,
415 sz: Size,
416 src1: Location,
417 src2: Location,
418 dst: Location,
419 ) -> Result<(), CompileError>;
420 fn emit_fdiv(
421 &mut self,
422 sz: Size,
423 src1: Location,
424 src2: Location,
425 dst: Location,
426 ) -> Result<(), CompileError>;
427
428 fn emit_fmin(
429 &mut self,
430 sz: Size,
431 src1: Location,
432 src2: Location,
433 dst: Location,
434 ) -> Result<(), CompileError>;
435 fn emit_fmax(
436 &mut self,
437 sz: Size,
438 src1: Location,
439 src2: Location,
440 dst: Location,
441 ) -> Result<(), CompileError>;
442
443 fn emit_frintz(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
444 fn emit_frintn(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
445 fn emit_frintm(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
446 fn emit_frintp(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
447
448 fn emit_scvtf(
449 &mut self,
450 sz_in: Size,
451 src: Location,
452 sz_out: Size,
453 dst: Location,
454 ) -> Result<(), CompileError>;
455 fn emit_ucvtf(
456 &mut self,
457 sz_in: Size,
458 src: Location,
459 sz_out: Size,
460 dst: Location,
461 ) -> Result<(), CompileError>;
462 fn emit_fcvt(&mut self, sz_in: Size, src: Location, dst: Location) -> Result<(), CompileError>;
463 fn emit_fcvtzs(
464 &mut self,
465 sz_in: Size,
466 src: Location,
467 sz_out: Size,
468 dst: Location,
469 ) -> Result<(), CompileError>;
470 fn emit_fcvtzu(
471 &mut self,
472 sz_in: Size,
473 src: Location,
474 sz_out: Size,
475 dst: Location,
476 ) -> Result<(), CompileError>;
477
478 fn emit_fmov(
479 &mut self,
480 src_size: Size,
481 src: Location,
482 dst_size: Size,
483 dst: Location,
484 ) -> Result<(), CompileError>;
485 fn emit_cnt(&mut self, src: NEON, dst: NEON) -> Result<(), CompileError>;
486 fn emit_addv(&mut self, src: NEON, dst: NEON) -> Result<(), CompileError>;
487 fn emit_read_fpcr(&mut self, reg: GPR) -> Result<(), CompileError>;
488 fn emit_write_fpcr(&mut self, reg: GPR) -> Result<(), CompileError>;
489 fn emit_read_fpsr(&mut self, reg: GPR) -> Result<(), CompileError>;
490 fn emit_write_fpsr(&mut self, reg: GPR) -> Result<(), CompileError>;
491
492 fn arch_supports_canonicalize_nan(&self) -> bool {
493 true
494 }
495
496 fn arch_requires_indirect_call_trampoline(&self) -> bool {
497 false
498 }
499
500 fn arch_emit_indirect_call_with_trampoline(
501 &mut self,
502 _loc: Location,
503 ) -> Result<(), CompileError> {
504 codegen_error!("singlepass arch_emit_indirect_call_with_trampoline unimplemented")
505 }
506}
507
508impl EmitterARM64 for Assembler {
509 fn get_label(&mut self) -> DynamicLabel {
510 self.new_dynamic_label()
511 }
512
513 fn get_offset(&self) -> AssemblyOffset {
514 self.offset()
515 }
516
517 fn get_jmp_instr_size(&self) -> u8 {
518 4 }
520
521 fn finalize_function(&mut self) {
522 dynasm!(
523 self
524 ; const_neg_one_32:
525 ; .i32 -1
526 ; const_zero_32:
527 ; .i32 0
528 ; const_pos_one_32:
529 ; .i32 1
530 );
531 }
532
533 fn emit_str(&mut self, sz: Size, reg: Location, addr: Location) -> Result<(), CompileError> {
534 match (sz, reg, addr) {
535 (Size::S64, Location::GPR(reg), Location::Memory(addr, disp)) => {
536 let disp = disp as u32;
537 assert!((disp & 0x7) == 0 && (disp < 0x8000));
538 dynasm!(self ; str X(reg), [X(addr), disp]);
539 }
540 (Size::S32, Location::GPR(reg), Location::Memory(addr, disp)) => {
541 let disp = disp as u32;
542 assert!((disp & 0x3) == 0 && (disp < 0x4000));
543 dynasm!(self ; str W(reg), [X(addr), disp]);
544 }
545 (Size::S16, Location::GPR(reg), Location::Memory(addr, disp)) => {
546 let disp = disp as u32;
547 assert!((disp & 0x1) == 0 && (disp < 0x2000));
548 dynasm!(self ; strh W(reg), [X(addr), disp]);
549 }
550 (Size::S8, Location::GPR(reg), Location::Memory(addr, disp)) => {
551 let disp = disp as u32;
552 assert!(disp < 0x1000);
553 dynasm!(self ; strb W(reg), [X(addr), disp]);
554 }
555 (Size::S64, Location::SIMD(reg), Location::Memory(addr, disp)) => {
556 let disp = disp as u32;
557 assert!((disp & 0x7) == 0 && (disp < 0x8000));
558 dynasm!(self ; str D(reg), [X(addr), disp]);
559 }
560 (Size::S32, Location::SIMD(reg), Location::Memory(addr, disp)) => {
561 let disp = disp as u32;
562 assert!((disp & 0x3) == 0 && (disp < 0x4000));
563 dynasm!(self ; str S(reg), [X(addr), disp]);
564 }
565 (Size::S64, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
566 assert!(offs == 0);
567 let mult = mult as u32;
568 match mult {
569 0 => dynasm!(self ; str X(reg), [X(addr)]),
570 1 => dynasm!(self ; str X(reg), [X(addr), X(r2)]),
571 _ => dynasm!(self ; str X(reg), [X(addr), X(r2), LSL mult]),
572 };
573 }
574 (Size::S32, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
575 assert!(offs == 0);
576 let mult = mult as u32;
577 match mult {
578 0 => dynasm!(self ; str W(reg), [X(addr)]),
579 1 => dynasm!(self ; str W(reg), [X(addr), X(r2)]),
580 _ => dynasm!(self ; str W(reg), [X(addr), X(r2), LSL mult]),
581 };
582 }
583 _ => codegen_error!("singlepass can't emit STR {:?}, {:?}, {:?}", sz, reg, addr),
584 }
585 Ok(())
586 }
587 fn emit_ldr(&mut self, sz: Size, reg: Location, addr: Location) -> Result<(), CompileError> {
588 match (sz, reg, addr) {
589 (Size::S64, Location::GPR(reg), Location::Memory(addr, disp)) => {
590 assert!((disp & 0x7) == 0 && (disp < 0x8000));
591 let disp = disp as u32;
592 dynasm!(self ; ldr X(reg), [X(addr), disp]);
593 }
594 (Size::S32, Location::GPR(reg), Location::Memory(addr, disp)) => {
595 assert!((disp & 0x3) == 0 && (disp < 0x4000));
596 let disp = disp as u32;
597 dynasm!(self ; ldr W(reg), [X(addr), disp]);
598 }
599 (Size::S16, Location::GPR(reg), Location::Memory(addr, disp)) => {
600 assert!((disp & 0x1 == 0) && (disp < 0x2000));
601 let disp = disp as u32;
602 dynasm!(self ; ldrh W(reg), [X(addr), disp]);
603 }
604 (Size::S8, Location::GPR(reg), Location::Memory(addr, disp)) => {
605 assert!(disp < 0x1000);
606 let disp = disp as u32;
607 dynasm!(self ; ldrb W(reg), [X(addr), disp]);
608 }
609 (Size::S64, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
610 assert!(offs == 0);
611 let mult = mult as u32;
612 match mult {
613 0 => dynasm!(self ; ldr X(reg), [X(addr)]),
614 1 => dynasm!(self ; ldr X(reg), [X(addr), X(r2)]),
615 _ => dynasm!(self ; ldr X(reg), [X(addr), X(r2), LSL mult]),
616 };
617 }
618 (Size::S32, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
619 assert!(offs == 0);
620 let mult = mult as u32;
621 match mult {
622 0 => dynasm!(self ; ldr W(reg), [X(addr)]),
623 1 => dynasm!(self ; ldr W(reg), [X(addr), X(r2)]),
624 _ => dynasm!(self ; ldr W(reg), [X(addr), X(r2), LSL mult]),
625 };
626 }
627 (Size::S64, Location::SIMD(reg), Location::Memory(addr, disp)) => {
628 let disp = disp as u32;
629 assert!((disp & 0x7) == 0 && (disp < 0x8000));
630 dynasm!(self ; ldr D(reg), [X(addr), disp]);
631 }
632 (Size::S32, Location::SIMD(reg), Location::Memory(addr, disp)) => {
633 let disp = disp as u32;
634 assert!((disp & 0x3) == 0 && (disp < 0x4000));
635 dynasm!(self ; ldr S(reg), [X(addr), disp]);
636 }
637 (Size::S64, Location::SIMD(reg), Location::Memory2(addr, r2, mult, offs)) => {
638 assert!(offs == 0);
639 let mult = mult as u32;
640 match mult {
641 0 => dynasm!(self ; ldr D(reg), [X(addr)]),
642 1 => dynasm!(self ; ldr D(reg), [X(addr), X(r2)]),
643 _ => dynasm!(self ; ldr D(reg), [X(addr), X(r2), LSL mult]),
644 };
645 }
646 (Size::S32, Location::SIMD(reg), Location::Memory2(addr, r2, mult, offs)) => {
647 assert!(offs == 0);
648 let mult = mult as u32;
649 match mult {
650 0 => dynasm!(self ; ldr S(reg), [X(addr)]),
651 1 => dynasm!(self ; ldr S(reg), [X(addr), X(r2)]),
652 _ => dynasm!(self ; ldr S(reg), [X(addr), X(r2), LSL mult]),
653 };
654 }
655 _ => codegen_error!("singlepass can't emit LDR {:?}, {:?}, {:?}", sz, reg, addr),
656 }
657 Ok(())
658 }
659 fn emit_stur(
660 &mut self,
661 sz: Size,
662 reg: Location,
663 addr: GPR,
664 offset: i32,
665 ) -> Result<(), CompileError> {
666 assert!((-255..=255).contains(&offset));
667 match (sz, reg) {
668 (Size::S64, Location::GPR(reg)) => {
669 dynasm!(self ; stur X(reg), [X(addr), offset]);
670 }
671 (Size::S32, Location::GPR(reg)) => {
672 dynasm!(self ; stur W(reg), [X(addr), offset]);
673 }
674 (Size::S64, Location::SIMD(reg)) => {
675 dynasm!(self ; stur D(reg), [X(addr), offset]);
676 }
677 (Size::S32, Location::SIMD(reg)) => {
678 dynasm!(self ; stur S(reg), [X(addr), offset]);
679 }
680 _ => codegen_error!(
681 "singlepass can't emit STUR {:?}, {:?}, {:?}, {:?}",
682 sz,
683 reg,
684 addr,
685 offset
686 ),
687 }
688 Ok(())
689 }
690 fn emit_ldur(
691 &mut self,
692 sz: Size,
693 reg: Location,
694 addr: GPR,
695 offset: i32,
696 ) -> Result<(), CompileError> {
697 assert!((-255..=255).contains(&offset));
698 match (sz, reg) {
699 (Size::S64, Location::GPR(reg)) => {
700 dynasm!(self ; ldur X(reg), [X(addr), offset]);
701 }
702 (Size::S32, Location::GPR(reg)) => {
703 dynasm!(self ; ldur W(reg), [X(addr), offset]);
704 }
705 (Size::S64, Location::SIMD(reg)) => {
706 dynasm!(self ; ldur D(reg), [X(addr), offset]);
707 }
708 (Size::S32, Location::SIMD(reg)) => {
709 dynasm!(self ; ldur S(reg), [X(addr), offset]);
710 }
711 _ => codegen_error!(
712 "singlepass can't emit LDUR {:?}, {:?}, {:?}, {:?}",
713 sz,
714 reg,
715 addr,
716 offset
717 ),
718 }
719 Ok(())
720 }
721
722 fn emit_strdb(
723 &mut self,
724 sz: Size,
725 reg: Location,
726 addr: GPR,
727 offset: u32,
728 ) -> Result<(), CompileError> {
729 assert!(offset <= 255);
730 match (sz, reg) {
731 (Size::S64, Location::GPR(reg)) => {
732 dynasm!(self ; str X(reg), [X(addr), -(offset as i32)]!);
733 }
734 (Size::S64, Location::SIMD(reg)) => {
735 dynasm!(self ; str D(reg), [X(addr), -(offset as i32)]!);
736 }
737 _ => codegen_error!("singlepass can't emit STRDB"),
738 }
739 Ok(())
740 }
741 fn emit_stria(
742 &mut self,
743 sz: Size,
744 reg: Location,
745 addr: GPR,
746 offset: u32,
747 ) -> Result<(), CompileError> {
748 assert!(offset <= 255);
749 match (sz, reg) {
750 (Size::S64, Location::GPR(reg)) => {
751 dynasm!(self ; str X(reg), [X(addr)], offset as i32);
752 }
753 (Size::S64, Location::SIMD(reg)) => {
754 dynasm!(self ; str D(reg), [X(addr)], offset as i32);
755 }
756 _ => codegen_error!("singlepass can't emit STRIA"),
757 }
758 Ok(())
759 }
760 fn emit_ldria(
761 &mut self,
762 sz: Size,
763 reg: Location,
764 addr: GPR,
765 offset: u32,
766 ) -> Result<(), CompileError> {
767 assert!(offset <= 255);
768 match (sz, reg) {
769 (Size::S64, Location::GPR(reg)) => {
770 dynasm!(self ; ldr X(reg), [X(addr)], offset as _);
771 }
772 (Size::S64, Location::SIMD(reg)) => {
773 dynasm!(self ; ldr D(reg), [X(addr)], offset as _);
774 }
775 _ => codegen_error!("singlepass can't emit LDRIA"),
776 }
777 Ok(())
778 }
779
780 fn emit_stpdb(
781 &mut self,
782 sz: Size,
783 reg1: Location,
784 reg2: Location,
785 addr: GPR,
786 offset: u32,
787 ) -> Result<(), CompileError> {
788 assert!(offset <= 255);
789 match (sz, reg1, reg2) {
790 (Size::S64, Location::GPR(reg1), Location::GPR(reg2)) => {
791 dynasm!(self ; stp X(reg1), X(reg2), [X(addr), -(offset as i32)]!);
792 }
793 _ => codegen_error!("singlepass can't emit STPDB"),
794 }
795 Ok(())
796 }
797 fn emit_ldpia(
798 &mut self,
799 sz: Size,
800 reg1: Location,
801 reg2: Location,
802 addr: GPR,
803 offset: u32,
804 ) -> Result<(), CompileError> {
805 assert!(offset <= 255);
806 match (sz, reg1, reg2) {
807 (Size::S64, Location::GPR(reg1), Location::GPR(reg2)) => {
808 dynasm!(self ; ldp X(reg1), X(reg2), [X(addr)], offset as _);
809 }
810 _ => codegen_error!("singlepass can't emit LDPIA"),
811 }
812 Ok(())
813 }
814
815 fn emit_ldrb(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
816 match (reg, dst) {
817 (Location::GPR(reg), Location::Memory(addr, offset)) => {
818 let offset = offset as u32;
819 assert!(offset < 0x1000);
820 dynasm!(self ; ldrb W(reg), [X(addr), offset]);
821 }
822 (Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
823 assert!(offs == 0);
824 let mult = mult as u32;
825 match mult {
826 0 => dynasm!(self ; ldrb W(reg), [X(addr)]),
827 1 => dynasm!(self ; ldrb W(reg), [X(addr), X(r2)]),
828 _ => dynasm!(self ; ldrb W(reg), [X(addr), X(r2), LSL mult]),
829 };
830 }
831 _ => codegen_error!("singlepass can't emit LDRB {:?}, {:?}", reg, dst),
832 }
833 Ok(())
834 }
835 fn emit_ldrh(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
836 match (reg, dst) {
837 (Location::GPR(reg), Location::Memory(addr, offset)) => {
838 let offset = offset as u32;
839 assert!((offset & 1 == 0) && (offset < 0x2000));
840 dynasm!(self ; ldrh W(reg), [X(addr), offset]);
841 }
842 (Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
843 assert!(offs == 0);
844 let mult = mult as u32;
845 match mult {
846 0 => dynasm!(self ; ldrh W(reg), [X(addr)]),
847 1 => dynasm!(self ; ldrh W(reg), [X(addr), X(r2)]),
848 _ => dynasm!(self ; ldrh W(reg), [X(addr), X(r2), LSL mult]),
849 };
850 }
851 _ => codegen_error!("singlepass can't emit LDRH {:?}, {:?}", reg, dst),
852 }
853 Ok(())
854 }
855 fn emit_ldrsb(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
856 match (sz, reg, dst) {
857 (Size::S64, Location::GPR(reg), Location::Memory(addr, offset)) => {
858 let offset = offset as u32;
859 assert!(offset < 0x1000);
860 dynasm!(self ; ldrsb X(reg), [X(addr), offset]);
861 }
862 (Size::S32, Location::GPR(reg), Location::Memory(addr, offset)) => {
863 let offset = offset as u32;
864 assert!(offset < 0x1000);
865 dynasm!(self ; ldrsb W(reg), [X(addr), offset]);
866 }
867 (Size::S64, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
868 assert!(offs == 0);
869 let mult = mult as u32;
870 match mult {
871 0 => dynasm!(self ; ldrsb X(reg), [X(addr)]),
872 1 => dynasm!(self ; ldrsb X(reg), [X(addr), X(r2)]),
873 _ => dynasm!(self ; ldrsb X(reg), [X(addr), X(r2), LSL mult]),
874 };
875 }
876 (Size::S32, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
877 assert!(offs == 0);
878 let mult = mult as u32;
879 match mult {
880 0 => dynasm!(self ; ldrsb W(reg), [X(addr)]),
881 1 => dynasm!(self ; ldrsb W(reg), [X(addr), X(r2)]),
882 _ => dynasm!(self ; ldrsb W(reg), [X(addr), X(r2), LSL mult]),
883 };
884 }
885 _ => codegen_error!("singlepass can't emit LDRSB {:?}, {:?}, {:?}", sz, reg, dst),
886 }
887 Ok(())
888 }
889 fn emit_ldrsh(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
890 match (sz, reg, dst) {
891 (Size::S64, Location::GPR(reg), Location::Memory(addr, offset)) => {
892 let offset = offset as u32;
893 assert!((offset & 1 == 0) && (offset < 0x2000));
894 dynasm!(self ; ldrsh X(reg), [X(addr), offset]);
895 }
896 (Size::S32, Location::GPR(reg), Location::Memory(addr, offset)) => {
897 let offset = offset as u32;
898 assert!((offset & 1 == 0) && (offset < 0x2000));
899 dynasm!(self ; ldrsh W(reg), [X(addr), offset]);
900 }
901 (Size::S64, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
902 assert!(offs == 0);
903 let mult = mult as u32;
904 match mult {
905 0 => dynasm!(self ; ldrsh X(reg), [X(addr)]),
906 1 => dynasm!(self ; ldrsh X(reg), [X(addr), X(r2)]),
907 _ => dynasm!(self ; ldrsh X(reg), [X(addr), X(r2), LSL mult]),
908 };
909 }
910 (Size::S32, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
911 assert!(offs == 0);
912 let mult = mult as u32;
913 match mult {
914 0 => dynasm!(self ; ldrsh W(reg), [X(addr)]),
915 1 => dynasm!(self ; ldrsh W(reg), [X(addr), X(r2)]),
916 _ => dynasm!(self ; ldrsh W(reg), [X(addr), X(r2), LSL mult]),
917 };
918 }
919 _ => codegen_error!("singlepass can't emit LDRSH {:?}, {:?}, {:?}", sz, reg, dst),
920 }
921 Ok(())
922 }
923 fn emit_ldrsw(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
924 match (sz, reg, dst) {
925 (Size::S64, Location::GPR(reg), Location::Memory(addr, offset)) => {
926 let offset = offset as u32;
927 assert!((offset & 3 == 0) && (offset < 0x4000));
928 dynasm!(self ; ldrsw X(reg), [X(addr), offset]);
929 }
930 (Size::S64, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
931 assert!(offs == 0);
932 let mult = mult as u32;
933 match mult {
934 0 => dynasm!(self ; ldrsw X(reg), [X(addr)]),
935 1 => dynasm!(self ; ldrsw X(reg), [X(addr), X(r2)]),
936 _ => dynasm!(self ; ldrsw X(reg), [X(addr), X(r2), LSL mult]),
937 };
938 }
939 _ => codegen_error!("singlepass can't emit LDRSW {:?}, {:?}, {:?}", sz, reg, dst),
940 }
941 Ok(())
942 }
943 fn emit_strb(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
944 match (reg, dst) {
945 (Location::GPR(reg), Location::Memory(addr, offset)) => {
946 let offset = offset as u32;
947 assert!(offset < 0x1000);
948 dynasm!(self ; strb W(reg), [X(addr), offset]);
949 }
950 (Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
951 assert!(offs == 0);
952 let mult = mult as u32;
953 match mult {
954 0 => dynasm!(self ; strb W(reg), [X(addr)]),
955 1 => dynasm!(self ; strb W(reg), [X(addr), X(r2)]),
956 _ => dynasm!(self ; strb W(reg), [X(addr), X(r2), LSL mult]),
957 };
958 }
959 _ => codegen_error!("singlepass can't emit STRB {:?}, {:?}", reg, dst),
960 }
961 Ok(())
962 }
963 fn emit_strh(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
964 match (reg, dst) {
965 (Location::GPR(reg), Location::Memory(addr, offset)) => {
966 let offset = offset as u32;
967 assert!((offset & 1 == 0) && (offset < 0x2000));
968 dynasm!(self ; strh W(reg), [X(addr), offset]);
969 }
970 (Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
971 assert!(offs == 0);
972 let mult = mult as u32;
973 match mult {
974 0 => dynasm!(self ; strh W(reg), [X(addr)]),
975 1 => dynasm!(self ; strh W(reg), [X(addr), X(r2)]),
976 _ => dynasm!(self ; strh W(reg), [X(addr), X(r2), LSL mult]),
977 };
978 }
979 _ => codegen_error!("singlepass can't emit STRH {:?}, {:?}", reg, dst),
980 }
981 Ok(())
982 }
983
984 fn emit_ldaxr(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
985 match (sz, reg, dst) {
986 (Size::S32, Location::GPR(reg), Location::GPR(dst)) => {
987 dynasm!(self ; ldaxr W(reg), [X(dst)]);
988 }
989 (Size::S64, Location::GPR(reg), Location::GPR(dst)) => {
990 dynasm!(self ; ldaxr X(reg), [X(dst)]);
991 }
992 _ => codegen_error!("singlepass can't emit LDAXR {:?}, {:?}", reg, dst),
993 }
994 Ok(())
995 }
996 fn emit_ldaxrb(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
997 match (reg, dst) {
998 (Location::GPR(reg), Location::GPR(dst)) => {
999 dynasm!(self ; ldaxrb W(reg), [X(dst)]);
1000 }
1001 _ => codegen_error!("singlepass can't emit LDAXRB {:?}, {:?}", reg, dst),
1002 }
1003 Ok(())
1004 }
1005 fn emit_ldaxrh(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
1006 match (reg, dst) {
1007 (Location::GPR(reg), Location::GPR(dst)) => {
1008 dynasm!(self ; ldaxrh W(reg), [X(dst)]);
1009 }
1010 _ => codegen_error!("singlepass can't emit LDAXRH {:?}, {:?}", reg, dst),
1011 }
1012 Ok(())
1013 }
1014 fn emit_stlxr(
1015 &mut self,
1016 sz: Size,
1017 status: Location,
1018 reg: Location,
1019 dst: Location,
1020 ) -> Result<(), CompileError> {
1021 match (sz, status, reg, dst) {
1022 (Size::S32, Location::GPR(status), Location::GPR(reg), Location::GPR(dst)) => {
1023 dynasm!(self ; stlxr W(status), W(reg), [X(dst)]);
1024 }
1025 (Size::S64, Location::GPR(status), Location::GPR(reg), Location::GPR(dst)) => {
1026 dynasm!(self ; stlxr W(status), X(reg), [X(dst)]);
1027 }
1028 _ => codegen_error!("singlepass can't emit STLXR {:?}, {:?}", reg, dst),
1029 }
1030 Ok(())
1031 }
1032 fn emit_stlxrb(
1033 &mut self,
1034 _sz: Size,
1035 status: Location,
1036 reg: Location,
1037 dst: Location,
1038 ) -> Result<(), CompileError> {
1039 match (status, reg, dst) {
1040 (Location::GPR(status), Location::GPR(reg), Location::GPR(dst)) => {
1041 dynasm!(self ; stlxrb W(status), W(reg), [X(dst)]);
1042 }
1043 _ => codegen_error!("singlepass can't emit STLXRB {:?}, {:?}", reg, dst),
1044 }
1045 Ok(())
1046 }
1047 fn emit_stlxrh(
1048 &mut self,
1049 _sz: Size,
1050 status: Location,
1051 reg: Location,
1052 dst: Location,
1053 ) -> Result<(), CompileError> {
1054 match (status, reg, dst) {
1055 (Location::GPR(status), Location::GPR(reg), Location::GPR(dst)) => {
1056 dynasm!(self ; stlxrh W(status), W(reg), [X(dst)]);
1057 }
1058 _ => codegen_error!("singlepass can't emit STLXRH {:?}, {:?}", reg, dst),
1059 }
1060 Ok(())
1061 }
1062
1063 fn emit_mov(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1064 match (sz, src, dst) {
1065 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1066 dynasm!(self ; mov X(dst), X(src));
1067 }
1068 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1069 dynasm!(self ; mov W(dst), W(src));
1070 }
1071 (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
1072 dynasm!(self ; mov V(dst).D[0], V(src).D[0]);
1073 }
1074 (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
1075 dynasm!(self ; mov V(dst).S[0], V(src).S[0]);
1076 }
1077 (Size::S64, Location::GPR(src), Location::SIMD(dst)) => {
1078 dynasm!(self ; mov V(dst).D[0], X(src));
1079 }
1080 (Size::S32, Location::GPR(src), Location::SIMD(dst)) => {
1081 dynasm!(self ; mov V(dst).S[0], W(src));
1082 }
1083 (Size::S64, Location::SIMD(src), Location::GPR(dst)) => {
1084 dynasm!(self ; mov X(dst), V(src).D[0]);
1085 }
1086 (Size::S32, Location::SIMD(src), Location::GPR(dst)) => {
1087 dynasm!(self ; mov W(dst), V(src).S[0]);
1088 }
1089 (Size::S32, Location::Imm32(val), Location::GPR(dst)) => {
1090 if val < 0x1000 || encode_logical_immediate_32bit(val as _).is_some() {
1091 dynasm!(self ; mov W(dst), val);
1092 } else {
1093 dynasm!(self
1094 ; movz W(dst), val
1095 ; movk W(dst), val, lsl #16);
1096 }
1097 }
1098 (Size::S64, Location::Imm32(val), Location::GPR(dst)) => {
1099 if val < 0x1000 {
1100 dynasm!(self ; mov W(dst), val);
1101 } else if encode_logical_immediate_64bit(val as _).is_some() {
1102 dynasm!(self ; mov X(dst), val as _);
1103 } else {
1104 dynasm!(self
1105 ; movz X(dst), val
1106 ; movk X(dst), val, lsl #16);
1107 }
1108 }
1109 (Size::S64, Location::Imm64(val), Location::GPR(dst)) => {
1110 if val < 0x1000 || encode_logical_immediate_64bit(val as _).is_some() {
1111 dynasm!(self ; mov X(dst), val as _);
1112 } else {
1113 dynasm!(self
1114 ; movz X(dst), val as _
1115 ; movk X(dst), val as _, lsl #16
1116 ; movk X(dst), val as _, lsl #32
1117 ; movk X(dst), val as _, lsl #48);
1118 }
1119 }
1120 _ => codegen_error!("singlepass can't emit MOV {:?}, {:?}, {:?}", sz, src, dst),
1121 }
1122 Ok(())
1123 }
1124
1125 fn emit_movn(&mut self, sz: Size, reg: Location, val: u32) -> Result<(), CompileError> {
1126 match (sz, reg) {
1127 (Size::S32, Location::GPR(reg)) => {
1128 dynasm!(self ; movn W(reg), val);
1129 }
1130 (Size::S64, Location::GPR(reg)) => {
1131 dynasm!(self ; movn X(reg), val);
1132 }
1133 _ => codegen_error!("singlepass can't emit MOVN"),
1134 }
1135 Ok(())
1136 }
1137 fn emit_movz(&mut self, reg: Location, val: u32) -> Result<(), CompileError> {
1138 match reg {
1139 Location::GPR(reg) => {
1140 dynasm!(self ; movz W(reg), val);
1141 }
1142 _ => codegen_error!("singlepass can't emit MOVZ"),
1143 }
1144 Ok(())
1145 }
1146 fn emit_movk(&mut self, reg: Location, val: u32, shift: u32) -> Result<(), CompileError> {
1147 match reg {
1148 Location::GPR(reg) => {
1149 dynasm!(self ; movk X(reg), val, LSL shift);
1150 }
1151 _ => codegen_error!("singlepass can't emit MOVK"),
1152 }
1153 Ok(())
1154 }
1155
1156 fn emit_mov_imm(&mut self, dst: Location, val: u64) -> Result<(), CompileError> {
1157 match dst {
1158 Location::GPR(dst) => {
1159 let offset = val.trailing_zeros() & 48;
1160 let masked = 0xffff & (val >> offset);
1161 if (masked << offset) == val {
1162 dynasm!(self ; movz X(dst), masked as u32, LSL offset);
1163 } else if val >> 16 == 0xffff_ffff_ffff {
1164 let val: u16 = !((val & 0xffff) as u16);
1165 dynasm!(self ; movn X(dst), val as u32);
1166 } else if val >> 16 == 0xffff {
1167 let val: u16 = !((val & 0xffff) as u16);
1168 dynasm!(self ; movn W(dst), val as u32);
1169 } else {
1170 dynasm!(self ; movz W(dst), (val&0xffff) as u32);
1171 let val = val >> 16;
1172 if val != 0 {
1173 dynasm!(self ; movk X(dst), (val&0xffff) as u32, LSL 16);
1174 let val = val >> 16;
1175 if val != 0 {
1176 dynasm!(self ; movk X(dst), (val&0xffff) as u32, LSL 32);
1177 let val = val >> 16;
1178 if val != 0 {
1179 dynasm!(self ; movk X(dst), (val&0xffff) as u32, LSL 48);
1180 }
1181 }
1182 }
1183 }
1184 }
1185 _ => codegen_error!("singlepass can't emit MOVW {:?}", dst),
1186 }
1187 Ok(())
1188 }
1189
1190 fn emit_add(
1191 &mut self,
1192 sz: Size,
1193 src1: Location,
1194 src2: Location,
1195 dst: Location,
1196 ) -> Result<(), CompileError> {
1197 match (sz, src1, src2, dst) {
1198 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1199 dynasm!(self ; add XSP(dst), XSP(src1), X(src2), UXTX);
1200 }
1201 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1202 dynasm!(self ; add WSP(dst), WSP(src1), W(src2), UXTX);
1203 }
1204 (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1205 | (Size::S64, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1206 dynasm!(self ; add XSP(dst), XSP(src1), imm as _);
1207 }
1208 (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
1209 | (Size::S64, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
1210 if imm >= 0x1000 {
1211 unreachable!();
1212 }
1213 dynasm!(self ; add XSP(dst), XSP(src1), imm);
1214 }
1215 (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst))
1216 | (Size::S64, Location::Imm64(imm), Location::GPR(src1), Location::GPR(dst)) => {
1217 if imm >= 0x1000 {
1218 unreachable!();
1219 }
1220 let imm = imm as u32;
1221 dynasm!(self ; add XSP(dst), XSP(src1), imm);
1222 }
1223 (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1224 | (Size::S32, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1225 dynasm!(self ; add WSP(dst), WSP(src1), imm as u32);
1226 }
1227 (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
1228 | (Size::S32, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
1229 if imm >= 0x1000 {
1230 unreachable!();
1231 }
1232 dynasm!(self ; add WSP(dst), WSP(src1), imm);
1233 }
1234 _ => codegen_error!(
1235 "singlepass can't emit ADD {:?} {:?} {:?} {:?}",
1236 sz,
1237 src1,
1238 src2,
1239 dst
1240 ),
1241 }
1242 Ok(())
1243 }
1244 fn emit_sub(
1245 &mut self,
1246 sz: Size,
1247 src1: Location,
1248 src2: Location,
1249 dst: Location,
1250 ) -> Result<(), CompileError> {
1251 match (sz, src1, src2, dst) {
1252 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1253 dynasm!(self; sub XSP(dst), XSP(src1), X(src2), UXTX 0);
1254 }
1255 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1256 dynasm!(self ; sub WSP(dst), WSP(src1), W(src2), UXTX 0);
1257 }
1258 (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1259 dynasm!(self ; sub XSP(dst), XSP(src1), imm as u32);
1260 }
1261 (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1262 dynasm!(self ; sub WSP(dst), WSP(src1), imm as u32);
1263 }
1264 (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1265 if imm >= 0x1000 {
1266 unreachable!();
1267 }
1268 dynasm!(self ; sub WSP(dst), WSP(src1), imm);
1269 }
1270 (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1271 if imm >= 0x1000 {
1272 unreachable!();
1273 }
1274 dynasm!(self ; sub XSP(dst), XSP(src1), imm);
1275 }
1276 (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
1277 if imm >= 0x1000 {
1278 unreachable!();
1279 }
1280 dynasm!(self ; sub XSP(dst), XSP(src1), imm as u32);
1281 }
1282 _ => codegen_error!(
1283 "singlepass can't emit SUB {:?} {:?} {:?} {:?}",
1284 sz,
1285 src1,
1286 src2,
1287 dst
1288 ),
1289 }
1290 Ok(())
1291 }
1292 fn emit_mul(
1293 &mut self,
1294 sz: Size,
1295 src1: Location,
1296 src2: Location,
1297 dst: Location,
1298 ) -> Result<(), CompileError> {
1299 match (sz, src1, src2, dst) {
1300 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1301 dynasm!(self ; mul X(dst), X(src1), X(src2));
1302 }
1303 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1304 dynasm!(self ; mul W(dst), W(src1), W(src2));
1305 }
1306 _ => codegen_error!(
1307 "singlepass can't emit MUL {:?} {:?} {:?} {:?}",
1308 sz,
1309 src1,
1310 src2,
1311 dst
1312 ),
1313 }
1314 Ok(())
1315 }
1316 fn emit_adds(
1317 &mut self,
1318 sz: Size,
1319 src1: Location,
1320 src2: Location,
1321 dst: Location,
1322 ) -> Result<(), CompileError> {
1323 match (sz, src1, src2, dst) {
1324 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1325 dynasm!(self ; adds X(dst), X(src1), X(src2));
1326 }
1327 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1328 dynasm!(self ; adds W(dst), W(src1), W(src2));
1329 }
1330 (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1331 | (Size::S64, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1332 dynasm!(self ; adds X(dst), XSP(src1), imm as u32);
1333 }
1334 (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
1335 | (Size::S64, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
1336 if imm >= 0x1000 {
1337 codegen_error!("singlepass ADD.S with imm too large {}", imm);
1338 }
1339 dynasm!(self ; adds X(dst), XSP(src1), imm);
1340 }
1341 (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1342 | (Size::S32, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1343 dynasm!(self ; adds W(dst), WSP(src1), imm as u32);
1344 }
1345 (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
1346 | (Size::S32, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
1347 if imm >= 0x1000 {
1348 codegen_error!("singlepass ADD.S with imm too large {}", imm);
1349 }
1350 dynasm!(self ; adds W(dst), WSP(src1), imm);
1351 }
1352 _ => codegen_error!(
1353 "singlepass can't emit ADD.S {:?} {:?} {:?} {:?}",
1354 sz,
1355 src1,
1356 src2,
1357 dst
1358 ),
1359 }
1360 Ok(())
1361 }
1362 fn emit_subs(
1363 &mut self,
1364 sz: Size,
1365 src1: Location,
1366 src2: Location,
1367 dst: Location,
1368 ) -> Result<(), CompileError> {
1369 match (sz, src1, src2, dst) {
1370 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1371 dynasm!(self ; subs X(dst), X(src1), X(src2));
1372 }
1373 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1374 dynasm!(self ; subs W(dst), W(src1), W(src2));
1375 }
1376 (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1377 dynasm!(self ; subs X(dst), XSP(src1), imm as u32);
1378 }
1379 (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1380 dynasm!(self ; subs W(dst), WSP(src1), imm as u32);
1381 }
1382 _ => codegen_error!(
1383 "singlepass can't emit SUB.S {:?} {:?} {:?} {:?}",
1384 sz,
1385 src1,
1386 src2,
1387 dst
1388 ),
1389 }
1390 Ok(())
1391 }
1392 fn emit_add_lsl(
1393 &mut self,
1394 sz: Size,
1395 src1: Location,
1396 src2: Location,
1397 lsl: u32,
1398 dst: Location,
1399 ) -> Result<(), CompileError> {
1400 match (sz, src1, src2, dst) {
1401 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1402 dynasm!(self ; add X(dst), X(src1), X(src2), LSL lsl);
1403 }
1404 _ => codegen_error!(
1405 "singlepass can't emit LSL {:?} {:?} {:?} {:?} LSL {:?}",
1406 sz,
1407 src1,
1408 src2,
1409 dst,
1410 lsl
1411 ),
1412 }
1413 Ok(())
1414 }
1415
1416 fn emit_cmp(&mut self, sz: Size, left: Location, right: Location) -> Result<(), CompileError> {
1422 match (sz, left, right) {
1423 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1424 dynasm!(self ; cmp X(dst), X(src));
1425 }
1426 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1427 dynasm!(self ; cmp W(dst), W(src));
1428 }
1429 (Size::S64, Location::Imm8(imm), Location::GPR(dst)) => {
1430 dynasm!(self ; cmp XSP(dst), imm as u32);
1431 }
1432 (Size::S64, Location::Imm32(imm), Location::GPR(dst)) => {
1433 if imm >= 0x1000 {
1434 codegen_error!("singlepass CMP with imm too large {}", imm);
1435 }
1436 dynasm!(self ; cmp XSP(dst), imm as u32);
1437 }
1438 (Size::S64, Location::Imm64(imm), Location::GPR(dst)) => {
1439 if imm >= 0x1000 {
1440 codegen_error!("singlepass CMP with imm too large {}", imm);
1441 }
1442 dynasm!(self ; cmp XSP(dst), imm as u32);
1443 }
1444 (Size::S32, Location::Imm8(imm), Location::GPR(dst)) => {
1445 dynasm!(self ; cmp WSP(dst), imm as u32);
1446 }
1447 (Size::S32, Location::Imm32(imm), Location::GPR(dst)) => {
1448 if imm >= 0x1000 {
1449 codegen_error!("singlepass CMP with imm too large {}", imm);
1450 }
1451 dynasm!(self ; cmp WSP(dst), imm as u32);
1452 }
1453 _ => codegen_error!("singlepass can't emit CMP {:?} {:?} {:?}", sz, left, right),
1454 }
1455 Ok(())
1456 }
1457
1458 fn emit_tst(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1459 match (sz, src, dst) {
1460 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1461 dynasm!(self ; tst X(dst), X(src));
1462 }
1463 (Size::S64, Location::Imm32(imm), Location::GPR(dst)) => {
1464 if encode_logical_immediate_64bit(imm as u64).is_none() {
1465 codegen_error!("singlepass TST with incompatible imm {}", imm);
1466 }
1467 dynasm!(self ; tst X(dst), imm as u64);
1468 }
1469 (Size::S64, Location::Imm64(imm), Location::GPR(dst)) => {
1470 if encode_logical_immediate_64bit(imm as u64).is_none() {
1471 codegen_error!("singlepass TST with incompatible imm {}", imm);
1472 }
1473 dynasm!(self ; tst X(dst), imm as u64);
1474 }
1475 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1476 dynasm!(self ; tst W(dst), W(src));
1477 }
1478 (Size::S32, Location::Imm32(imm), Location::GPR(dst)) => {
1479 if encode_logical_immediate_64bit(imm as u64).is_none() {
1480 codegen_error!("singlepass TST with incompatible imm {}", imm);
1481 }
1482 dynasm!(self ; tst W(dst), imm);
1483 }
1484 _ => codegen_error!("singlepass can't emit TST"),
1485 }
1486 Ok(())
1487 }
1488
1489 fn emit_lsl(
1490 &mut self,
1491 sz: Size,
1492 src1: Location,
1493 src2: Location,
1494 dst: Location,
1495 ) -> Result<(), CompileError> {
1496 match (sz, src1, src2, dst) {
1497 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1498 dynasm!(self ; lsl X(dst), X(src1), X(src2));
1499 }
1500 (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1501 if imm > 63 {
1502 codegen_error!("singlepass LSL with incompatible imm {}", imm);
1503 }
1504 let imm = imm as u32;
1505
1506 dynasm!(self ; lsl X(dst), X(src1), imm);
1507 }
1508 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1509 dynasm!(self ; lsl W(dst), W(src1), W(src2));
1510 }
1511 (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1512 if imm > 63 {
1513 codegen_error!("singlepass LSL with incompatible imm {}", imm);
1514 }
1515 dynasm!(self ; lsl X(dst), X(src1), imm as u32);
1516 }
1517 (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
1518 if imm > 63 {
1519 codegen_error!("singlepass LSL with incompatible imm {}", imm);
1520 }
1521 dynasm!(self ; lsl X(dst), X(src1), imm as u32);
1522 }
1523 (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1524 if imm > 31 {
1525 codegen_error!("singlepass LSL with incompatible imm {}", imm);
1526 }
1527 dynasm!(self ; lsl W(dst), W(src1), imm as u32);
1528 }
1529 (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1530 if imm > 31 {
1531 codegen_error!("singlepass LSL with incompatible imm {}", imm);
1532 }
1533 dynasm!(self ; lsl W(dst), W(src1), imm as u32);
1534 }
1535 _ => codegen_error!(
1536 "singlepass can't emit LSL {:?} {:?} {:?} {:?}",
1537 sz,
1538 src1,
1539 src2,
1540 dst
1541 ),
1542 }
1543 Ok(())
1544 }
1545 fn emit_asr(
1546 &mut self,
1547 sz: Size,
1548 src1: Location,
1549 src2: Location,
1550 dst: Location,
1551 ) -> Result<(), CompileError> {
1552 match (sz, src1, src2, dst) {
1553 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1554 dynasm!(self ; asr X(dst), X(src1), X(src2));
1555 }
1556 (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1557 let imm = imm as u32;
1558
1559 if imm == 0 || imm > 63 {
1560 codegen_error!("singlepass ASR with incompatible imm {}", imm);
1561 }
1562 dynasm!(self ; asr X(dst), X(src1), imm);
1563 }
1564 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1565 dynasm!(self ; asr W(dst), W(src1), W(src2));
1566 }
1567 (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1568 if imm == 0 || imm > 63 {
1569 codegen_error!("singlepass ASR with incompatible imm {}", imm);
1570 }
1571 dynasm!(self ; asr X(dst), X(src1), imm as u32);
1572 }
1573 (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
1574 if imm == 0 || imm > 63 {
1575 codegen_error!("singlepass ASR with incompatible imm {}", imm);
1576 }
1577 dynasm!(self ; asr X(dst), X(src1), imm as u32);
1578 }
1579 (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1580 if imm == 0 || imm > 31 {
1581 codegen_error!("singlepass ASR with incompatible imm {}", imm);
1582 }
1583 dynasm!(self ; asr W(dst), W(src1), imm as u32);
1584 }
1585 (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1586 if imm == 0 || imm > 31 {
1587 codegen_error!("singlepass ASR with incompatible imm {}", imm);
1588 }
1589 dynasm!(self ; asr W(dst), W(src1), imm as u32);
1590 }
1591 _ => codegen_error!(
1592 "singlepass can't emit ASR {:?} {:?} {:?} {:?}",
1593 sz,
1594 src1,
1595 src2,
1596 dst
1597 ),
1598 }
1599 Ok(())
1600 }
1601 fn emit_lsr(
1602 &mut self,
1603 sz: Size,
1604 src1: Location,
1605 src2: Location,
1606 dst: Location,
1607 ) -> Result<(), CompileError> {
1608 match (sz, src1, src2, dst) {
1609 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1610 dynasm!(self ; lsr X(dst), X(src1), X(src2));
1611 }
1612 (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1613 let imm = imm as u32;
1614
1615 if imm == 0 || imm > 63 {
1616 codegen_error!("singlepass LSR with incompatible imm {}", imm);
1617 }
1618 dynasm!(self ; lsr X(dst), X(src1), imm);
1619 }
1620 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1621 dynasm!(self ; lsr W(dst), W(src1), W(src2));
1622 }
1623 (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1624 if imm == 0 || imm > 63 {
1625 codegen_error!("singlepass LSR with incompatible imm {}", imm);
1626 }
1627 dynasm!(self ; lsr X(dst), X(src1), imm as u32);
1628 }
1629 (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
1630 if imm == 0 || imm > 63 {
1631 codegen_error!("singlepass LSR with incompatible imm {}", imm);
1632 }
1633 dynasm!(self ; lsr X(dst), X(src1), imm as u32);
1634 }
1635 (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1636 if imm == 0 || imm > 31 {
1637 codegen_error!("singlepass LSR with incompatible imm {}", imm);
1638 }
1639 dynasm!(self ; lsr W(dst), W(src1), imm as u32);
1640 }
1641 (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1642 if imm == 0 || imm > 31 {
1643 codegen_error!("singlepass LSR with incompatible imm {}", imm);
1644 }
1645 dynasm!(self ; lsr W(dst), W(src1), imm as u32);
1646 }
1647 _ => codegen_error!(
1648 "singlepass can't emit LSR {:?} {:?} {:?} {:?}",
1649 sz,
1650 src1,
1651 src2,
1652 dst
1653 ),
1654 }
1655 Ok(())
1656 }
1657 fn emit_ror(
1658 &mut self,
1659 sz: Size,
1660 src1: Location,
1661 src2: Location,
1662 dst: Location,
1663 ) -> Result<(), CompileError> {
1664 match (sz, src1, src2, dst) {
1665 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1666 dynasm!(self ; ror X(dst), X(src1), X(src2));
1667 }
1668 (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1669 let imm = imm as u32;
1670
1671 if imm == 0 || imm > 63 {
1672 codegen_error!("singlepass ROR with incompatible imm {}", imm);
1673 }
1674 dynasm!(self ; ror X(dst), X(src1), imm);
1675 }
1676 (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
1677 let imm = imm as u32;
1678
1679 if imm == 0 || imm > 63 {
1680 codegen_error!("singlepass ROR with incompatible imm {}", imm);
1681 }
1682 dynasm!(self ; ror X(dst), X(src1), imm);
1683 }
1684 (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1685 if imm == 0 || imm > 31 {
1686 codegen_error!("singlepass ROR with incompatible imm {}", imm);
1687 }
1688 dynasm!(self ; ror W(dst), W(src1), imm as u32);
1689 }
1690 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1691 dynasm!(self ; ror W(dst), W(src1), W(src2));
1692 }
1693 (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1694 if imm == 0 || imm > 63 {
1695 codegen_error!("singlepass ROR with incompatible imm {}", imm);
1696 }
1697 dynasm!(self ; ror X(dst), X(src1), imm as u32);
1698 }
1699 (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1700 if imm == 0 || imm > 31 {
1701 codegen_error!("singlepass ROR with incompatible imm {}", imm);
1702 }
1703 dynasm!(self ; ror W(dst), W(src1), imm as u32);
1704 }
1705 _ => codegen_error!(
1706 "singlepass can't emit ROR {:?} {:?} {:?} {:?}",
1707 sz,
1708 src1,
1709 src2,
1710 dst
1711 ),
1712 }
1713 Ok(())
1714 }
1715
1716 fn emit_or(
1717 &mut self,
1718 sz: Size,
1719 src1: Location,
1720 src2: Location,
1721 dst: Location,
1722 ) -> Result<(), CompileError> {
1723 match (sz, src1, src2, dst) {
1724 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1725 dynasm!(self ; orr X(dst), X(src1), X(src2));
1726 }
1727 (Size::S64, Location::GPR(src1), Location::Imm64(src2), Location::GPR(dst)) => {
1728 let src2 = src2 as u64;
1729
1730 if encode_logical_immediate_64bit(src2 as u64).is_none() {
1731 codegen_error!("singlepass OR with incompatible imm {}", src2);
1732 }
1733 dynasm!(self ; orr XSP(dst), X(src1), src2);
1734 }
1735 (Size::S32, Location::GPR(src1), Location::Imm32(src2), Location::GPR(dst)) => {
1736 let src2 = src2 as u32;
1737
1738 if encode_logical_immediate_32bit(src2).is_none() {
1739 codegen_error!("singlepass OR with incompatible imm {}", src2);
1740 }
1741 dynasm!(self ; orr WSP(dst), W(src1), src2);
1742 }
1743 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1744 dynasm!(self ; orr W(dst), W(src1), W(src2));
1745 }
1746 _ => codegen_error!(
1747 "singlepass can't emit OR {:?} {:?} {:?} {:?}",
1748 sz,
1749 src1,
1750 src2,
1751 dst
1752 ),
1753 }
1754 Ok(())
1755 }
1756 fn emit_and(
1757 &mut self,
1758 sz: Size,
1759 src1: Location,
1760 src2: Location,
1761 dst: Location,
1762 ) -> Result<(), CompileError> {
1763 match (sz, src1, src2, dst) {
1764 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1765 dynasm!(self ; and X(dst), X(src1), X(src2));
1766 }
1767 (Size::S64, Location::GPR(src1), Location::Imm64(src2), Location::GPR(dst)) => {
1768 let src2 = src2 as u64;
1769
1770 if encode_logical_immediate_64bit(src2 as u64).is_none() {
1771 codegen_error!("singlepass AND with incompatible imm {}", src2);
1772 }
1773 dynasm!(self ; and XSP(dst), X(src1), src2);
1774 }
1775 (Size::S32, Location::GPR(src1), Location::Imm32(src2), Location::GPR(dst)) => {
1776 let src2 = src2 as u32;
1777
1778 if encode_logical_immediate_32bit(src2).is_none() {
1779 codegen_error!("singlepass AND with incompatible imm {}", src2);
1780 }
1781 dynasm!(self ; and WSP(dst), W(src1), src2);
1782 }
1783 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1784 dynasm!(self ; and W(dst), W(src1), W(src2));
1785 }
1786 _ => codegen_error!(
1787 "singlepass can't emit AND {:?} {:?} {:?} {:?}",
1788 sz,
1789 src1,
1790 src2,
1791 dst
1792 ),
1793 }
1794 Ok(())
1795 }
1796 fn emit_eor(
1797 &mut self,
1798 sz: Size,
1799 src1: Location,
1800 src2: Location,
1801 dst: Location,
1802 ) -> Result<(), CompileError> {
1803 match (sz, src1, src2, dst) {
1804 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1805 dynasm!(self ; eor X(dst), X(src1), X(src2));
1806 }
1807 (Size::S64, Location::GPR(src1), Location::Imm64(src2), Location::GPR(dst)) => {
1808 let src2 = src2 as u64;
1809
1810 if encode_logical_immediate_64bit(src2 as u64).is_none() {
1811 codegen_error!("singlepass EOR with incompatible imm {}", src2);
1812 }
1813 dynasm!(self ; eor XSP(dst), X(src1), src2);
1814 }
1815 (Size::S32, Location::GPR(src1), Location::Imm32(src2), Location::GPR(dst)) => {
1816 let src2 = src2 as u32;
1817
1818 if encode_logical_immediate_32bit(src2).is_none() {
1819 codegen_error!("singlepass EOR with incompatible imm {}", src2);
1820 }
1821 dynasm!(self ; eor WSP(dst), W(src1), src2);
1822 }
1823 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1824 dynasm!(self ; eor W(dst), W(src1), W(src2));
1825 }
1826 _ => codegen_error!(
1827 "singlepass can't emit EOR {:?} {:?} {:?} {:?}",
1828 sz,
1829 src1,
1830 src2,
1831 dst
1832 ),
1833 }
1834 Ok(())
1835 }
1836
1837 fn emit_bfc(
1838 &mut self,
1839 sz: Size,
1840 lsb: u32,
1841 width: u32,
1842 dst: Location,
1843 ) -> Result<(), CompileError> {
1844 match (sz, dst) {
1845 (Size::S32, Location::GPR(dst)) => {
1846 dynasm!(self ; bfc W(dst), lsb, width);
1847 }
1848 (Size::S64, Location::GPR(dst)) => {
1849 dynasm!(self ; bfc X(dst), lsb, width);
1850 }
1851 _ => codegen_error!("singlepass can't emit BFC"),
1852 }
1853 Ok(())
1854 }
1855 fn emit_bfi(
1856 &mut self,
1857 sz: Size,
1858 src: Location,
1859 lsb: u32,
1860 width: u32,
1861 dst: Location,
1862 ) -> Result<(), CompileError> {
1863 match (sz, src, dst) {
1864 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1865 dynasm!(self ; bfi W(dst), W(src), lsb, width);
1866 }
1867 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1868 dynasm!(self ; bfi X(dst), X(src), lsb, width);
1869 }
1870 _ => codegen_error!("singlepass can't emit BFI"),
1871 }
1872 Ok(())
1873 }
1874
1875 fn emit_udiv(
1876 &mut self,
1877 sz: Size,
1878 src1: Location,
1879 src2: Location,
1880 dst: Location,
1881 ) -> Result<(), CompileError> {
1882 match (sz, src1, src2, dst) {
1883 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1884 dynasm!(self ; udiv W(dst), W(src1), W(src2));
1885 }
1886 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1887 dynasm!(self ; udiv X(dst), X(src1), X(src2));
1888 }
1889 _ => codegen_error!(
1890 "singlepass can't emit UDIV {:?} {:?} {:?} {:?}",
1891 sz,
1892 src1,
1893 src2,
1894 dst
1895 ),
1896 }
1897 Ok(())
1898 }
1899 fn emit_sdiv(
1900 &mut self,
1901 sz: Size,
1902 src1: Location,
1903 src2: Location,
1904 dst: Location,
1905 ) -> Result<(), CompileError> {
1906 match (sz, src1, src2, dst) {
1907 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1908 dynasm!(self ; sdiv W(dst), W(src1), W(src2));
1909 }
1910 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1911 dynasm!(self ; sdiv X(dst), X(src1), X(src2));
1912 }
1913 _ => codegen_error!(
1914 "singlepass can't emit SDIV {:?} {:?} {:?} {:?}",
1915 sz,
1916 src1,
1917 src2,
1918 dst
1919 ),
1920 }
1921 Ok(())
1922 }
1923
1924 fn emit_msub(
1926 &mut self,
1927 sz: Size,
1928 a: Location,
1929 b: Location,
1930 c: Location,
1931 dst: Location,
1932 ) -> Result<(), CompileError> {
1933 match (sz, a, b, c, dst) {
1934 (
1935 Size::S32,
1936 Location::GPR(a),
1937 Location::GPR(b),
1938 Location::GPR(c),
1939 Location::GPR(dst),
1940 ) => {
1941 dynasm!(self ; msub W(dst), W(a), W(b), W(c));
1942 }
1943 (
1944 Size::S64,
1945 Location::GPR(a),
1946 Location::GPR(b),
1947 Location::GPR(c),
1948 Location::GPR(dst),
1949 ) => {
1950 dynasm!(self ; msub X(dst), X(a), X(b), X(c));
1951 }
1952 _ => codegen_error!(
1953 "singlepass can't emit msub {:?} {:?} {:?} {:?} {:?}",
1954 sz,
1955 a,
1956 b,
1957 c,
1958 dst
1959 ),
1960 }
1961 Ok(())
1962 }
1963
1964 fn emit_sxtb(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1965 match (sz, src, dst) {
1966 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1967 dynasm!(self ; sxtb W(dst), W(src));
1968 }
1969 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1970 dynasm!(self ; sxtb X(dst), W(src));
1971 }
1972 _ => codegen_error!("singlepass can't emit SXTB {:?} {:?} {:?}", sz, src, dst),
1973 }
1974 Ok(())
1975 }
1976 fn emit_sxth(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1977 match (sz, src, dst) {
1978 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1979 dynasm!(self ; sxth W(dst), W(src));
1980 }
1981 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1982 dynasm!(self ; sxth X(dst), W(src));
1983 }
1984 _ => codegen_error!("singlepass can't emit SXTH {:?} {:?} {:?}", sz, src, dst),
1985 }
1986 Ok(())
1987 }
1988 fn emit_sxtw(&mut self, _sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1989 match (src, dst) {
1990 (Location::GPR(src), Location::GPR(dst)) => {
1991 dynasm!(self ; sxtw X(dst), W(src));
1992 }
1993 _ => codegen_error!("singlepass can't emit SXTW {:?} {:?}", src, dst),
1994 }
1995 Ok(())
1996 }
1997 fn emit_uxtb(&mut self, _sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1998 match (src, dst) {
1999 (Location::GPR(src), Location::GPR(dst)) => {
2000 dynasm!(self ; uxtb W(dst), W(src));
2001 }
2002 _ => codegen_error!("singlepass can't emit UXTB {:?} {:?}", src, dst),
2003 }
2004 Ok(())
2005 }
2006 fn emit_uxth(&mut self, _sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2007 match (src, dst) {
2008 (Location::GPR(src), Location::GPR(dst)) => {
2009 dynasm!(self ; uxth W(dst), W(src));
2010 }
2011 _ => codegen_error!("singlepass can't emit UXTH {:?} {:?}", src, dst),
2012 }
2013 Ok(())
2014 }
2015
2016 fn emit_cset(&mut self, sz: Size, dst: Location, cond: Condition) -> Result<(), CompileError> {
2017 match (sz, dst) {
2018 (Size::S32, Location::GPR(reg)) => match cond {
2019 Condition::Eq => dynasm!(self ; cset W(reg), eq),
2020 Condition::Ne => dynasm!(self ; cset W(reg), ne),
2021 Condition::Cs => dynasm!(self ; cset W(reg), cs),
2022 Condition::Cc => dynasm!(self ; cset W(reg), cc),
2023 Condition::Mi => dynasm!(self ; cset W(reg), mi),
2024 Condition::Pl => dynasm!(self ; cset W(reg), pl),
2025 Condition::Vs => dynasm!(self ; cset W(reg), vs),
2026 Condition::Vc => dynasm!(self ; cset W(reg), vc),
2027 Condition::Hi => dynasm!(self ; cset W(reg), hi),
2028 Condition::Ls => dynasm!(self ; cset W(reg), ls),
2029 Condition::Ge => dynasm!(self ; cset W(reg), ge),
2030 Condition::Lt => dynasm!(self ; cset W(reg), lt),
2031 Condition::Gt => dynasm!(self ; cset W(reg), gt),
2032 Condition::Le => dynasm!(self ; cset W(reg), le),
2033 Condition::Al => dynasm!(self ; cset W(reg), al),
2034 },
2035 (Size::S64, Location::GPR(reg)) => match cond {
2036 Condition::Eq => dynasm!(self ; cset X(reg), eq),
2037 Condition::Ne => dynasm!(self ; cset X(reg), ne),
2038 Condition::Cs => dynasm!(self ; cset X(reg), cs),
2039 Condition::Cc => dynasm!(self ; cset X(reg), cc),
2040 Condition::Mi => dynasm!(self ; cset X(reg), mi),
2041 Condition::Pl => dynasm!(self ; cset X(reg), pl),
2042 Condition::Vs => dynasm!(self ; cset X(reg), vs),
2043 Condition::Vc => dynasm!(self ; cset X(reg), vc),
2044 Condition::Hi => dynasm!(self ; cset X(reg), hi),
2045 Condition::Ls => dynasm!(self ; cset X(reg), ls),
2046 Condition::Ge => dynasm!(self ; cset X(reg), ge),
2047 Condition::Lt => dynasm!(self ; cset X(reg), lt),
2048 Condition::Gt => dynasm!(self ; cset X(reg), gt),
2049 Condition::Le => dynasm!(self ; cset X(reg), le),
2050 Condition::Al => dynasm!(self ; cset X(reg), al),
2051 },
2052 _ => codegen_error!("singlepass can't emit CSET {:?} {:?} {:?}", sz, dst, cond),
2053 }
2054 Ok(())
2055 }
2056 fn emit_csetm(&mut self, sz: Size, dst: Location, cond: Condition) -> Result<(), CompileError> {
2057 match (sz, dst) {
2058 (Size::S32, Location::GPR(reg)) => match cond {
2059 Condition::Eq => dynasm!(self ; csetm W(reg), eq),
2060 Condition::Ne => dynasm!(self ; csetm W(reg), ne),
2061 Condition::Cs => dynasm!(self ; csetm W(reg), cs),
2062 Condition::Cc => dynasm!(self ; csetm W(reg), cc),
2063 Condition::Mi => dynasm!(self ; csetm W(reg), mi),
2064 Condition::Pl => dynasm!(self ; csetm W(reg), pl),
2065 Condition::Vs => dynasm!(self ; csetm W(reg), vs),
2066 Condition::Vc => dynasm!(self ; csetm W(reg), vc),
2067 Condition::Hi => dynasm!(self ; csetm W(reg), hi),
2068 Condition::Ls => dynasm!(self ; csetm W(reg), ls),
2069 Condition::Ge => dynasm!(self ; csetm W(reg), ge),
2070 Condition::Lt => dynasm!(self ; csetm W(reg), lt),
2071 Condition::Gt => dynasm!(self ; csetm W(reg), gt),
2072 Condition::Le => dynasm!(self ; csetm W(reg), le),
2073 Condition::Al => dynasm!(self ; csetm W(reg), al),
2074 },
2075 (Size::S64, Location::GPR(reg)) => match cond {
2076 Condition::Eq => dynasm!(self ; csetm X(reg), eq),
2077 Condition::Ne => dynasm!(self ; csetm X(reg), ne),
2078 Condition::Cs => dynasm!(self ; csetm X(reg), cs),
2079 Condition::Cc => dynasm!(self ; csetm X(reg), cc),
2080 Condition::Mi => dynasm!(self ; csetm X(reg), mi),
2081 Condition::Pl => dynasm!(self ; csetm X(reg), pl),
2082 Condition::Vs => dynasm!(self ; csetm X(reg), vs),
2083 Condition::Vc => dynasm!(self ; csetm X(reg), vc),
2084 Condition::Hi => dynasm!(self ; csetm X(reg), hi),
2085 Condition::Ls => dynasm!(self ; csetm X(reg), ls),
2086 Condition::Ge => dynasm!(self ; csetm X(reg), ge),
2087 Condition::Lt => dynasm!(self ; csetm X(reg), lt),
2088 Condition::Gt => dynasm!(self ; csetm X(reg), gt),
2089 Condition::Le => dynasm!(self ; csetm X(reg), le),
2090 Condition::Al => dynasm!(self ; csetm X(reg), al),
2091 },
2092 _ => codegen_error!("singlepass can't emit CSETM {:?} {:?} {:?}", sz, dst, cond),
2093 }
2094 Ok(())
2095 }
2096 fn emit_cinc(
2097 &mut self,
2098 sz: Size,
2099 src: Location,
2100 dst: Location,
2101 cond: Condition,
2102 ) -> Result<(), CompileError> {
2103 match (sz, src, dst) {
2104 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
2105 match cond {
2106 Condition::Eq => dynasm!(self ; cinc W(dst), W(src), eq),
2107 Condition::Ne => dynasm!(self ; cinc W(dst), W(src), ne),
2108 Condition::Cs => dynasm!(self ; cinc W(dst), W(src), cs),
2109 Condition::Cc => dynasm!(self ; cinc W(dst), W(src), cc),
2110 Condition::Mi => dynasm!(self ; cinc W(dst), W(src), mi),
2111 Condition::Pl => dynasm!(self ; cinc W(dst), W(src), pl),
2112 Condition::Vs => dynasm!(self ; cinc W(dst), W(src), vs),
2113 Condition::Vc => dynasm!(self ; cinc W(dst), W(src), vc),
2114 Condition::Hi => dynasm!(self ; cinc W(dst), W(src), hi),
2115 Condition::Ls => dynasm!(self ; cinc W(dst), W(src), ls),
2116 Condition::Ge => dynasm!(self ; cinc W(dst), W(src), ge),
2117 Condition::Lt => dynasm!(self ; cinc W(dst), W(src), lt),
2118 Condition::Gt => dynasm!(self ; cinc W(dst), W(src), gt),
2119 Condition::Le => dynasm!(self ; cinc W(dst), W(src), le),
2120 Condition::Al => dynasm!(self ; cinc W(dst), W(src), al),
2121 };
2122 }
2123 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
2124 match cond {
2125 Condition::Eq => dynasm!(self ; cinc X(src), X(dst), eq),
2126 Condition::Ne => dynasm!(self ; cinc X(src), X(dst), ne),
2127 Condition::Cs => dynasm!(self ; cinc X(src), X(dst), cs),
2128 Condition::Cc => dynasm!(self ; cinc X(src), X(dst), cc),
2129 Condition::Mi => dynasm!(self ; cinc X(src), X(dst), mi),
2130 Condition::Pl => dynasm!(self ; cinc X(src), X(dst), pl),
2131 Condition::Vs => dynasm!(self ; cinc X(src), X(dst), vs),
2132 Condition::Vc => dynasm!(self ; cinc X(src), X(dst), vc),
2133 Condition::Hi => dynasm!(self ; cinc X(src), X(dst), hi),
2134 Condition::Ls => dynasm!(self ; cinc X(src), X(dst), ls),
2135 Condition::Ge => dynasm!(self ; cinc X(src), X(dst), ge),
2136 Condition::Lt => dynasm!(self ; cinc X(src), X(dst), lt),
2137 Condition::Gt => dynasm!(self ; cinc X(src), X(dst), gt),
2138 Condition::Le => dynasm!(self ; cinc X(src), X(dst), le),
2139 Condition::Al => dynasm!(self ; cinc X(src), X(dst), al),
2140 };
2141 }
2142 _ => codegen_error!("singlepass can't emit CINC"),
2143 }
2144 Ok(())
2145 }
2146
2147 fn emit_clz(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2148 match (sz, src, dst) {
2149 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
2150 dynasm!(self ; clz X(dst), X(src));
2151 }
2152 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
2153 dynasm!(self ; clz W(dst), W(src));
2154 }
2155 _ => codegen_error!("singlepass can't emit CLS {:?} {:?} {:?}", sz, src, dst),
2156 }
2157 Ok(())
2158 }
2159 fn emit_rbit(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2160 match (sz, src, dst) {
2161 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
2162 dynasm!(self ; rbit X(dst), X(src));
2163 }
2164 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
2165 dynasm!(self ; rbit W(dst), W(src));
2166 }
2167 _ => codegen_error!("singlepass can't emit CLS {:?} {:?} {:?}", sz, src, dst),
2168 }
2169 Ok(())
2170 }
2171
2172 fn emit_label(&mut self, label: Label) -> Result<(), CompileError> {
2173 dynasm!(self ; => label);
2174 Ok(())
2175 }
2176 fn emit_load_label(&mut self, reg: GPR, label: Label) -> Result<(), CompileError> {
2177 dynasm!(self ; adr X(reg), =>label);
2178 Ok(())
2179 }
2180
2181 fn emit_b_label(&mut self, label: Label) -> Result<(), CompileError> {
2182 dynasm!(self ; b =>label);
2183 Ok(())
2184 }
2185 fn emit_cbz_label(
2186 &mut self,
2187 sz: Size,
2188 reg: Location,
2189 label: Label,
2190 ) -> Result<(), CompileError> {
2191 match (sz, reg) {
2192 (Size::S32, Location::GPR(reg)) => {
2193 dynasm!(self ; cbz W(reg), =>label);
2194 }
2195 (Size::S64, Location::GPR(reg)) => {
2196 dynasm!(self ; cbz X(reg), =>label);
2197 }
2198 _ => codegen_error!("singlepass can't emit CBZ {:?} {:?} {:?}", sz, reg, label),
2199 }
2200 Ok(())
2201 }
2202 fn emit_cbnz_label(
2203 &mut self,
2204 sz: Size,
2205 reg: Location,
2206 label: Label,
2207 ) -> Result<(), CompileError> {
2208 match (sz, reg) {
2209 (Size::S32, Location::GPR(reg)) => {
2210 dynasm!(self ; cbnz W(reg), =>label);
2211 }
2212 (Size::S64, Location::GPR(reg)) => {
2213 dynasm!(self ; cbnz X(reg), =>label);
2214 }
2215 _ => codegen_error!("singlepass can't emit CBNZ {:?} {:?} {:?}", sz, reg, label),
2216 }
2217 Ok(())
2218 }
2219 fn emit_cbz_label_far(
2220 &mut self,
2221 sz: Size,
2222 reg: Location,
2223 label: Label,
2224 ) -> Result<(), CompileError> {
2225 let near_label: Label = self.get_label();
2226 let continue_label: Label = self.get_label();
2227
2228 match (sz, reg) {
2229 (Size::S32, Location::GPR(reg)) => {
2230 dynasm!(self ; cbz W(reg), => near_label);
2231 dynasm!(self ; b => continue_label);
2232 }
2233 (Size::S64, Location::GPR(reg)) => {
2234 dynasm!(self ; cbz X(reg), => near_label);
2235 dynasm!(self ; b => continue_label);
2236 }
2237 _ => codegen_error!("singlepass can't emit CBZ {:?} {:?} {:?}", sz, reg, label),
2238 }
2239 self.emit_label(near_label)?;
2240 dynasm!(self ; b => label );
2241
2242 self.emit_label(continue_label)?;
2243
2244 Ok(())
2245 }
2246 fn emit_tbz_label(
2247 &mut self,
2248 sz: Size,
2249 reg: Location,
2250 n: u32,
2251 label: Label,
2252 ) -> Result<(), CompileError> {
2253 match (sz, reg) {
2254 (Size::S32, Location::GPR(reg)) => {
2255 dynasm!(self ; tbz W(reg), n, =>label);
2256 }
2257 (Size::S64, Location::GPR(reg)) => {
2258 dynasm!(self ; tbz X(reg), n, =>label);
2259 }
2260 _ => codegen_error!(
2261 "singlepass can't emit TBZ {:?} {:?} {:?} {:?}",
2262 sz,
2263 reg,
2264 n,
2265 label
2266 ),
2267 }
2268 Ok(())
2269 }
2270 fn emit_tbnz_label(
2271 &mut self,
2272 sz: Size,
2273 reg: Location,
2274 n: u32,
2275 label: Label,
2276 ) -> Result<(), CompileError> {
2277 match (sz, reg) {
2278 (Size::S32, Location::GPR(reg)) => {
2279 dynasm!(self ; tbnz W(reg), n, =>label);
2280 }
2281 (Size::S64, Location::GPR(reg)) => {
2282 dynasm!(self ; tbnz X(reg), n, =>label);
2283 }
2284 _ => codegen_error!(
2285 "singlepass can't emit TBNZ {:?} {:?} {:?} {:?}",
2286 sz,
2287 reg,
2288 n,
2289 label
2290 ),
2291 }
2292 Ok(())
2293 }
2294 fn emit_bcond_label(&mut self, condition: Condition, label: Label) -> Result<(), CompileError> {
2295 match condition {
2296 Condition::Eq => dynasm!(self ; b.eq => label),
2297 Condition::Ne => dynasm!(self ; b.ne => label),
2298 Condition::Cs => dynasm!(self ; b.cs => label),
2299 Condition::Cc => dynasm!(self ; b.cc => label),
2300 Condition::Mi => dynasm!(self ; b.mi => label),
2301 Condition::Pl => dynasm!(self ; b.pl => label),
2302 Condition::Vs => dynasm!(self ; b.vs => label),
2303 Condition::Vc => dynasm!(self ; b.vc => label),
2304 Condition::Hi => dynasm!(self ; b.hi => label),
2305 Condition::Ls => dynasm!(self ; b.ls => label),
2306 Condition::Ge => dynasm!(self ; b.ge => label),
2307 Condition::Lt => dynasm!(self ; b.lt => label),
2308 Condition::Gt => dynasm!(self ; b.gt => label),
2309 Condition::Le => dynasm!(self ; b.le => label),
2310 Condition::Al => dynasm!(self ; b => label),
2311 }
2312 Ok(())
2313 }
2314 fn emit_bcond_label_far(
2315 &mut self,
2316 condition: Condition,
2317 label: Label,
2318 ) -> Result<(), CompileError> {
2319 let cont: Label = self.get_label();
2320 match condition {
2321 Condition::Eq => dynasm!(self ; b.ne => cont),
2323 Condition::Ne => dynasm!(self ; b.eq => cont),
2324 Condition::Cs => dynasm!(self ; b.cc => cont),
2325 Condition::Cc => dynasm!(self ; b.cs => cont),
2326 Condition::Mi => dynasm!(self ; b.pl => cont),
2327 Condition::Pl => dynasm!(self ; b.mi => cont),
2328 Condition::Vs => dynasm!(self ; b.vc => cont),
2329 Condition::Vc => dynasm!(self ; b.vs => cont),
2330 Condition::Hi => dynasm!(self ; b.ls => cont),
2331 Condition::Ls => dynasm!(self ; b.hi => cont),
2332 Condition::Ge => dynasm!(self ; b.lt => cont),
2333 Condition::Lt => dynasm!(self ; b.ge => cont),
2334 Condition::Gt => dynasm!(self ; b.le => cont),
2335 Condition::Le => dynasm!(self ; b.gt => cont),
2336 Condition::Al => { }
2337 }
2338 dynasm!(self ; b => label);
2339 self.emit_label(cont)?;
2340 Ok(())
2341 }
2342 fn emit_b_register(&mut self, reg: GPR) -> Result<(), CompileError> {
2343 dynasm!(self ; br X(reg));
2344 Ok(())
2345 }
2346 fn emit_call_label(&mut self, label: Label) -> Result<(), CompileError> {
2347 dynasm!(self ; bl =>label);
2348 Ok(())
2349 }
2350 fn emit_call_register(&mut self, reg: GPR) -> Result<(), CompileError> {
2351 dynasm!(self ; blr X(reg));
2352 Ok(())
2353 }
2354 fn emit_ret(&mut self) -> Result<(), CompileError> {
2355 dynasm!(self ; ret);
2356 Ok(())
2357 }
2358
2359 fn emit_udf(&mut self, payload: u16) -> Result<(), CompileError> {
2360 dynasm!(self ; udf payload as u32);
2361 Ok(())
2362 }
2363 fn emit_dmb(&mut self) -> Result<(), CompileError> {
2364 dynasm!(self ; dmb ish);
2365 Ok(())
2366 }
2367 fn emit_brk(&mut self) -> Result<(), CompileError> {
2368 dynasm!(self ; brk 0);
2369 Ok(())
2370 }
2371
2372 fn emit_fcmp(&mut self, sz: Size, src1: Location, src2: Location) -> Result<(), CompileError> {
2373 match (sz, src1, src2) {
2374 (Size::S32, Location::SIMD(src1), Location::SIMD(src2)) => {
2375 dynasm!(self ; fcmp S(src1), S(src2));
2376 }
2377 (Size::S64, Location::SIMD(src1), Location::SIMD(src2)) => {
2378 dynasm!(self ; fcmp D(src1), D(src2));
2379 }
2380 _ => codegen_error!("singlepass can't emit FCMP {:?} {:?} {:?}", sz, src1, src2),
2381 }
2382 Ok(())
2383 }
2384
2385 fn emit_fneg(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2386 match (sz, src, dst) {
2387 (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2388 dynasm!(self ; fneg S(dst), S(src));
2389 }
2390 (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2391 dynasm!(self ; fneg D(dst), D(src));
2392 }
2393 _ => codegen_error!("singlepass can't emit FNEG {:?} {:?} {:?}", sz, src, dst),
2394 }
2395 Ok(())
2396 }
2397 fn emit_fsqrt(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2398 match (sz, src, dst) {
2399 (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2400 dynasm!(self ; fsqrt S(dst), S(src));
2401 }
2402 (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2403 dynasm!(self ; fsqrt D(dst), D(src));
2404 }
2405 _ => codegen_error!("singlepass can't emit FSQRT {:?} {:?} {:?}", sz, src, dst),
2406 }
2407 Ok(())
2408 }
2409
2410 fn emit_fadd(
2411 &mut self,
2412 sz: Size,
2413 src1: Location,
2414 src2: Location,
2415 dst: Location,
2416 ) -> Result<(), CompileError> {
2417 match (sz, src1, src2, dst) {
2418 (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2419 dynasm!(self ; fadd S(dst), S(src1), S(src2));
2420 }
2421 (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2422 dynasm!(self ; fadd D(dst), D(src1), D(src2));
2423 }
2424 _ => codegen_error!(
2425 "singlepass can't emit FADD {:?} {:?} {:?} {:?}",
2426 sz,
2427 src1,
2428 src2,
2429 dst
2430 ),
2431 }
2432 Ok(())
2433 }
2434 fn emit_fsub(
2435 &mut self,
2436 sz: Size,
2437 src1: Location,
2438 src2: Location,
2439 dst: Location,
2440 ) -> Result<(), CompileError> {
2441 match (sz, src1, src2, dst) {
2442 (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2443 dynasm!(self ; fsub S(dst), S(src1), S(src2));
2444 }
2445 (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2446 dynasm!(self ; fsub D(dst), D(src1), D(src2));
2447 }
2448 _ => codegen_error!(
2449 "singlepass can't emit FSUB {:?} {:?} {:?} {:?}",
2450 sz,
2451 src1,
2452 src2,
2453 dst
2454 ),
2455 }
2456 Ok(())
2457 }
2458 fn emit_fmul(
2459 &mut self,
2460 sz: Size,
2461 src1: Location,
2462 src2: Location,
2463 dst: Location,
2464 ) -> Result<(), CompileError> {
2465 match (sz, src1, src2, dst) {
2466 (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2467 dynasm!(self ; fmul S(dst), S(src1), S(src2));
2468 }
2469 (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2470 dynasm!(self ; fmul D(dst), D(src1), D(src2));
2471 }
2472 _ => codegen_error!(
2473 "singlepass can't emit FMUL {:?} {:?} {:?} {:?}",
2474 sz,
2475 src1,
2476 src2,
2477 dst
2478 ),
2479 }
2480 Ok(())
2481 }
2482 fn emit_fdiv(
2483 &mut self,
2484 sz: Size,
2485 src1: Location,
2486 src2: Location,
2487 dst: Location,
2488 ) -> Result<(), CompileError> {
2489 match (sz, src1, src2, dst) {
2490 (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2491 dynasm!(self ; fdiv S(dst), S(src1), S(src2));
2492 }
2493 (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2494 dynasm!(self ; fdiv D(dst), D(src1), D(src2));
2495 }
2496 _ => codegen_error!(
2497 "singlepass can't emit FDIV {:?} {:?} {:?} {:?}",
2498 sz,
2499 src1,
2500 src2,
2501 dst
2502 ),
2503 }
2504 Ok(())
2505 }
2506
2507 fn emit_fmin(
2508 &mut self,
2509 sz: Size,
2510 src1: Location,
2511 src2: Location,
2512 dst: Location,
2513 ) -> Result<(), CompileError> {
2514 match (sz, src1, src2, dst) {
2515 (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2516 dynasm!(self ; fmin S(dst), S(src1), S(src2));
2517 }
2518 (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2519 dynasm!(self ; fmin D(dst), D(src1), D(src2));
2520 }
2521 _ => codegen_error!(
2522 "singlepass can't emit FMIN {:?} {:?} {:?} {:?}",
2523 sz,
2524 src1,
2525 src2,
2526 dst
2527 ),
2528 }
2529 Ok(())
2530 }
2531 fn emit_fmax(
2532 &mut self,
2533 sz: Size,
2534 src1: Location,
2535 src2: Location,
2536 dst: Location,
2537 ) -> Result<(), CompileError> {
2538 match (sz, src1, src2, dst) {
2539 (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2540 dynasm!(self ; fmax S(dst), S(src1), S(src2));
2541 }
2542 (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2543 dynasm!(self ; fmax D(dst), D(src1), D(src2));
2544 }
2545 _ => codegen_error!(
2546 "singlepass can't emit FMAX {:?} {:?} {:?} {:?}",
2547 sz,
2548 src1,
2549 src2,
2550 dst
2551 ),
2552 }
2553 Ok(())
2554 }
2555
2556 fn emit_frintz(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2557 match (sz, src, dst) {
2558 (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2559 dynasm!(self ; frintz S(dst), S(src));
2560 }
2561 (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2562 dynasm!(self ; frintz D(dst), D(src));
2563 }
2564 _ => codegen_error!("singlepass can't emit FRINTZ {:?} {:?} {:?}", sz, src, dst),
2565 }
2566 Ok(())
2567 }
2568 fn emit_frintn(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2569 match (sz, src, dst) {
2570 (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2571 dynasm!(self ; frintn S(dst), S(src));
2572 }
2573 (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2574 dynasm!(self ; frintn D(dst), D(src));
2575 }
2576 _ => codegen_error!("singlepass can't emit FRINTN {:?} {:?} {:?}", sz, src, dst),
2577 }
2578 Ok(())
2579 }
2580 fn emit_frintm(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2581 match (sz, src, dst) {
2582 (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2583 dynasm!(self ; frintm S(dst), S(src));
2584 }
2585 (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2586 dynasm!(self ; frintm D(dst), D(src));
2587 }
2588 _ => codegen_error!("singlepass can't emit FRINTM {:?} {:?} {:?}", sz, src, dst),
2589 }
2590 Ok(())
2591 }
2592 fn emit_frintp(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2593 match (sz, src, dst) {
2594 (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2595 dynasm!(self ; frintp S(dst), S(src));
2596 }
2597 (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2598 dynasm!(self ; frintp D(dst), D(src));
2599 }
2600 _ => codegen_error!("singlepass can't emit FRINTP {:?} {:?} {:?}", sz, src, dst),
2601 }
2602 Ok(())
2603 }
2604
2605 fn emit_scvtf(
2606 &mut self,
2607 sz_in: Size,
2608 src: Location,
2609 sz_out: Size,
2610 dst: Location,
2611 ) -> Result<(), CompileError> {
2612 match (sz_in, src, sz_out, dst) {
2613 (Size::S32, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
2614 dynasm!(self ; scvtf S(dst), W(src));
2615 }
2616 (Size::S64, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
2617 dynasm!(self ; scvtf S(dst), X(src));
2618 }
2619 (Size::S32, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
2620 dynasm!(self ; scvtf D(dst), W(src));
2621 }
2622 (Size::S64, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
2623 dynasm!(self ; scvtf D(dst), X(src));
2624 }
2625 _ => codegen_error!(
2626 "singlepass can't emit SCVTF {:?} {:?} {:?} {:?}",
2627 sz_in,
2628 src,
2629 sz_out,
2630 dst
2631 ),
2632 }
2633 Ok(())
2634 }
2635 fn emit_ucvtf(
2636 &mut self,
2637 sz_in: Size,
2638 src: Location,
2639 sz_out: Size,
2640 dst: Location,
2641 ) -> Result<(), CompileError> {
2642 match (sz_in, src, sz_out, dst) {
2643 (Size::S32, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
2644 dynasm!(self ; ucvtf S(dst), W(src));
2645 }
2646 (Size::S64, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
2647 dynasm!(self ; ucvtf S(dst), X(src));
2648 }
2649 (Size::S32, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
2650 dynasm!(self ; ucvtf D(dst), W(src));
2651 }
2652 (Size::S64, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
2653 dynasm!(self ; ucvtf D(dst), X(src));
2654 }
2655 _ => codegen_error!(
2656 "singlepass can't emit UCVTF {:?} {:?} {:?} {:?}",
2657 sz_in,
2658 src,
2659 sz_out,
2660 dst
2661 ),
2662 }
2663 Ok(())
2664 }
2665 fn emit_fcvt(&mut self, sz_in: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2666 match (sz_in, src, dst) {
2667 (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2668 dynasm!(self ; fcvt D(dst), S(src));
2669 }
2670 (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2671 dynasm!(self ; fcvt S(dst), D(src));
2672 }
2673 _ => codegen_error!(
2674 "singlepass can't emit UCVTF {:?} {:?} {:?}",
2675 sz_in,
2676 src,
2677 dst
2678 ),
2679 }
2680 Ok(())
2681 }
2682 fn emit_fcvtzs(
2683 &mut self,
2684 sz_in: Size,
2685 src: Location,
2686 sz_out: Size,
2687 dst: Location,
2688 ) -> Result<(), CompileError> {
2689 match (sz_in, src, sz_out, dst) {
2690 (Size::S32, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
2691 dynasm!(self ; fcvtzs W(dst), S(src));
2692 }
2693 (Size::S64, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
2694 dynasm!(self ; fcvtzs W(dst), D(src));
2695 }
2696 (Size::S32, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
2697 dynasm!(self ; fcvtzs X(dst), S(src));
2698 }
2699 (Size::S64, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
2700 dynasm!(self ; fcvtzs X(dst), D(src));
2701 }
2702 _ => codegen_error!(
2703 "singlepass can't emit FCVTZS {:?} {:?} {:?} {:?}",
2704 sz_in,
2705 src,
2706 sz_out,
2707 dst
2708 ),
2709 }
2710 Ok(())
2711 }
2712 fn emit_fcvtzu(
2713 &mut self,
2714 sz_in: Size,
2715 src: Location,
2716 sz_out: Size,
2717 dst: Location,
2718 ) -> Result<(), CompileError> {
2719 match (sz_in, src, sz_out, dst) {
2720 (Size::S32, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
2721 dynasm!(self ; fcvtzu W(dst), S(src));
2722 }
2723 (Size::S64, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
2724 dynasm!(self ; fcvtzu W(dst), D(src));
2725 }
2726 (Size::S32, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
2727 dynasm!(self ; fcvtzu X(dst), S(src));
2728 }
2729 (Size::S64, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
2730 dynasm!(self ; fcvtzu X(dst), D(src));
2731 }
2732 _ => codegen_error!(
2733 "singlepass can't emit FCVTZU {:?} {:?} {:?} {:?}",
2734 sz_in,
2735 src,
2736 sz_out,
2737 dst
2738 ),
2739 }
2740 Ok(())
2741 }
2742
2743 fn emit_read_fpcr(&mut self, reg: GPR) -> Result<(), CompileError> {
2745 dynasm!(self ; mrs X(reg), 0b1_011_0100_0100_000);
2746 Ok(())
2747 }
2748 fn emit_write_fpcr(&mut self, reg: GPR) -> Result<(), CompileError> {
2749 dynasm!(self ; msr 0b1_011_0100_0100_000, X(reg));
2750 Ok(())
2751 }
2752 fn emit_read_fpsr(&mut self, reg: GPR) -> Result<(), CompileError> {
2754 dynasm!(self ; mrs X(reg), 0b1_011_0100_0100_001);
2755 Ok(())
2756 }
2757 fn emit_write_fpsr(&mut self, reg: GPR) -> Result<(), CompileError> {
2758 dynasm!(self ; msr 0b1_011_0100_0100_001, X(reg));
2759 Ok(())
2760 }
2761
2762 fn emit_cnt(&mut self, src: NEON, dst: NEON) -> Result<(), CompileError> {
2763 dynasm!(self ; cnt V(dst).B8, V(src).B8);
2764 Ok(())
2765 }
2766
2767 fn emit_addv(&mut self, src: NEON, dst: NEON) -> Result<(), CompileError> {
2768 dynasm!(self ; addv B(dst), V(src).B8);
2769 Ok(())
2770 }
2771
2772 #[allow(clippy::unit_arg)]
2773 fn emit_fmov(
2774 &mut self,
2775 src_size: Size,
2776 src: Location,
2777 dst_size: Size,
2778 dst: Location,
2779 ) -> Result<(), CompileError> {
2780 let mut err = false;
2781 match (src, dst) {
2782 (AbstractLocation::GPR(src), AbstractLocation::SIMD(dst)) => {
2783 match (src_size, dst_size) {
2784 (Size::S32, Size::S32) => dynasm!(self ; fmov S(dst), W(src)),
2785 (Size::S64, Size::S64) => dynasm!(self ; fmov D(dst), X(src)),
2786 _ => {
2787 err = true;
2788 }
2789 }
2790 }
2791 (AbstractLocation::SIMD(src), AbstractLocation::GPR(dst)) => {
2792 match (src_size, dst_size) {
2793 (Size::S32, Size::S32) => dynasm!(self ; fmov W(dst), S(src)),
2794 (Size::S64, Size::S64) => dynasm!(self ; fmov X(dst), D(src)),
2795 _ => {
2796 err = true;
2797 }
2798 }
2799 }
2800 _ => {
2804 err = true;
2805 }
2806 };
2807
2808 if err {
2809 codegen_error!(
2810 "singlepass can't generate fmov instruction for src_size: {:?}, src: {:?}, dst_size: {:?}, dst: {:?}",
2811 src_size,
2812 src,
2813 dst_size,
2814 dst,
2815 )
2816 }
2817 Ok(())
2818 }
2819}
2820
2821pub fn gen_std_trampoline_arm64(
2822 sig: &FunctionType,
2823 calling_convention: CallingConvention,
2824) -> Result<FunctionBody, CompileError> {
2825 let mut a = Assembler::new(0);
2826
2827 let fptr = GPR::X27;
2828 let args = GPR::X28;
2829
2830 dynasm!(a
2831 ; sub sp, sp, 32
2832 ; stp x29, x30, [sp]
2833 ; stp X(fptr), X(args), [sp, 16]
2834 ; mov x29, sp
2835 ; mov X(fptr), x1
2836 ; mov X(args), x2
2837 );
2838
2839 let stack_args = sig.params().len().saturating_sub(7); let mut stack_offset = stack_args as u32 * 8;
2841 if stack_args > 0 {
2842 if !stack_offset.is_multiple_of(16) {
2843 stack_offset += 8;
2844 assert!(stack_offset.is_multiple_of(16));
2845 }
2846 if stack_offset < 0x1000 {
2847 dynasm!(a ; sub sp, sp, stack_offset);
2848 } else {
2849 a.emit_mov_imm(Location::GPR(GPR::X26), stack_offset as u64)?;
2850 dynasm!(a ; sub sp, sp, x26);
2851 }
2852 }
2853
2854 let mut caller_stack_offset: i32 = 0;
2857 for (i, param) in sig.params().iter().enumerate() {
2858 let sz = match *param {
2859 Type::I32 | Type::F32 => Size::S32,
2860 Type::I64 | Type::F64 => Size::S64,
2861 Type::ExternRef => Size::S64,
2862 Type::FuncRef => Size::S64,
2863 _ => codegen_error!(
2864 "singlepass unsupported param type for trampoline {:?}",
2865 *param
2866 ),
2867 };
2868 match i {
2869 0..=6 => {
2870 a.emit_ldr(
2871 sz,
2872 Location::GPR(GPR::from_index(i + 1).unwrap()),
2873 Location::Memory(args, (i * 16) as i32),
2874 )?;
2875 }
2876 _ => {
2877 #[allow(clippy::single_match)]
2878 match calling_convention {
2879 CallingConvention::AppleAarch64 => {
2880 let sz = 1
2881 << match sz {
2882 Size::S8 => 0,
2883 Size::S16 => 1,
2884 Size::S32 => 2,
2885 Size::S64 => 3,
2886 };
2887 if sz > 1 && caller_stack_offset & (sz - 1) != 0 {
2889 caller_stack_offset = (caller_stack_offset + (sz - 1)) & !(sz - 1);
2890 }
2891 }
2892 _ => (),
2893 };
2894 a.emit_ldr(
2896 sz,
2897 Location::GPR(GPR::X16),
2898 Location::Memory(args, (i * 16) as i32),
2899 )?;
2900 a.emit_str(
2901 sz,
2902 Location::GPR(GPR::X16),
2903 Location::Memory(GPR::XzrSp, caller_stack_offset),
2904 )?;
2905 match calling_convention {
2906 CallingConvention::AppleAarch64 => {
2907 caller_stack_offset += 1
2908 << match sz {
2909 Size::S8 => 0,
2910 Size::S16 => 1,
2911 Size::S32 => 2,
2912 Size::S64 => 3,
2913 };
2914 }
2915 _ => {
2916 caller_stack_offset += 8;
2917 }
2918 }
2919 }
2920 }
2921 }
2922
2923 dynasm!(a ; blr X(fptr));
2924
2925 if !sig.results().is_empty() {
2927 a.emit_str(Size::S64, Location::GPR(GPR::X0), Location::Memory(args, 0))?;
2928 }
2929
2930 dynasm!(a
2932 ; ldp X(fptr), X(args), [x29, 16]
2933 ; ldp x29, x30, [x29]
2934 );
2935 let restored_stack_offset = 32 + stack_offset;
2936 if restored_stack_offset < 0x1000 {
2937 dynasm!(a; add sp, sp, restored_stack_offset);
2938 } else {
2939 a.emit_mov_imm(Location::GPR(GPR::X26), restored_stack_offset as u64)?;
2940 dynasm!(a; add sp, sp, x26);
2941 }
2942 dynasm!(a; ret);
2943
2944 let mut body = a.finalize().unwrap();
2945 body.shrink_to_fit();
2946 Ok(FunctionBody {
2947 body,
2948 unwind_info: None,
2949 })
2950}
2951pub fn gen_std_dynamic_import_trampoline_arm64(
2953 vmoffsets: &VMOffsets,
2954 sig: &FunctionType,
2955 calling_convention: CallingConvention,
2956) -> Result<FunctionBody, CompileError> {
2957 let mut a = Assembler::new(0);
2958 let stack_offset: usize = 16 * std::cmp::max(sig.params().len(), sig.results().len());
2960 a.emit_stpdb(
2962 Size::S64,
2963 Location::GPR(GPR::X30),
2964 Location::GPR(GPR::X26),
2965 GPR::XzrSp,
2966 16,
2967 )?;
2968
2969 if stack_offset != 0 {
2970 if stack_offset < 0x1000 {
2971 a.emit_sub(
2972 Size::S64,
2973 Location::GPR(GPR::XzrSp),
2974 Location::Imm32(stack_offset as _),
2975 Location::GPR(GPR::XzrSp),
2976 )?;
2977 } else {
2978 a.emit_mov_imm(Location::GPR(GPR::X26), stack_offset as u64)?;
2979 a.emit_sub(
2980 Size::S64,
2981 Location::GPR(GPR::XzrSp),
2982 Location::GPR(GPR::X26),
2983 Location::GPR(GPR::XzrSp),
2984 )?;
2985 }
2986 }
2987
2988 if !sig.params().is_empty() {
2990 let mut argalloc = ArgumentRegisterAllocator::default();
2991 argalloc.next(Type::I64, calling_convention).unwrap(); let mut stack_param_count: usize = 0;
2994
2995 for (i, ty) in sig.params().iter().enumerate() {
2996 let source_loc = match argalloc.next(*ty, calling_convention)? {
2997 Some(ARM64Register::GPR(gpr)) => Location::GPR(gpr),
2998 Some(ARM64Register::NEON(neon)) => Location::SIMD(neon),
2999 None => {
3000 let sz = match calling_convention {
3001 CallingConvention::AppleAarch64 => match *ty {
3002 Type::I32 | Type::F32 => Size::S32,
3003 _ => {
3004 if stack_param_count & 7 != 0 {
3005 stack_param_count = (stack_param_count + 7) & !7;
3006 };
3007 Size::S64
3008 }
3009 },
3010 _ => Size::S64,
3011 };
3012 a.emit_ldr(
3013 sz,
3014 Location::GPR(GPR::X26),
3015 Location::Memory(GPR::XzrSp, (stack_offset + 16 + stack_param_count) as _),
3016 )?;
3017 stack_param_count += match sz {
3018 Size::S32 => 4,
3019 Size::S64 => 8,
3020 _ => codegen_error!(
3021 "singlepass unreachable in gen_std_dynamic_import_trampoline_arm64"
3022 ),
3023 };
3024 Location::GPR(GPR::X26)
3025 }
3026 };
3027 a.emit_str(
3028 Size::S64,
3029 source_loc,
3030 Location::Memory(GPR::XzrSp, (i * 16) as _),
3031 )?;
3032
3033 a.emit_str(
3035 Size::S64,
3036 Location::GPR(GPR::XzrSp), Location::Memory(GPR::XzrSp, (i * 16 + 8) as _), )?;
3039 }
3040 }
3041
3042 #[allow(clippy::match_single_binding)]
3043 match calling_convention {
3044 _ => {
3045 let offset = vmoffsets.vmdynamicfunction_import_context_address();
3047 a.emit_ldur(Size::S64, Location::GPR(GPR::X26), GPR::X0, offset as i32)?;
3048 a.emit_add(
3050 Size::S64,
3051 Location::GPR(GPR::XzrSp),
3052 Location::Imm8(0),
3053 Location::GPR(GPR::X1),
3054 )?;
3055 }
3056 };
3057
3058 a.emit_call_register(GPR::X26)?;
3060
3061 if !sig.results().is_empty() {
3063 assert_eq!(sig.results().len(), 1);
3064 a.emit_ldr(
3065 Size::S64,
3066 Location::GPR(GPR::X0),
3067 Location::Memory(GPR::XzrSp, 0),
3068 )?;
3069 }
3070
3071 if stack_offset != 0 {
3073 if stack_offset < 0x1000 {
3074 a.emit_add(
3075 Size::S64,
3076 Location::GPR(GPR::XzrSp),
3077 Location::Imm32(stack_offset as _),
3078 Location::GPR(GPR::XzrSp),
3079 )?;
3080 } else {
3081 a.emit_mov_imm(Location::GPR(GPR::X26), stack_offset as u64)?;
3082 a.emit_add(
3083 Size::S64,
3084 Location::GPR(GPR::XzrSp),
3085 Location::GPR(GPR::X26),
3086 Location::GPR(GPR::XzrSp),
3087 )?;
3088 }
3089 }
3090 a.emit_ldpia(
3091 Size::S64,
3092 Location::GPR(GPR::X30),
3093 Location::GPR(GPR::X26),
3094 GPR::XzrSp,
3095 16,
3096 )?;
3097
3098 a.emit_ret()?;
3100
3101 let mut body = a.finalize().unwrap();
3102 body.shrink_to_fit();
3103 Ok(FunctionBody {
3104 body,
3105 unwind_info: None,
3106 })
3107}
3108pub fn gen_import_call_trampoline_arm64(
3110 vmoffsets: &VMOffsets,
3111 index: FunctionIndex,
3112 sig: &FunctionType,
3113 calling_convention: CallingConvention,
3114) -> Result<CustomSection, CompileError> {
3115 let mut a = Assembler::new(0);
3116
3117 if sig
3122 .params()
3123 .iter()
3124 .any(|&x| x == Type::F32 || x == Type::F64)
3125 {
3126 #[allow(clippy::match_single_binding)]
3127 match calling_convention {
3128 _ => {
3129 let stack_offset: i32 = if sig.params().len() > 7 {
3131 7 * 8
3132 } else {
3133 (sig.params().len() as i32) * 8
3134 };
3135 let stack_offset = if stack_offset & 15 != 0 {
3136 stack_offset + 8
3137 } else {
3138 stack_offset
3139 };
3140 if stack_offset > 0 {
3141 if stack_offset < 0x1000 {
3142 a.emit_sub(
3143 Size::S64,
3144 Location::GPR(GPR::XzrSp),
3145 Location::Imm32(stack_offset as u32),
3146 Location::GPR(GPR::XzrSp),
3147 )?;
3148 } else {
3149 a.emit_mov_imm(Location::GPR(GPR::X16), stack_offset as u64)?;
3150 a.emit_sub(
3151 Size::S64,
3152 Location::GPR(GPR::XzrSp),
3153 Location::GPR(GPR::X16),
3154 Location::GPR(GPR::XzrSp),
3155 )?;
3156 }
3157 }
3158
3159 static PARAM_REGS: &[GPR] = &[
3161 GPR::X1,
3162 GPR::X2,
3163 GPR::X3,
3164 GPR::X4,
3165 GPR::X5,
3166 GPR::X6,
3167 GPR::X7,
3168 ];
3169 let mut param_locations = vec![];
3170 for (i, param_reg) in PARAM_REGS.iter().enumerate().take(sig.params().len()) {
3171 let loc = match i {
3172 0..=6 => {
3173 let loc = Location::Memory(GPR::XzrSp, (i * 8) as i32);
3174 a.emit_str(Size::S64, Location::GPR(*param_reg), loc)?;
3175 loc
3176 }
3177 _ => Location::Memory(GPR::XzrSp, stack_offset + ((i - 7) * 8) as i32),
3178 };
3179 param_locations.push(loc);
3180 }
3181
3182 let mut caller_stack_offset: i32 = 0;
3184 let mut argalloc = ArgumentRegisterAllocator::default();
3185 argalloc.next(Type::I64, calling_convention).unwrap(); for (i, ty) in sig.params().iter().enumerate() {
3187 let prev_loc = param_locations[i];
3188 let targ = match argalloc.next(*ty, calling_convention)? {
3189 Some(ARM64Register::GPR(gpr)) => Location::GPR(gpr),
3190 Some(ARM64Register::NEON(neon)) => Location::SIMD(neon),
3191 None => {
3192 a.emit_ldr(Size::S64, Location::GPR(GPR::X16), prev_loc)?;
3194 a.emit_str(
3195 Size::S64,
3196 Location::GPR(GPR::X16),
3197 Location::Memory(GPR::XzrSp, stack_offset + caller_stack_offset),
3198 )?;
3199 caller_stack_offset += 8;
3200 continue;
3201 }
3202 };
3203 a.emit_ldr(Size::S64, targ, prev_loc)?;
3204 }
3205
3206 if stack_offset > 0 {
3208 if stack_offset < 0x1000 {
3209 a.emit_add(
3210 Size::S64,
3211 Location::GPR(GPR::XzrSp),
3212 Location::Imm32(stack_offset as u32),
3213 Location::GPR(GPR::XzrSp),
3214 )?;
3215 } else {
3216 a.emit_mov_imm(Location::GPR(GPR::X16), stack_offset as u64)?;
3217 a.emit_add(
3218 Size::S64,
3219 Location::GPR(GPR::XzrSp),
3220 Location::GPR(GPR::X16),
3221 Location::GPR(GPR::XzrSp),
3222 )?;
3223 }
3224 }
3225 }
3226 }
3227 }
3228
3229 let offset = vmoffsets.vmctx_vmfunction_import(index);
3233 let offset =
3236 if (offset > 0 && offset < 0xF8) || (offset > 0 && offset < 0x7FF8 && (offset & 7) == 0) {
3237 offset
3238 } else {
3239 a.emit_mov_imm(Location::GPR(GPR::X16), (offset as i64) as u64)?;
3240 a.emit_add(
3241 Size::S64,
3242 Location::GPR(GPR::X0),
3243 Location::GPR(GPR::X16),
3244 Location::GPR(GPR::X0),
3245 )?;
3246 0
3247 };
3248 #[allow(clippy::match_single_binding)]
3249 match calling_convention {
3250 _ => {
3251 if (offset & 7) == 0 {
3252 a.emit_ldr(
3253 Size::S64,
3254 Location::GPR(GPR::X16),
3255 Location::Memory(GPR::X0, offset as i32), )?;
3257 a.emit_ldr(
3258 Size::S64,
3259 Location::GPR(GPR::X0),
3260 Location::Memory(GPR::X0, offset as i32 + 8), )?;
3262 } else {
3263 a.emit_ldur(
3264 Size::S64,
3265 Location::GPR(GPR::X16),
3266 GPR::X0,
3267 offset as i32, )?;
3269 a.emit_ldur(
3270 Size::S64,
3271 Location::GPR(GPR::X0),
3272 GPR::X0,
3273 offset as i32 + 8, )?;
3275 }
3276 }
3277 }
3278 a.emit_b_register(GPR::X16)?;
3279
3280 let mut contents = a.finalize().unwrap();
3281 contents.shrink_to_fit();
3282 let section_body = SectionBody::new_with_vec(contents);
3283
3284 Ok(CustomSection {
3285 protection: CustomSectionProtection::ReadExecute,
3286 alignment: None,
3287 bytes: section_body,
3288 relocations: vec![],
3289 })
3290}