1pub use crate::{
2 arm64_decl::{ARM64Register, ArgumentRegisterAllocator, GPR, NEON},
3 location::{Multiplier, Reg},
4 machine::{Label, Offset},
5};
6use crate::{
7 codegen_error, common_decl::Size, location::Location as AbstractLocation,
8 machine_arm64::ARM64_RETURN_VALUE_REGISTERS,
9};
10pub use dynasmrt::aarch64::{encode_logical_immediate_32bit, encode_logical_immediate_64bit};
11use dynasmrt::{
12 AssemblyOffset, DynamicLabel, DynasmApi, DynasmLabelApi, VecAssembler,
13 aarch64::Aarch64Relocation,
14};
15use wasmer_compiler::types::{
16 function::FunctionBody,
17 section::{CustomSection, CustomSectionProtection, SectionBody},
18};
19use wasmer_types::{
20 CompileError, FunctionIndex, FunctionType, Type, VMOffsets, target::CallingConvention,
21};
22
23type Assembler = VecAssembler<Aarch64Relocation>;
24
25macro_rules! dynasm {
30 ($a:expr_2021 ; $($tt:tt)*) => {
31 dynasm::dynasm!(
32 $a
33 ; .arch aarch64
34 ; $($tt)*
35 )
36 };
37}
38
39pub type Location = AbstractLocation<GPR, NEON>;
40
41#[derive(Copy, Clone, Debug, Eq, PartialEq)]
42#[allow(dead_code)]
43#[repr(u8)]
44pub enum Condition {
45 Eq = 0,
48 Ne = 1,
50 Cs = 2,
52 Cc = 3,
54 Mi = 4,
56 Pl = 5,
58 Vs = 6,
60 Vc = 7,
62 Hi = 8,
64 Ls = 9,
66 Ge = 10,
68 Lt = 11,
70 Gt = 12,
72 Le = 13,
74 Al = 14,
76}
77
78#[derive(Copy, Clone, Debug, Eq, PartialEq)]
79#[allow(dead_code, clippy::upper_case_acronyms)]
80pub enum NeonOrMemory {
81 NEON(NEON),
82 Memory(GPR, i32),
83}
84
85#[derive(Copy, Clone, Debug)]
86#[allow(dead_code, clippy::upper_case_acronyms)]
87pub enum GPROrMemory {
88 GPR(GPR),
89 Memory(GPR, i32),
90}
91
92#[allow(unused)]
93pub trait EmitterARM64 {
94 fn get_label(&mut self) -> Label;
95 fn get_offset(&self) -> Offset;
96 fn get_jmp_instr_size(&self) -> u8;
97
98 fn finalize_function(&mut self);
99
100 fn emit_str(&mut self, sz: Size, reg: Location, addr: Location) -> Result<(), CompileError>;
101 fn emit_ldr(&mut self, sz: Size, reg: Location, addr: Location) -> Result<(), CompileError>;
102 fn emit_stur(
103 &mut self,
104 sz: Size,
105 reg: Location,
106 addr: GPR,
107 offset: i32,
108 ) -> Result<(), CompileError>;
109 fn emit_ldur(
110 &mut self,
111 sz: Size,
112 reg: Location,
113 addr: GPR,
114 offset: i32,
115 ) -> Result<(), CompileError>;
116 fn emit_strdb(
117 &mut self,
118 sz: Size,
119 reg: Location,
120 addr: GPR,
121 offset: u32,
122 ) -> Result<(), CompileError>;
123 fn emit_stria(
124 &mut self,
125 sz: Size,
126 reg: Location,
127 addr: GPR,
128 offset: u32,
129 ) -> Result<(), CompileError>;
130 fn emit_ldria(
131 &mut self,
132 sz: Size,
133 reg: Location,
134 addr: GPR,
135 offset: u32,
136 ) -> Result<(), CompileError>;
137 fn emit_stpdb(
138 &mut self,
139 sz: Size,
140 reg1: Location,
141 reg2: Location,
142 addr: GPR,
143 offset: u32,
144 ) -> Result<(), CompileError>;
145 fn emit_ldpia(
146 &mut self,
147 sz: Size,
148 reg1: Location,
149 reg2: Location,
150 addr: GPR,
151 offset: u32,
152 ) -> Result<(), CompileError>;
153
154 fn emit_ldrb(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
155 fn emit_ldrh(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
156 fn emit_ldrsb(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
157 fn emit_ldrsh(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
158 fn emit_ldrsw(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
159 fn emit_strb(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
160 fn emit_strh(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
161
162 fn emit_ldaxr(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
163 fn emit_ldaxrb(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
164 fn emit_ldaxrh(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
165 fn emit_stlxr(
166 &mut self,
167 sz: Size,
168 status: Location,
169 reg: Location,
170 dst: Location,
171 ) -> Result<(), CompileError>;
172 fn emit_stlxrb(
173 &mut self,
174 sz: Size,
175 status: Location,
176 reg: Location,
177 dst: Location,
178 ) -> Result<(), CompileError>;
179 fn emit_stlxrh(
180 &mut self,
181 sz: Size,
182 status: Location,
183 reg: Location,
184 dst: Location,
185 ) -> Result<(), CompileError>;
186
187 fn emit_mov(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
188
189 fn emit_movn(&mut self, sz: Size, reg: Location, val: u32) -> Result<(), CompileError>;
190 fn emit_movz(&mut self, reg: Location, val: u32) -> Result<(), CompileError>;
191 fn emit_movk(&mut self, reg: Location, val: u32, shift: u32) -> Result<(), CompileError>;
192
193 fn emit_mov_imm(&mut self, dst: Location, val: u64) -> Result<(), CompileError>;
194
195 fn emit_add(
196 &mut self,
197 sz: Size,
198 src1: Location,
199 src2: Location,
200 dst: Location,
201 ) -> Result<(), CompileError>;
202 fn emit_sub(
203 &mut self,
204 sz: Size,
205 src1: Location,
206 src2: Location,
207 dst: Location,
208 ) -> Result<(), CompileError>;
209 fn emit_mul(
210 &mut self,
211 sz: Size,
212 src1: Location,
213 src2: Location,
214 dst: Location,
215 ) -> Result<(), CompileError>;
216 fn emit_adds(
217 &mut self,
218 sz: Size,
219 src1: Location,
220 src2: Location,
221 dst: Location,
222 ) -> Result<(), CompileError>;
223 fn emit_subs(
224 &mut self,
225 sz: Size,
226 src1: Location,
227 src2: Location,
228 dst: Location,
229 ) -> Result<(), CompileError>;
230
231 fn emit_add_lsl(
232 &mut self,
233 sz: Size,
234 src1: Location,
235 src2: Location,
236 lsl: u32,
237 dst: Location,
238 ) -> Result<(), CompileError>;
239
240 fn emit_cmp(&mut self, sz: Size, left: Location, right: Location) -> Result<(), CompileError>;
241 fn emit_tst(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
242
243 fn emit_lsl(
244 &mut self,
245 sz: Size,
246 src1: Location,
247 src2: Location,
248 dst: Location,
249 ) -> Result<(), CompileError>;
250 fn emit_lsr(
251 &mut self,
252 sz: Size,
253 src1: Location,
254 src2: Location,
255 dst: Location,
256 ) -> Result<(), CompileError>;
257 fn emit_asr(
258 &mut self,
259 sz: Size,
260 src1: Location,
261 src2: Location,
262 dst: Location,
263 ) -> Result<(), CompileError>;
264 fn emit_ror(
265 &mut self,
266 sz: Size,
267 src1: Location,
268 src2: Location,
269 dst: Location,
270 ) -> Result<(), CompileError>;
271
272 fn emit_or(
273 &mut self,
274 sz: Size,
275 src1: Location,
276 src2: Location,
277 dst: Location,
278 ) -> Result<(), CompileError>;
279 fn emit_and(
280 &mut self,
281 sz: Size,
282 src1: Location,
283 src2: Location,
284 dst: Location,
285 ) -> Result<(), CompileError>;
286 fn emit_eor(
287 &mut self,
288 sz: Size,
289 src1: Location,
290 src2: Location,
291 dst: Location,
292 ) -> Result<(), CompileError>;
293
294 fn emit_bfc(
295 &mut self,
296 se: Size,
297 lsb: u32,
298 width: u32,
299 dst: Location,
300 ) -> Result<(), CompileError>;
301 fn emit_bfi(
302 &mut self,
303 se: Size,
304 src: Location,
305 lsb: u32,
306 width: u32,
307 dst: Location,
308 ) -> Result<(), CompileError>;
309
310 fn emit_udiv(
311 &mut self,
312 sz: Size,
313 src1: Location,
314 src2: Location,
315 dst: Location,
316 ) -> Result<(), CompileError>;
317 fn emit_sdiv(
318 &mut self,
319 sz: Size,
320 src1: Location,
321 src2: Location,
322 dst: Location,
323 ) -> Result<(), CompileError>;
324 fn emit_msub(
326 &mut self,
327 sz: Size,
328 a: Location,
329 b: Location,
330 c: Location,
331 dst: Location,
332 ) -> Result<(), CompileError>;
333
334 fn emit_sxtb(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
335 fn emit_sxth(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
336 fn emit_sxtw(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
337 fn emit_uxtb(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
338 fn emit_uxth(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
339
340 fn emit_cset(&mut self, sz: Size, dst: Location, cond: Condition) -> Result<(), CompileError>;
341 fn emit_csetm(&mut self, sz: Size, dst: Location, cond: Condition) -> Result<(), CompileError>;
342 fn emit_cinc(
343 &mut self,
344 sz: Size,
345 src: Location,
346 dst: Location,
347 cond: Condition,
348 ) -> Result<(), CompileError>;
349 fn emit_clz(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
350 fn emit_rbit(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
351
352 fn emit_label(&mut self, label: Label) -> Result<(), CompileError>;
353 fn emit_load_label(&mut self, reg: GPR, label: Label) -> Result<(), CompileError>;
354 fn emit_b_label(&mut self, label: Label) -> Result<(), CompileError>;
355 fn emit_cbz_label(&mut self, sz: Size, reg: Location, label: Label)
356 -> Result<(), CompileError>;
357 fn emit_cbnz_label(
358 &mut self,
359 sz: Size,
360 reg: Location,
361 label: Label,
362 ) -> Result<(), CompileError>;
363 fn emit_cbz_label_far(
364 &mut self,
365 sz: Size,
366 reg: Location,
367 label: Label,
368 ) -> Result<(), CompileError>;
369 fn emit_tbz_label(
370 &mut self,
371 sz: Size,
372 reg: Location,
373 n: u32,
374 label: Label,
375 ) -> Result<(), CompileError>;
376 fn emit_tbnz_label(
377 &mut self,
378 sz: Size,
379 reg: Location,
380 n: u32,
381 label: Label,
382 ) -> Result<(), CompileError>;
383 fn emit_bcond_label(&mut self, condition: Condition, label: Label) -> Result<(), CompileError>;
384 fn emit_bcond_label_far(
385 &mut self,
386 condition: Condition,
387 label: Label,
388 ) -> Result<(), CompileError>;
389 fn emit_b_register(&mut self, reg: GPR) -> Result<(), CompileError>;
390 fn emit_call_label(&mut self, label: Label) -> Result<(), CompileError>;
391 fn emit_call_register(&mut self, reg: GPR) -> Result<(), CompileError>;
392 fn emit_ret(&mut self) -> Result<(), CompileError>;
393
394 fn emit_udf(&mut self, payload: u16) -> Result<(), CompileError>;
395 fn emit_dmb(&mut self) -> Result<(), CompileError>;
396 fn emit_brk(&mut self) -> Result<(), CompileError>;
397
398 fn emit_fcmp(&mut self, sz: Size, src1: Location, src2: Location) -> Result<(), CompileError>;
399 fn emit_fneg(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
400 fn emit_fsqrt(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
401
402 fn emit_fadd(
403 &mut self,
404 sz: Size,
405 src1: Location,
406 src2: Location,
407 dst: Location,
408 ) -> Result<(), CompileError>;
409 fn emit_fsub(
410 &mut self,
411 sz: Size,
412 src1: Location,
413 src2: Location,
414 dst: Location,
415 ) -> Result<(), CompileError>;
416 fn emit_fmul(
417 &mut self,
418 sz: Size,
419 src1: Location,
420 src2: Location,
421 dst: Location,
422 ) -> Result<(), CompileError>;
423 fn emit_fdiv(
424 &mut self,
425 sz: Size,
426 src1: Location,
427 src2: Location,
428 dst: Location,
429 ) -> Result<(), CompileError>;
430
431 fn emit_fmin(
432 &mut self,
433 sz: Size,
434 src1: Location,
435 src2: Location,
436 dst: Location,
437 ) -> Result<(), CompileError>;
438 fn emit_fmax(
439 &mut self,
440 sz: Size,
441 src1: Location,
442 src2: Location,
443 dst: Location,
444 ) -> Result<(), CompileError>;
445
446 fn emit_frintz(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
447 fn emit_frintn(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
448 fn emit_frintm(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
449 fn emit_frintp(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
450
451 fn emit_scvtf(
452 &mut self,
453 sz_in: Size,
454 src: Location,
455 sz_out: Size,
456 dst: Location,
457 ) -> Result<(), CompileError>;
458 fn emit_ucvtf(
459 &mut self,
460 sz_in: Size,
461 src: Location,
462 sz_out: Size,
463 dst: Location,
464 ) -> Result<(), CompileError>;
465 fn emit_fcvt(&mut self, sz_in: Size, src: Location, dst: Location) -> Result<(), CompileError>;
466 fn emit_fcvtzs(
467 &mut self,
468 sz_in: Size,
469 src: Location,
470 sz_out: Size,
471 dst: Location,
472 ) -> Result<(), CompileError>;
473 fn emit_fcvtzu(
474 &mut self,
475 sz_in: Size,
476 src: Location,
477 sz_out: Size,
478 dst: Location,
479 ) -> Result<(), CompileError>;
480
481 fn emit_fmov(
482 &mut self,
483 src_size: Size,
484 src: Location,
485 dst_size: Size,
486 dst: Location,
487 ) -> Result<(), CompileError>;
488 fn emit_cnt(&mut self, src: NEON, dst: NEON) -> Result<(), CompileError>;
489 fn emit_addv(&mut self, src: NEON, dst: NEON) -> Result<(), CompileError>;
490 fn emit_read_fpcr(&mut self, reg: GPR) -> Result<(), CompileError>;
491 fn emit_write_fpcr(&mut self, reg: GPR) -> Result<(), CompileError>;
492 fn emit_read_fpsr(&mut self, reg: GPR) -> Result<(), CompileError>;
493 fn emit_write_fpsr(&mut self, reg: GPR) -> Result<(), CompileError>;
494
495 fn arch_supports_canonicalize_nan(&self) -> bool {
496 true
497 }
498
499 fn arch_requires_indirect_call_trampoline(&self) -> bool {
500 false
501 }
502
503 fn arch_emit_indirect_call_with_trampoline(
504 &mut self,
505 _loc: Location,
506 ) -> Result<(), CompileError> {
507 codegen_error!("singlepass arch_emit_indirect_call_with_trampoline unimplemented")
508 }
509}
510
511impl EmitterARM64 for Assembler {
512 fn get_label(&mut self) -> DynamicLabel {
513 self.new_dynamic_label()
514 }
515
516 fn get_offset(&self) -> AssemblyOffset {
517 self.offset()
518 }
519
520 fn get_jmp_instr_size(&self) -> u8 {
521 4 }
523
524 fn finalize_function(&mut self) {
525 dynasm!(
526 self
527 ; const_neg_one_32:
528 ; .i32 -1
529 ; const_zero_32:
530 ; .i32 0
531 ; const_pos_one_32:
532 ; .i32 1
533 );
534 }
535
536 fn emit_str(&mut self, sz: Size, reg: Location, addr: Location) -> Result<(), CompileError> {
537 match (sz, reg, addr) {
538 (Size::S64, Location::GPR(reg), Location::Memory(addr, disp)) => {
539 let disp = disp as u32;
540 assert!(disp.is_multiple_of(8) && (disp < 0x8000));
541 dynasm!(self ; str X(reg), [X(addr), disp]);
542 }
543 (Size::S32, Location::GPR(reg), Location::Memory(addr, disp)) => {
544 let disp = disp as u32;
545 assert!(disp.is_multiple_of(4) && (disp < 0x4000));
546 dynasm!(self ; str W(reg), [X(addr), disp]);
547 }
548 (Size::S16, Location::GPR(reg), Location::Memory(addr, disp)) => {
549 let disp = disp as u32;
550 assert!(disp.is_multiple_of(2) && (disp < 0x2000));
551 dynasm!(self ; strh W(reg), [X(addr), disp]);
552 }
553 (Size::S8, Location::GPR(reg), Location::Memory(addr, disp)) => {
554 let disp = disp as u32;
555 assert!(disp < 0x1000);
556 dynasm!(self ; strb W(reg), [X(addr), disp]);
557 }
558 (Size::S64, Location::SIMD(reg), Location::Memory(addr, disp)) => {
559 let disp = disp as u32;
560 assert!(disp.is_multiple_of(8) && (disp < 0x8000));
561 dynasm!(self ; str D(reg), [X(addr), disp]);
562 }
563 (Size::S32, Location::SIMD(reg), Location::Memory(addr, disp)) => {
564 let disp = disp as u32;
565 assert!(disp.is_multiple_of(4) && (disp < 0x4000));
566 dynasm!(self ; str S(reg), [X(addr), disp]);
567 }
568 (Size::S64, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
569 assert!(offs == 0);
570 let mult = mult as u32;
571 match mult {
572 0 => dynasm!(self ; str X(reg), [X(addr)]),
573 1 => dynasm!(self ; str X(reg), [X(addr), X(r2)]),
574 _ => dynasm!(self ; str X(reg), [X(addr), X(r2), LSL mult]),
575 };
576 }
577 (Size::S32, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
578 assert!(offs == 0);
579 let mult = mult as u32;
580 match mult {
581 0 => dynasm!(self ; str W(reg), [X(addr)]),
582 1 => dynasm!(self ; str W(reg), [X(addr), X(r2)]),
583 _ => dynasm!(self ; str W(reg), [X(addr), X(r2), LSL mult]),
584 };
585 }
586 _ => codegen_error!("singlepass can't emit STR {:?}, {:?}, {:?}", sz, reg, addr),
587 }
588 Ok(())
589 }
590 fn emit_ldr(&mut self, sz: Size, reg: Location, addr: Location) -> Result<(), CompileError> {
591 match (sz, reg, addr) {
592 (Size::S64, Location::GPR(reg), Location::Memory(addr, disp)) => {
593 assert!((disp & 0x7) == 0 && (disp < 0x8000));
594 let disp = disp as u32;
595 dynasm!(self ; ldr X(reg), [X(addr), disp]);
596 }
597 (Size::S32, Location::GPR(reg), Location::Memory(addr, disp)) => {
598 assert!((disp & 0x3) == 0 && (disp < 0x4000));
599 let disp = disp as u32;
600 dynasm!(self ; ldr W(reg), [X(addr), disp]);
601 }
602 (Size::S16, Location::GPR(reg), Location::Memory(addr, disp)) => {
603 assert!((disp & 0x1 == 0) && (disp < 0x2000));
604 let disp = disp as u32;
605 dynasm!(self ; ldrh W(reg), [X(addr), disp]);
606 }
607 (Size::S8, Location::GPR(reg), Location::Memory(addr, disp)) => {
608 assert!(disp < 0x1000);
609 let disp = disp as u32;
610 dynasm!(self ; ldrb W(reg), [X(addr), disp]);
611 }
612 (Size::S64, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
613 assert!(offs == 0);
614 let mult = mult as u32;
615 match mult {
616 0 => dynasm!(self ; ldr X(reg), [X(addr)]),
617 1 => dynasm!(self ; ldr X(reg), [X(addr), X(r2)]),
618 _ => dynasm!(self ; ldr X(reg), [X(addr), X(r2), LSL mult]),
619 };
620 }
621 (Size::S32, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
622 assert!(offs == 0);
623 let mult = mult as u32;
624 match mult {
625 0 => dynasm!(self ; ldr W(reg), [X(addr)]),
626 1 => dynasm!(self ; ldr W(reg), [X(addr), X(r2)]),
627 _ => dynasm!(self ; ldr W(reg), [X(addr), X(r2), LSL mult]),
628 };
629 }
630 (Size::S64, Location::SIMD(reg), Location::Memory(addr, disp)) => {
631 let disp = disp as u32;
632 assert!((disp & 0x7) == 0 && (disp < 0x8000));
633 dynasm!(self ; ldr D(reg), [X(addr), disp]);
634 }
635 (Size::S32, Location::SIMD(reg), Location::Memory(addr, disp)) => {
636 let disp = disp as u32;
637 assert!((disp & 0x3) == 0 && (disp < 0x4000));
638 dynasm!(self ; ldr S(reg), [X(addr), disp]);
639 }
640 (Size::S64, Location::SIMD(reg), Location::Memory2(addr, r2, mult, offs)) => {
641 assert!(offs == 0);
642 let mult = mult as u32;
643 match mult {
644 0 => dynasm!(self ; ldr D(reg), [X(addr)]),
645 1 => dynasm!(self ; ldr D(reg), [X(addr), X(r2)]),
646 _ => dynasm!(self ; ldr D(reg), [X(addr), X(r2), LSL mult]),
647 };
648 }
649 (Size::S32, Location::SIMD(reg), Location::Memory2(addr, r2, mult, offs)) => {
650 assert!(offs == 0);
651 let mult = mult as u32;
652 match mult {
653 0 => dynasm!(self ; ldr S(reg), [X(addr)]),
654 1 => dynasm!(self ; ldr S(reg), [X(addr), X(r2)]),
655 _ => dynasm!(self ; ldr S(reg), [X(addr), X(r2), LSL mult]),
656 };
657 }
658 _ => codegen_error!("singlepass can't emit LDR {:?}, {:?}, {:?}", sz, reg, addr),
659 }
660 Ok(())
661 }
662 fn emit_stur(
663 &mut self,
664 sz: Size,
665 reg: Location,
666 addr: GPR,
667 offset: i32,
668 ) -> Result<(), CompileError> {
669 assert!((-255..=255).contains(&offset));
670 match (sz, reg) {
671 (Size::S64, Location::GPR(reg)) => {
672 dynasm!(self ; stur X(reg), [X(addr), offset]);
673 }
674 (Size::S32, Location::GPR(reg)) => {
675 dynasm!(self ; stur W(reg), [X(addr), offset]);
676 }
677 (Size::S64, Location::SIMD(reg)) => {
678 dynasm!(self ; stur D(reg), [X(addr), offset]);
679 }
680 (Size::S32, Location::SIMD(reg)) => {
681 dynasm!(self ; stur S(reg), [X(addr), offset]);
682 }
683 _ => codegen_error!(
684 "singlepass can't emit STUR {:?}, {:?}, {:?}, {:?}",
685 sz,
686 reg,
687 addr,
688 offset
689 ),
690 }
691 Ok(())
692 }
693 fn emit_ldur(
694 &mut self,
695 sz: Size,
696 reg: Location,
697 addr: GPR,
698 offset: i32,
699 ) -> Result<(), CompileError> {
700 assert!((-255..=255).contains(&offset));
701 match (sz, reg) {
702 (Size::S64, Location::GPR(reg)) => {
703 dynasm!(self ; ldur X(reg), [X(addr), offset]);
704 }
705 (Size::S32, Location::GPR(reg)) => {
706 dynasm!(self ; ldur W(reg), [X(addr), offset]);
707 }
708 (Size::S64, Location::SIMD(reg)) => {
709 dynasm!(self ; ldur D(reg), [X(addr), offset]);
710 }
711 (Size::S32, Location::SIMD(reg)) => {
712 dynasm!(self ; ldur S(reg), [X(addr), offset]);
713 }
714 _ => codegen_error!(
715 "singlepass can't emit LDUR {:?}, {:?}, {:?}, {:?}",
716 sz,
717 reg,
718 addr,
719 offset
720 ),
721 }
722 Ok(())
723 }
724
725 fn emit_strdb(
726 &mut self,
727 sz: Size,
728 reg: Location,
729 addr: GPR,
730 offset: u32,
731 ) -> Result<(), CompileError> {
732 assert!(offset <= 255);
733 match (sz, reg) {
734 (Size::S64, Location::GPR(reg)) => {
735 dynasm!(self ; str X(reg), [X(addr), -(offset as i32)]!);
736 }
737 (Size::S64, Location::SIMD(reg)) => {
738 dynasm!(self ; str D(reg), [X(addr), -(offset as i32)]!);
739 }
740 _ => codegen_error!("singlepass can't emit STRDB"),
741 }
742 Ok(())
743 }
744 fn emit_stria(
745 &mut self,
746 sz: Size,
747 reg: Location,
748 addr: GPR,
749 offset: u32,
750 ) -> Result<(), CompileError> {
751 assert!(offset <= 255);
752 match (sz, reg) {
753 (Size::S64, Location::GPR(reg)) => {
754 dynasm!(self ; str X(reg), [X(addr)], offset as i32);
755 }
756 (Size::S64, Location::SIMD(reg)) => {
757 dynasm!(self ; str D(reg), [X(addr)], offset as i32);
758 }
759 _ => codegen_error!("singlepass can't emit STRIA"),
760 }
761 Ok(())
762 }
763 fn emit_ldria(
764 &mut self,
765 sz: Size,
766 reg: Location,
767 addr: GPR,
768 offset: u32,
769 ) -> Result<(), CompileError> {
770 assert!(offset <= 255);
771 match (sz, reg) {
772 (Size::S64, Location::GPR(reg)) => {
773 dynasm!(self ; ldr X(reg), [X(addr)], offset as _);
774 }
775 (Size::S64, Location::SIMD(reg)) => {
776 dynasm!(self ; ldr D(reg), [X(addr)], offset as _);
777 }
778 _ => codegen_error!("singlepass can't emit LDRIA"),
779 }
780 Ok(())
781 }
782
783 fn emit_stpdb(
784 &mut self,
785 sz: Size,
786 reg1: Location,
787 reg2: Location,
788 addr: GPR,
789 offset: u32,
790 ) -> Result<(), CompileError> {
791 assert!(offset <= 255);
792 match (sz, reg1, reg2) {
793 (Size::S64, Location::GPR(reg1), Location::GPR(reg2)) => {
794 dynasm!(self ; stp X(reg1), X(reg2), [X(addr), -(offset as i32)]!);
795 }
796 _ => codegen_error!("singlepass can't emit STPDB"),
797 }
798 Ok(())
799 }
800 fn emit_ldpia(
801 &mut self,
802 sz: Size,
803 reg1: Location,
804 reg2: Location,
805 addr: GPR,
806 offset: u32,
807 ) -> Result<(), CompileError> {
808 assert!(offset <= 255);
809 match (sz, reg1, reg2) {
810 (Size::S64, Location::GPR(reg1), Location::GPR(reg2)) => {
811 dynasm!(self ; ldp X(reg1), X(reg2), [X(addr)], offset as _);
812 }
813 _ => codegen_error!("singlepass can't emit LDPIA"),
814 }
815 Ok(())
816 }
817
818 fn emit_ldrb(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
819 match (reg, dst) {
820 (Location::GPR(reg), Location::Memory(addr, offset)) => {
821 let offset = offset as u32;
822 assert!(offset < 0x1000);
823 dynasm!(self ; ldrb W(reg), [X(addr), offset]);
824 }
825 (Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
826 assert!(offs == 0);
827 let mult = mult as u32;
828 match mult {
829 0 => dynasm!(self ; ldrb W(reg), [X(addr)]),
830 1 => dynasm!(self ; ldrb W(reg), [X(addr), X(r2)]),
831 _ => dynasm!(self ; ldrb W(reg), [X(addr), X(r2), LSL mult]),
832 };
833 }
834 _ => codegen_error!("singlepass can't emit LDRB {:?}, {:?}", reg, dst),
835 }
836 Ok(())
837 }
838 fn emit_ldrh(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
839 match (reg, dst) {
840 (Location::GPR(reg), Location::Memory(addr, offset)) => {
841 let offset = offset as u32;
842 assert!((offset & 1 == 0) && (offset < 0x2000));
843 dynasm!(self ; ldrh W(reg), [X(addr), offset]);
844 }
845 (Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
846 assert!(offs == 0);
847 let mult = mult as u32;
848 match mult {
849 0 => dynasm!(self ; ldrh W(reg), [X(addr)]),
850 1 => dynasm!(self ; ldrh W(reg), [X(addr), X(r2)]),
851 _ => dynasm!(self ; ldrh W(reg), [X(addr), X(r2), LSL mult]),
852 };
853 }
854 _ => codegen_error!("singlepass can't emit LDRH {:?}, {:?}", reg, dst),
855 }
856 Ok(())
857 }
858 fn emit_ldrsb(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
859 match (sz, reg, dst) {
860 (Size::S64, Location::GPR(reg), Location::Memory(addr, offset)) => {
861 let offset = offset as u32;
862 assert!(offset < 0x1000);
863 dynasm!(self ; ldrsb X(reg), [X(addr), offset]);
864 }
865 (Size::S32, Location::GPR(reg), Location::Memory(addr, offset)) => {
866 let offset = offset as u32;
867 assert!(offset < 0x1000);
868 dynasm!(self ; ldrsb W(reg), [X(addr), offset]);
869 }
870 (Size::S64, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
871 assert!(offs == 0);
872 let mult = mult as u32;
873 match mult {
874 0 => dynasm!(self ; ldrsb X(reg), [X(addr)]),
875 1 => dynasm!(self ; ldrsb X(reg), [X(addr), X(r2)]),
876 _ => dynasm!(self ; ldrsb X(reg), [X(addr), X(r2), LSL mult]),
877 };
878 }
879 (Size::S32, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
880 assert!(offs == 0);
881 let mult = mult as u32;
882 match mult {
883 0 => dynasm!(self ; ldrsb W(reg), [X(addr)]),
884 1 => dynasm!(self ; ldrsb W(reg), [X(addr), X(r2)]),
885 _ => dynasm!(self ; ldrsb W(reg), [X(addr), X(r2), LSL mult]),
886 };
887 }
888 _ => codegen_error!("singlepass can't emit LDRSB {:?}, {:?}, {:?}", sz, reg, dst),
889 }
890 Ok(())
891 }
892 fn emit_ldrsh(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
893 match (sz, reg, dst) {
894 (Size::S64, Location::GPR(reg), Location::Memory(addr, offset)) => {
895 let offset = offset as u32;
896 assert!((offset & 1 == 0) && (offset < 0x2000));
897 dynasm!(self ; ldrsh X(reg), [X(addr), offset]);
898 }
899 (Size::S32, Location::GPR(reg), Location::Memory(addr, offset)) => {
900 let offset = offset as u32;
901 assert!((offset & 1 == 0) && (offset < 0x2000));
902 dynasm!(self ; ldrsh W(reg), [X(addr), offset]);
903 }
904 (Size::S64, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
905 assert!(offs == 0);
906 let mult = mult as u32;
907 match mult {
908 0 => dynasm!(self ; ldrsh X(reg), [X(addr)]),
909 1 => dynasm!(self ; ldrsh X(reg), [X(addr), X(r2)]),
910 _ => dynasm!(self ; ldrsh X(reg), [X(addr), X(r2), LSL mult]),
911 };
912 }
913 (Size::S32, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
914 assert!(offs == 0);
915 let mult = mult as u32;
916 match mult {
917 0 => dynasm!(self ; ldrsh W(reg), [X(addr)]),
918 1 => dynasm!(self ; ldrsh W(reg), [X(addr), X(r2)]),
919 _ => dynasm!(self ; ldrsh W(reg), [X(addr), X(r2), LSL mult]),
920 };
921 }
922 _ => codegen_error!("singlepass can't emit LDRSH {:?}, {:?}, {:?}", sz, reg, dst),
923 }
924 Ok(())
925 }
926 fn emit_ldrsw(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
927 match (sz, reg, dst) {
928 (Size::S64, Location::GPR(reg), Location::Memory(addr, offset)) => {
929 let offset = offset as u32;
930 assert!((offset & 3 == 0) && (offset < 0x4000));
931 dynasm!(self ; ldrsw X(reg), [X(addr), offset]);
932 }
933 (Size::S64, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
934 assert!(offs == 0);
935 let mult = mult as u32;
936 match mult {
937 0 => dynasm!(self ; ldrsw X(reg), [X(addr)]),
938 1 => dynasm!(self ; ldrsw X(reg), [X(addr), X(r2)]),
939 _ => dynasm!(self ; ldrsw X(reg), [X(addr), X(r2), LSL mult]),
940 };
941 }
942 _ => codegen_error!("singlepass can't emit LDRSW {:?}, {:?}, {:?}", sz, reg, dst),
943 }
944 Ok(())
945 }
946 fn emit_strb(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
947 match (reg, dst) {
948 (Location::GPR(reg), Location::Memory(addr, offset)) => {
949 let offset = offset as u32;
950 assert!(offset < 0x1000);
951 dynasm!(self ; strb W(reg), [X(addr), offset]);
952 }
953 (Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
954 assert!(offs == 0);
955 let mult = mult as u32;
956 match mult {
957 0 => dynasm!(self ; strb W(reg), [X(addr)]),
958 1 => dynasm!(self ; strb W(reg), [X(addr), X(r2)]),
959 _ => dynasm!(self ; strb W(reg), [X(addr), X(r2), LSL mult]),
960 };
961 }
962 _ => codegen_error!("singlepass can't emit STRB {:?}, {:?}", reg, dst),
963 }
964 Ok(())
965 }
966 fn emit_strh(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
967 match (reg, dst) {
968 (Location::GPR(reg), Location::Memory(addr, offset)) => {
969 let offset = offset as u32;
970 assert!((offset & 1 == 0) && (offset < 0x2000));
971 dynasm!(self ; strh W(reg), [X(addr), offset]);
972 }
973 (Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
974 assert!(offs == 0);
975 let mult = mult as u32;
976 match mult {
977 0 => dynasm!(self ; strh W(reg), [X(addr)]),
978 1 => dynasm!(self ; strh W(reg), [X(addr), X(r2)]),
979 _ => dynasm!(self ; strh W(reg), [X(addr), X(r2), LSL mult]),
980 };
981 }
982 _ => codegen_error!("singlepass can't emit STRH {:?}, {:?}", reg, dst),
983 }
984 Ok(())
985 }
986
987 fn emit_ldaxr(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
988 match (sz, reg, dst) {
989 (Size::S32, Location::GPR(reg), Location::GPR(dst)) => {
990 dynasm!(self ; ldaxr W(reg), [X(dst)]);
991 }
992 (Size::S64, Location::GPR(reg), Location::GPR(dst)) => {
993 dynasm!(self ; ldaxr X(reg), [X(dst)]);
994 }
995 _ => codegen_error!("singlepass can't emit LDAXR {:?}, {:?}", reg, dst),
996 }
997 Ok(())
998 }
999 fn emit_ldaxrb(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
1000 match (reg, dst) {
1001 (Location::GPR(reg), Location::GPR(dst)) => {
1002 dynasm!(self ; ldaxrb W(reg), [X(dst)]);
1003 }
1004 _ => codegen_error!("singlepass can't emit LDAXRB {:?}, {:?}", reg, dst),
1005 }
1006 Ok(())
1007 }
1008 fn emit_ldaxrh(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
1009 match (reg, dst) {
1010 (Location::GPR(reg), Location::GPR(dst)) => {
1011 dynasm!(self ; ldaxrh W(reg), [X(dst)]);
1012 }
1013 _ => codegen_error!("singlepass can't emit LDAXRH {:?}, {:?}", reg, dst),
1014 }
1015 Ok(())
1016 }
1017 fn emit_stlxr(
1018 &mut self,
1019 sz: Size,
1020 status: Location,
1021 reg: Location,
1022 dst: Location,
1023 ) -> Result<(), CompileError> {
1024 match (sz, status, reg, dst) {
1025 (Size::S32, Location::GPR(status), Location::GPR(reg), Location::GPR(dst)) => {
1026 dynasm!(self ; stlxr W(status), W(reg), [X(dst)]);
1027 }
1028 (Size::S64, Location::GPR(status), Location::GPR(reg), Location::GPR(dst)) => {
1029 dynasm!(self ; stlxr W(status), X(reg), [X(dst)]);
1030 }
1031 _ => codegen_error!("singlepass can't emit STLXR {:?}, {:?}", reg, dst),
1032 }
1033 Ok(())
1034 }
1035 fn emit_stlxrb(
1036 &mut self,
1037 _sz: Size,
1038 status: Location,
1039 reg: Location,
1040 dst: Location,
1041 ) -> Result<(), CompileError> {
1042 match (status, reg, dst) {
1043 (Location::GPR(status), Location::GPR(reg), Location::GPR(dst)) => {
1044 dynasm!(self ; stlxrb W(status), W(reg), [X(dst)]);
1045 }
1046 _ => codegen_error!("singlepass can't emit STLXRB {:?}, {:?}", reg, dst),
1047 }
1048 Ok(())
1049 }
1050 fn emit_stlxrh(
1051 &mut self,
1052 _sz: Size,
1053 status: Location,
1054 reg: Location,
1055 dst: Location,
1056 ) -> Result<(), CompileError> {
1057 match (status, reg, dst) {
1058 (Location::GPR(status), Location::GPR(reg), Location::GPR(dst)) => {
1059 dynasm!(self ; stlxrh W(status), W(reg), [X(dst)]);
1060 }
1061 _ => codegen_error!("singlepass can't emit STLXRH {:?}, {:?}", reg, dst),
1062 }
1063 Ok(())
1064 }
1065
1066 fn emit_mov(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1067 match (sz, src, dst) {
1068 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1069 dynasm!(self ; mov X(dst), X(src));
1070 }
1071 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1072 dynasm!(self ; mov W(dst), W(src));
1073 }
1074 (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
1075 dynasm!(self ; mov V(dst).D[0], V(src).D[0]);
1076 }
1077 (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
1078 dynasm!(self ; mov V(dst).S[0], V(src).S[0]);
1079 }
1080 (Size::S64, Location::GPR(src), Location::SIMD(dst)) => {
1081 dynasm!(self ; mov V(dst).D[0], X(src));
1082 }
1083 (Size::S32, Location::GPR(src), Location::SIMD(dst)) => {
1084 dynasm!(self ; mov V(dst).S[0], W(src));
1085 }
1086 (Size::S64, Location::SIMD(src), Location::GPR(dst)) => {
1087 dynasm!(self ; mov X(dst), V(src).D[0]);
1088 }
1089 (Size::S32, Location::SIMD(src), Location::GPR(dst)) => {
1090 dynasm!(self ; mov W(dst), V(src).S[0]);
1091 }
1092 (Size::S32, Location::Imm32(val), Location::GPR(dst)) => {
1093 if val < 0x1000 || encode_logical_immediate_32bit(val as _).is_some() {
1094 dynasm!(self ; mov W(dst), val);
1095 } else {
1096 dynasm!(self
1097 ; movz W(dst), val
1098 ; movk W(dst), val, lsl #16);
1099 }
1100 }
1101 (Size::S64, Location::Imm32(val), Location::GPR(dst)) => {
1102 if val < 0x1000 {
1103 dynasm!(self ; mov W(dst), val);
1104 } else if encode_logical_immediate_64bit(val as _).is_some() {
1105 dynasm!(self ; mov X(dst), val as _);
1106 } else {
1107 dynasm!(self
1108 ; movz X(dst), val
1109 ; movk X(dst), val, lsl #16);
1110 }
1111 }
1112 (Size::S64, Location::Imm64(val), Location::GPR(dst)) => {
1113 if val < 0x1000 || encode_logical_immediate_64bit(val as _).is_some() {
1114 dynasm!(self ; mov X(dst), val as _);
1115 } else {
1116 dynasm!(self
1117 ; movz X(dst), val as _
1118 ; movk X(dst), val as _, lsl #16
1119 ; movk X(dst), val as _, lsl #32
1120 ; movk X(dst), val as _, lsl #48);
1121 }
1122 }
1123 _ => codegen_error!("singlepass can't emit MOV {:?}, {:?}, {:?}", sz, src, dst),
1124 }
1125 Ok(())
1126 }
1127
1128 fn emit_movn(&mut self, sz: Size, reg: Location, val: u32) -> Result<(), CompileError> {
1129 match (sz, reg) {
1130 (Size::S32, Location::GPR(reg)) => {
1131 dynasm!(self ; movn W(reg), val);
1132 }
1133 (Size::S64, Location::GPR(reg)) => {
1134 dynasm!(self ; movn X(reg), val);
1135 }
1136 _ => codegen_error!("singlepass can't emit MOVN"),
1137 }
1138 Ok(())
1139 }
1140 fn emit_movz(&mut self, reg: Location, val: u32) -> Result<(), CompileError> {
1141 match reg {
1142 Location::GPR(reg) => {
1143 dynasm!(self ; movz W(reg), val);
1144 }
1145 _ => codegen_error!("singlepass can't emit MOVZ"),
1146 }
1147 Ok(())
1148 }
1149 fn emit_movk(&mut self, reg: Location, val: u32, shift: u32) -> Result<(), CompileError> {
1150 match reg {
1151 Location::GPR(reg) => {
1152 dynasm!(self ; movk X(reg), val, LSL shift);
1153 }
1154 _ => codegen_error!("singlepass can't emit MOVK"),
1155 }
1156 Ok(())
1157 }
1158
1159 fn emit_mov_imm(&mut self, dst: Location, val: u64) -> Result<(), CompileError> {
1160 match dst {
1161 Location::GPR(dst) => {
1162 let offset = val.trailing_zeros() & 48;
1163 let masked = 0xffff & (val >> offset);
1164 if (masked << offset) == val {
1165 dynasm!(self ; movz X(dst), masked as u32, LSL offset);
1166 } else if val >> 16 == 0xffff_ffff_ffff {
1167 let val: u16 = !((val & 0xffff) as u16);
1168 dynasm!(self ; movn X(dst), val as u32);
1169 } else if val >> 16 == 0xffff {
1170 let val: u16 = !((val & 0xffff) as u16);
1171 dynasm!(self ; movn W(dst), val as u32);
1172 } else {
1173 dynasm!(self ; movz W(dst), (val&0xffff) as u32);
1174 let val = val >> 16;
1175 if val != 0 {
1176 dynasm!(self ; movk X(dst), (val&0xffff) as u32, LSL 16);
1177 let val = val >> 16;
1178 if val != 0 {
1179 dynasm!(self ; movk X(dst), (val&0xffff) as u32, LSL 32);
1180 let val = val >> 16;
1181 if val != 0 {
1182 dynasm!(self ; movk X(dst), (val&0xffff) as u32, LSL 48);
1183 }
1184 }
1185 }
1186 }
1187 }
1188 _ => codegen_error!("singlepass can't emit MOVW {:?}", dst),
1189 }
1190 Ok(())
1191 }
1192
1193 fn emit_add(
1194 &mut self,
1195 sz: Size,
1196 src1: Location,
1197 src2: Location,
1198 dst: Location,
1199 ) -> Result<(), CompileError> {
1200 match (sz, src1, src2, dst) {
1201 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1202 dynasm!(self ; add XSP(dst), XSP(src1), X(src2), UXTX);
1203 }
1204 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1205 dynasm!(self ; add WSP(dst), WSP(src1), W(src2), UXTX);
1206 }
1207 (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1208 | (Size::S64, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1209 dynasm!(self ; add XSP(dst), XSP(src1), imm as _);
1210 }
1211 (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
1212 | (Size::S64, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
1213 if imm >= 0x1000 {
1214 unreachable!();
1215 }
1216 dynasm!(self ; add XSP(dst), XSP(src1), imm);
1217 }
1218 (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst))
1219 | (Size::S64, Location::Imm64(imm), Location::GPR(src1), Location::GPR(dst)) => {
1220 if imm >= 0x1000 {
1221 unreachable!();
1222 }
1223 let imm = imm as u32;
1224 dynasm!(self ; add XSP(dst), XSP(src1), imm);
1225 }
1226 (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1227 | (Size::S32, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1228 dynasm!(self ; add WSP(dst), WSP(src1), imm as u32);
1229 }
1230 (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
1231 | (Size::S32, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
1232 if imm >= 0x1000 {
1233 unreachable!();
1234 }
1235 dynasm!(self ; add WSP(dst), WSP(src1), imm);
1236 }
1237 _ => codegen_error!(
1238 "singlepass can't emit ADD {:?} {:?} {:?} {:?}",
1239 sz,
1240 src1,
1241 src2,
1242 dst
1243 ),
1244 }
1245 Ok(())
1246 }
1247 fn emit_sub(
1248 &mut self,
1249 sz: Size,
1250 src1: Location,
1251 src2: Location,
1252 dst: Location,
1253 ) -> Result<(), CompileError> {
1254 match (sz, src1, src2, dst) {
1255 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1256 dynasm!(self; sub XSP(dst), XSP(src1), X(src2), UXTX 0);
1257 }
1258 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1259 dynasm!(self ; sub WSP(dst), WSP(src1), W(src2), UXTX 0);
1260 }
1261 (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1262 dynasm!(self ; sub XSP(dst), XSP(src1), imm as u32);
1263 }
1264 (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1265 dynasm!(self ; sub WSP(dst), WSP(src1), imm as u32);
1266 }
1267 (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1268 if imm >= 0x1000 {
1269 unreachable!();
1270 }
1271 dynasm!(self ; sub WSP(dst), WSP(src1), imm);
1272 }
1273 (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1274 if imm >= 0x1000 {
1275 unreachable!();
1276 }
1277 dynasm!(self ; sub XSP(dst), XSP(src1), imm);
1278 }
1279 (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
1280 if imm >= 0x1000 {
1281 unreachable!();
1282 }
1283 dynasm!(self ; sub XSP(dst), XSP(src1), imm as u32);
1284 }
1285 _ => codegen_error!(
1286 "singlepass can't emit SUB {:?} {:?} {:?} {:?}",
1287 sz,
1288 src1,
1289 src2,
1290 dst
1291 ),
1292 }
1293 Ok(())
1294 }
1295 fn emit_mul(
1296 &mut self,
1297 sz: Size,
1298 src1: Location,
1299 src2: Location,
1300 dst: Location,
1301 ) -> Result<(), CompileError> {
1302 match (sz, src1, src2, dst) {
1303 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1304 dynasm!(self ; mul X(dst), X(src1), X(src2));
1305 }
1306 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1307 dynasm!(self ; mul W(dst), W(src1), W(src2));
1308 }
1309 _ => codegen_error!(
1310 "singlepass can't emit MUL {:?} {:?} {:?} {:?}",
1311 sz,
1312 src1,
1313 src2,
1314 dst
1315 ),
1316 }
1317 Ok(())
1318 }
1319 fn emit_adds(
1320 &mut self,
1321 sz: Size,
1322 src1: Location,
1323 src2: Location,
1324 dst: Location,
1325 ) -> Result<(), CompileError> {
1326 match (sz, src1, src2, dst) {
1327 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1328 dynasm!(self ; adds X(dst), X(src1), X(src2));
1329 }
1330 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1331 dynasm!(self ; adds W(dst), W(src1), W(src2));
1332 }
1333 (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1334 | (Size::S64, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1335 dynasm!(self ; adds X(dst), XSP(src1), imm as u32);
1336 }
1337 (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
1338 | (Size::S64, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
1339 if imm >= 0x1000 {
1340 codegen_error!("singlepass ADD.S with imm too large {}", imm);
1341 }
1342 dynasm!(self ; adds X(dst), XSP(src1), imm);
1343 }
1344 (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1345 | (Size::S32, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1346 dynasm!(self ; adds W(dst), WSP(src1), imm as u32);
1347 }
1348 (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
1349 | (Size::S32, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
1350 if imm >= 0x1000 {
1351 codegen_error!("singlepass ADD.S with imm too large {}", imm);
1352 }
1353 dynasm!(self ; adds W(dst), WSP(src1), imm);
1354 }
1355 _ => codegen_error!(
1356 "singlepass can't emit ADD.S {:?} {:?} {:?} {:?}",
1357 sz,
1358 src1,
1359 src2,
1360 dst
1361 ),
1362 }
1363 Ok(())
1364 }
1365 fn emit_subs(
1366 &mut self,
1367 sz: Size,
1368 src1: Location,
1369 src2: Location,
1370 dst: Location,
1371 ) -> Result<(), CompileError> {
1372 match (sz, src1, src2, dst) {
1373 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1374 dynasm!(self ; subs X(dst), X(src1), X(src2));
1375 }
1376 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1377 dynasm!(self ; subs W(dst), W(src1), W(src2));
1378 }
1379 (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1380 dynasm!(self ; subs X(dst), XSP(src1), imm as u32);
1381 }
1382 (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1383 dynasm!(self ; subs W(dst), WSP(src1), imm as u32);
1384 }
1385 _ => codegen_error!(
1386 "singlepass can't emit SUB.S {:?} {:?} {:?} {:?}",
1387 sz,
1388 src1,
1389 src2,
1390 dst
1391 ),
1392 }
1393 Ok(())
1394 }
1395 fn emit_add_lsl(
1396 &mut self,
1397 sz: Size,
1398 src1: Location,
1399 src2: Location,
1400 lsl: u32,
1401 dst: Location,
1402 ) -> Result<(), CompileError> {
1403 match (sz, src1, src2, dst) {
1404 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1405 dynasm!(self ; add X(dst), X(src1), X(src2), LSL lsl);
1406 }
1407 _ => codegen_error!(
1408 "singlepass can't emit LSL {:?} {:?} {:?} {:?} LSL {:?}",
1409 sz,
1410 src1,
1411 src2,
1412 dst,
1413 lsl
1414 ),
1415 }
1416 Ok(())
1417 }
1418
1419 fn emit_cmp(&mut self, sz: Size, left: Location, right: Location) -> Result<(), CompileError> {
1425 match (sz, left, right) {
1426 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1427 dynasm!(self ; cmp X(dst), X(src));
1428 }
1429 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1430 dynasm!(self ; cmp W(dst), W(src));
1431 }
1432 (Size::S64, Location::Imm8(imm), Location::GPR(dst)) => {
1433 dynasm!(self ; cmp XSP(dst), imm as u32);
1434 }
1435 (Size::S64, Location::Imm32(imm), Location::GPR(dst)) => {
1436 if imm >= 0x1000 {
1437 codegen_error!("singlepass CMP with imm too large {}", imm);
1438 }
1439 dynasm!(self ; cmp XSP(dst), imm as u32);
1440 }
1441 (Size::S64, Location::Imm64(imm), Location::GPR(dst)) => {
1442 if imm >= 0x1000 {
1443 codegen_error!("singlepass CMP with imm too large {}", imm);
1444 }
1445 dynasm!(self ; cmp XSP(dst), imm as u32);
1446 }
1447 (Size::S32, Location::Imm8(imm), Location::GPR(dst)) => {
1448 dynasm!(self ; cmp WSP(dst), imm as u32);
1449 }
1450 (Size::S32, Location::Imm32(imm), Location::GPR(dst)) => {
1451 if imm >= 0x1000 {
1452 codegen_error!("singlepass CMP with imm too large {}", imm);
1453 }
1454 dynasm!(self ; cmp WSP(dst), imm as u32);
1455 }
1456 _ => codegen_error!("singlepass can't emit CMP {:?} {:?} {:?}", sz, left, right),
1457 }
1458 Ok(())
1459 }
1460
1461 fn emit_tst(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1462 match (sz, src, dst) {
1463 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1464 dynasm!(self ; tst X(dst), X(src));
1465 }
1466 (Size::S64, Location::Imm32(imm), Location::GPR(dst)) => {
1467 if encode_logical_immediate_64bit(imm as u64).is_none() {
1468 codegen_error!("singlepass TST with incompatible imm {}", imm);
1469 }
1470 dynasm!(self ; tst X(dst), imm as u64);
1471 }
1472 (Size::S64, Location::Imm64(imm), Location::GPR(dst)) => {
1473 if encode_logical_immediate_64bit(imm as u64).is_none() {
1474 codegen_error!("singlepass TST with incompatible imm {}", imm);
1475 }
1476 dynasm!(self ; tst X(dst), imm as u64);
1477 }
1478 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1479 dynasm!(self ; tst W(dst), W(src));
1480 }
1481 (Size::S32, Location::Imm32(imm), Location::GPR(dst)) => {
1482 if encode_logical_immediate_64bit(imm as u64).is_none() {
1483 codegen_error!("singlepass TST with incompatible imm {}", imm);
1484 }
1485 dynasm!(self ; tst W(dst), imm);
1486 }
1487 _ => codegen_error!("singlepass can't emit TST"),
1488 }
1489 Ok(())
1490 }
1491
1492 fn emit_lsl(
1493 &mut self,
1494 sz: Size,
1495 src1: Location,
1496 src2: Location,
1497 dst: Location,
1498 ) -> Result<(), CompileError> {
1499 match (sz, src1, src2, dst) {
1500 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1501 dynasm!(self ; lsl X(dst), X(src1), X(src2));
1502 }
1503 (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1504 if imm > 63 {
1505 codegen_error!("singlepass LSL with incompatible imm {}", imm);
1506 }
1507 let imm = imm as u32;
1508
1509 dynasm!(self ; lsl X(dst), X(src1), imm);
1510 }
1511 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1512 dynasm!(self ; lsl W(dst), W(src1), W(src2));
1513 }
1514 (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1515 if imm > 63 {
1516 codegen_error!("singlepass LSL with incompatible imm {}", imm);
1517 }
1518 dynasm!(self ; lsl X(dst), X(src1), imm as u32);
1519 }
1520 (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
1521 if imm > 63 {
1522 codegen_error!("singlepass LSL with incompatible imm {}", imm);
1523 }
1524 dynasm!(self ; lsl X(dst), X(src1), imm as u32);
1525 }
1526 (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1527 if imm > 31 {
1528 codegen_error!("singlepass LSL with incompatible imm {}", imm);
1529 }
1530 dynasm!(self ; lsl W(dst), W(src1), imm as u32);
1531 }
1532 (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1533 if imm > 31 {
1534 codegen_error!("singlepass LSL with incompatible imm {}", imm);
1535 }
1536 dynasm!(self ; lsl W(dst), W(src1), imm as u32);
1537 }
1538 _ => codegen_error!(
1539 "singlepass can't emit LSL {:?} {:?} {:?} {:?}",
1540 sz,
1541 src1,
1542 src2,
1543 dst
1544 ),
1545 }
1546 Ok(())
1547 }
1548 fn emit_asr(
1549 &mut self,
1550 sz: Size,
1551 src1: Location,
1552 src2: Location,
1553 dst: Location,
1554 ) -> Result<(), CompileError> {
1555 match (sz, src1, src2, dst) {
1556 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1557 dynasm!(self ; asr X(dst), X(src1), X(src2));
1558 }
1559 (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1560 let imm = imm as u32;
1561
1562 if imm == 0 || imm > 63 {
1563 codegen_error!("singlepass ASR with incompatible imm {}", imm);
1564 }
1565 dynasm!(self ; asr X(dst), X(src1), imm);
1566 }
1567 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1568 dynasm!(self ; asr W(dst), W(src1), W(src2));
1569 }
1570 (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1571 if imm == 0 || imm > 63 {
1572 codegen_error!("singlepass ASR with incompatible imm {}", imm);
1573 }
1574 dynasm!(self ; asr X(dst), X(src1), imm as u32);
1575 }
1576 (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
1577 if imm == 0 || imm > 63 {
1578 codegen_error!("singlepass ASR with incompatible imm {}", imm);
1579 }
1580 dynasm!(self ; asr X(dst), X(src1), imm as u32);
1581 }
1582 (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1583 if imm == 0 || imm > 31 {
1584 codegen_error!("singlepass ASR with incompatible imm {}", imm);
1585 }
1586 dynasm!(self ; asr W(dst), W(src1), imm as u32);
1587 }
1588 (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1589 if imm == 0 || imm > 31 {
1590 codegen_error!("singlepass ASR with incompatible imm {}", imm);
1591 }
1592 dynasm!(self ; asr W(dst), W(src1), imm as u32);
1593 }
1594 _ => codegen_error!(
1595 "singlepass can't emit ASR {:?} {:?} {:?} {:?}",
1596 sz,
1597 src1,
1598 src2,
1599 dst
1600 ),
1601 }
1602 Ok(())
1603 }
1604 fn emit_lsr(
1605 &mut self,
1606 sz: Size,
1607 src1: Location,
1608 src2: Location,
1609 dst: Location,
1610 ) -> Result<(), CompileError> {
1611 match (sz, src1, src2, dst) {
1612 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1613 dynasm!(self ; lsr X(dst), X(src1), X(src2));
1614 }
1615 (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1616 let imm = imm as u32;
1617
1618 if imm == 0 || imm > 63 {
1619 codegen_error!("singlepass LSR with incompatible imm {}", imm);
1620 }
1621 dynasm!(self ; lsr X(dst), X(src1), imm);
1622 }
1623 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1624 dynasm!(self ; lsr W(dst), W(src1), W(src2));
1625 }
1626 (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1627 if imm == 0 || imm > 63 {
1628 codegen_error!("singlepass LSR with incompatible imm {}", imm);
1629 }
1630 dynasm!(self ; lsr X(dst), X(src1), imm as u32);
1631 }
1632 (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
1633 if imm == 0 || imm > 63 {
1634 codegen_error!("singlepass LSR with incompatible imm {}", imm);
1635 }
1636 dynasm!(self ; lsr X(dst), X(src1), imm as u32);
1637 }
1638 (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1639 if imm == 0 || imm > 31 {
1640 codegen_error!("singlepass LSR with incompatible imm {}", imm);
1641 }
1642 dynasm!(self ; lsr W(dst), W(src1), imm as u32);
1643 }
1644 (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1645 if imm == 0 || imm > 31 {
1646 codegen_error!("singlepass LSR with incompatible imm {}", imm);
1647 }
1648 dynasm!(self ; lsr W(dst), W(src1), imm as u32);
1649 }
1650 _ => codegen_error!(
1651 "singlepass can't emit LSR {:?} {:?} {:?} {:?}",
1652 sz,
1653 src1,
1654 src2,
1655 dst
1656 ),
1657 }
1658 Ok(())
1659 }
1660 fn emit_ror(
1661 &mut self,
1662 sz: Size,
1663 src1: Location,
1664 src2: Location,
1665 dst: Location,
1666 ) -> Result<(), CompileError> {
1667 match (sz, src1, src2, dst) {
1668 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1669 dynasm!(self ; ror X(dst), X(src1), X(src2));
1670 }
1671 (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1672 let imm = imm as u32;
1673
1674 if imm == 0 || imm > 63 {
1675 codegen_error!("singlepass ROR with incompatible imm {}", imm);
1676 }
1677 dynasm!(self ; ror X(dst), X(src1), imm);
1678 }
1679 (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
1680 let imm = imm as u32;
1681
1682 if imm == 0 || imm > 63 {
1683 codegen_error!("singlepass ROR with incompatible imm {}", imm);
1684 }
1685 dynasm!(self ; ror X(dst), X(src1), imm);
1686 }
1687 (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1688 if imm == 0 || imm > 31 {
1689 codegen_error!("singlepass ROR with incompatible imm {}", imm);
1690 }
1691 dynasm!(self ; ror W(dst), W(src1), imm as u32);
1692 }
1693 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1694 dynasm!(self ; ror W(dst), W(src1), W(src2));
1695 }
1696 (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1697 if imm == 0 || imm > 63 {
1698 codegen_error!("singlepass ROR with incompatible imm {}", imm);
1699 }
1700 dynasm!(self ; ror X(dst), X(src1), imm as u32);
1701 }
1702 (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1703 if imm == 0 || imm > 31 {
1704 codegen_error!("singlepass ROR with incompatible imm {}", imm);
1705 }
1706 dynasm!(self ; ror W(dst), W(src1), imm as u32);
1707 }
1708 _ => codegen_error!(
1709 "singlepass can't emit ROR {:?} {:?} {:?} {:?}",
1710 sz,
1711 src1,
1712 src2,
1713 dst
1714 ),
1715 }
1716 Ok(())
1717 }
1718
1719 fn emit_or(
1720 &mut self,
1721 sz: Size,
1722 src1: Location,
1723 src2: Location,
1724 dst: Location,
1725 ) -> Result<(), CompileError> {
1726 match (sz, src1, src2, dst) {
1727 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1728 dynasm!(self ; orr X(dst), X(src1), X(src2));
1729 }
1730 (Size::S64, Location::GPR(src1), Location::Imm64(src2), Location::GPR(dst)) => {
1731 let src2 = src2 as u64;
1732
1733 if encode_logical_immediate_64bit(src2 as u64).is_none() {
1734 codegen_error!("singlepass OR with incompatible imm {}", src2);
1735 }
1736 dynasm!(self ; orr XSP(dst), X(src1), src2);
1737 }
1738 (Size::S32, Location::GPR(src1), Location::Imm32(src2), Location::GPR(dst)) => {
1739 let src2 = src2 as u32;
1740
1741 if encode_logical_immediate_32bit(src2).is_none() {
1742 codegen_error!("singlepass OR with incompatible imm {}", src2);
1743 }
1744 dynasm!(self ; orr WSP(dst), W(src1), src2);
1745 }
1746 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1747 dynasm!(self ; orr W(dst), W(src1), W(src2));
1748 }
1749 _ => codegen_error!(
1750 "singlepass can't emit OR {:?} {:?} {:?} {:?}",
1751 sz,
1752 src1,
1753 src2,
1754 dst
1755 ),
1756 }
1757 Ok(())
1758 }
1759 fn emit_and(
1760 &mut self,
1761 sz: Size,
1762 src1: Location,
1763 src2: Location,
1764 dst: Location,
1765 ) -> Result<(), CompileError> {
1766 match (sz, src1, src2, dst) {
1767 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1768 dynasm!(self ; and X(dst), X(src1), X(src2));
1769 }
1770 (Size::S64, Location::GPR(src1), Location::Imm64(src2), Location::GPR(dst)) => {
1771 let src2 = src2 as u64;
1772
1773 if encode_logical_immediate_64bit(src2 as u64).is_none() {
1774 codegen_error!("singlepass AND with incompatible imm {}", src2);
1775 }
1776 dynasm!(self ; and XSP(dst), X(src1), src2);
1777 }
1778 (Size::S32, Location::GPR(src1), Location::Imm32(src2), Location::GPR(dst)) => {
1779 let src2 = src2 as u32;
1780
1781 if encode_logical_immediate_32bit(src2).is_none() {
1782 codegen_error!("singlepass AND with incompatible imm {}", src2);
1783 }
1784 dynasm!(self ; and WSP(dst), W(src1), src2);
1785 }
1786 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1787 dynasm!(self ; and W(dst), W(src1), W(src2));
1788 }
1789 _ => codegen_error!(
1790 "singlepass can't emit AND {:?} {:?} {:?} {:?}",
1791 sz,
1792 src1,
1793 src2,
1794 dst
1795 ),
1796 }
1797 Ok(())
1798 }
1799 fn emit_eor(
1800 &mut self,
1801 sz: Size,
1802 src1: Location,
1803 src2: Location,
1804 dst: Location,
1805 ) -> Result<(), CompileError> {
1806 match (sz, src1, src2, dst) {
1807 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1808 dynasm!(self ; eor X(dst), X(src1), X(src2));
1809 }
1810 (Size::S64, Location::GPR(src1), Location::Imm64(src2), Location::GPR(dst)) => {
1811 let src2 = src2 as u64;
1812
1813 if encode_logical_immediate_64bit(src2 as u64).is_none() {
1814 codegen_error!("singlepass EOR with incompatible imm {}", src2);
1815 }
1816 dynasm!(self ; eor XSP(dst), X(src1), src2);
1817 }
1818 (Size::S32, Location::GPR(src1), Location::Imm32(src2), Location::GPR(dst)) => {
1819 let src2 = src2 as u32;
1820
1821 if encode_logical_immediate_32bit(src2).is_none() {
1822 codegen_error!("singlepass EOR with incompatible imm {}", src2);
1823 }
1824 dynasm!(self ; eor WSP(dst), W(src1), src2);
1825 }
1826 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1827 dynasm!(self ; eor W(dst), W(src1), W(src2));
1828 }
1829 _ => codegen_error!(
1830 "singlepass can't emit EOR {:?} {:?} {:?} {:?}",
1831 sz,
1832 src1,
1833 src2,
1834 dst
1835 ),
1836 }
1837 Ok(())
1838 }
1839
1840 fn emit_bfc(
1841 &mut self,
1842 sz: Size,
1843 lsb: u32,
1844 width: u32,
1845 dst: Location,
1846 ) -> Result<(), CompileError> {
1847 match (sz, dst) {
1848 (Size::S32, Location::GPR(dst)) => {
1849 dynasm!(self ; bfc W(dst), lsb, width);
1850 }
1851 (Size::S64, Location::GPR(dst)) => {
1852 dynasm!(self ; bfc X(dst), lsb, width);
1853 }
1854 _ => codegen_error!("singlepass can't emit BFC"),
1855 }
1856 Ok(())
1857 }
1858 fn emit_bfi(
1859 &mut self,
1860 sz: Size,
1861 src: Location,
1862 lsb: u32,
1863 width: u32,
1864 dst: Location,
1865 ) -> Result<(), CompileError> {
1866 match (sz, src, dst) {
1867 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1868 dynasm!(self ; bfi W(dst), W(src), lsb, width);
1869 }
1870 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1871 dynasm!(self ; bfi X(dst), X(src), lsb, width);
1872 }
1873 _ => codegen_error!("singlepass can't emit BFI"),
1874 }
1875 Ok(())
1876 }
1877
1878 fn emit_udiv(
1879 &mut self,
1880 sz: Size,
1881 src1: Location,
1882 src2: Location,
1883 dst: Location,
1884 ) -> Result<(), CompileError> {
1885 match (sz, src1, src2, dst) {
1886 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1887 dynasm!(self ; udiv W(dst), W(src1), W(src2));
1888 }
1889 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1890 dynasm!(self ; udiv X(dst), X(src1), X(src2));
1891 }
1892 _ => codegen_error!(
1893 "singlepass can't emit UDIV {:?} {:?} {:?} {:?}",
1894 sz,
1895 src1,
1896 src2,
1897 dst
1898 ),
1899 }
1900 Ok(())
1901 }
1902 fn emit_sdiv(
1903 &mut self,
1904 sz: Size,
1905 src1: Location,
1906 src2: Location,
1907 dst: Location,
1908 ) -> Result<(), CompileError> {
1909 match (sz, src1, src2, dst) {
1910 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1911 dynasm!(self ; sdiv W(dst), W(src1), W(src2));
1912 }
1913 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1914 dynasm!(self ; sdiv X(dst), X(src1), X(src2));
1915 }
1916 _ => codegen_error!(
1917 "singlepass can't emit SDIV {:?} {:?} {:?} {:?}",
1918 sz,
1919 src1,
1920 src2,
1921 dst
1922 ),
1923 }
1924 Ok(())
1925 }
1926
1927 fn emit_msub(
1929 &mut self,
1930 sz: Size,
1931 a: Location,
1932 b: Location,
1933 c: Location,
1934 dst: Location,
1935 ) -> Result<(), CompileError> {
1936 match (sz, a, b, c, dst) {
1937 (
1938 Size::S32,
1939 Location::GPR(a),
1940 Location::GPR(b),
1941 Location::GPR(c),
1942 Location::GPR(dst),
1943 ) => {
1944 dynasm!(self ; msub W(dst), W(a), W(b), W(c));
1945 }
1946 (
1947 Size::S64,
1948 Location::GPR(a),
1949 Location::GPR(b),
1950 Location::GPR(c),
1951 Location::GPR(dst),
1952 ) => {
1953 dynasm!(self ; msub X(dst), X(a), X(b), X(c));
1954 }
1955 _ => codegen_error!(
1956 "singlepass can't emit msub {:?} {:?} {:?} {:?} {:?}",
1957 sz,
1958 a,
1959 b,
1960 c,
1961 dst
1962 ),
1963 }
1964 Ok(())
1965 }
1966
1967 fn emit_sxtb(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1968 match (sz, src, dst) {
1969 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1970 dynasm!(self ; sxtb W(dst), W(src));
1971 }
1972 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1973 dynasm!(self ; sxtb X(dst), W(src));
1974 }
1975 _ => codegen_error!("singlepass can't emit SXTB {:?} {:?} {:?}", sz, src, dst),
1976 }
1977 Ok(())
1978 }
1979 fn emit_sxth(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1980 match (sz, src, dst) {
1981 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1982 dynasm!(self ; sxth W(dst), W(src));
1983 }
1984 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1985 dynasm!(self ; sxth X(dst), W(src));
1986 }
1987 _ => codegen_error!("singlepass can't emit SXTH {:?} {:?} {:?}", sz, src, dst),
1988 }
1989 Ok(())
1990 }
1991 fn emit_sxtw(&mut self, _sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1992 match (src, dst) {
1993 (Location::GPR(src), Location::GPR(dst)) => {
1994 dynasm!(self ; sxtw X(dst), W(src));
1995 }
1996 _ => codegen_error!("singlepass can't emit SXTW {:?} {:?}", src, dst),
1997 }
1998 Ok(())
1999 }
2000 fn emit_uxtb(&mut self, _sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2001 match (src, dst) {
2002 (Location::GPR(src), Location::GPR(dst)) => {
2003 dynasm!(self ; uxtb W(dst), W(src));
2004 }
2005 _ => codegen_error!("singlepass can't emit UXTB {:?} {:?}", src, dst),
2006 }
2007 Ok(())
2008 }
2009 fn emit_uxth(&mut self, _sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2010 match (src, dst) {
2011 (Location::GPR(src), Location::GPR(dst)) => {
2012 dynasm!(self ; uxth W(dst), W(src));
2013 }
2014 _ => codegen_error!("singlepass can't emit UXTH {:?} {:?}", src, dst),
2015 }
2016 Ok(())
2017 }
2018
2019 fn emit_cset(&mut self, sz: Size, dst: Location, cond: Condition) -> Result<(), CompileError> {
2020 match (sz, dst) {
2021 (Size::S32, Location::GPR(reg)) => match cond {
2022 Condition::Eq => dynasm!(self ; cset W(reg), eq),
2023 Condition::Ne => dynasm!(self ; cset W(reg), ne),
2024 Condition::Cs => dynasm!(self ; cset W(reg), cs),
2025 Condition::Cc => dynasm!(self ; cset W(reg), cc),
2026 Condition::Mi => dynasm!(self ; cset W(reg), mi),
2027 Condition::Pl => dynasm!(self ; cset W(reg), pl),
2028 Condition::Vs => dynasm!(self ; cset W(reg), vs),
2029 Condition::Vc => dynasm!(self ; cset W(reg), vc),
2030 Condition::Hi => dynasm!(self ; cset W(reg), hi),
2031 Condition::Ls => dynasm!(self ; cset W(reg), ls),
2032 Condition::Ge => dynasm!(self ; cset W(reg), ge),
2033 Condition::Lt => dynasm!(self ; cset W(reg), lt),
2034 Condition::Gt => dynasm!(self ; cset W(reg), gt),
2035 Condition::Le => dynasm!(self ; cset W(reg), le),
2036 Condition::Al => dynasm!(self ; cset W(reg), al),
2037 },
2038 (Size::S64, Location::GPR(reg)) => match cond {
2039 Condition::Eq => dynasm!(self ; cset X(reg), eq),
2040 Condition::Ne => dynasm!(self ; cset X(reg), ne),
2041 Condition::Cs => dynasm!(self ; cset X(reg), cs),
2042 Condition::Cc => dynasm!(self ; cset X(reg), cc),
2043 Condition::Mi => dynasm!(self ; cset X(reg), mi),
2044 Condition::Pl => dynasm!(self ; cset X(reg), pl),
2045 Condition::Vs => dynasm!(self ; cset X(reg), vs),
2046 Condition::Vc => dynasm!(self ; cset X(reg), vc),
2047 Condition::Hi => dynasm!(self ; cset X(reg), hi),
2048 Condition::Ls => dynasm!(self ; cset X(reg), ls),
2049 Condition::Ge => dynasm!(self ; cset X(reg), ge),
2050 Condition::Lt => dynasm!(self ; cset X(reg), lt),
2051 Condition::Gt => dynasm!(self ; cset X(reg), gt),
2052 Condition::Le => dynasm!(self ; cset X(reg), le),
2053 Condition::Al => dynasm!(self ; cset X(reg), al),
2054 },
2055 _ => codegen_error!("singlepass can't emit CSET {:?} {:?} {:?}", sz, dst, cond),
2056 }
2057 Ok(())
2058 }
2059 fn emit_csetm(&mut self, sz: Size, dst: Location, cond: Condition) -> Result<(), CompileError> {
2060 match (sz, dst) {
2061 (Size::S32, Location::GPR(reg)) => match cond {
2062 Condition::Eq => dynasm!(self ; csetm W(reg), eq),
2063 Condition::Ne => dynasm!(self ; csetm W(reg), ne),
2064 Condition::Cs => dynasm!(self ; csetm W(reg), cs),
2065 Condition::Cc => dynasm!(self ; csetm W(reg), cc),
2066 Condition::Mi => dynasm!(self ; csetm W(reg), mi),
2067 Condition::Pl => dynasm!(self ; csetm W(reg), pl),
2068 Condition::Vs => dynasm!(self ; csetm W(reg), vs),
2069 Condition::Vc => dynasm!(self ; csetm W(reg), vc),
2070 Condition::Hi => dynasm!(self ; csetm W(reg), hi),
2071 Condition::Ls => dynasm!(self ; csetm W(reg), ls),
2072 Condition::Ge => dynasm!(self ; csetm W(reg), ge),
2073 Condition::Lt => dynasm!(self ; csetm W(reg), lt),
2074 Condition::Gt => dynasm!(self ; csetm W(reg), gt),
2075 Condition::Le => dynasm!(self ; csetm W(reg), le),
2076 Condition::Al => dynasm!(self ; csetm W(reg), al),
2077 },
2078 (Size::S64, Location::GPR(reg)) => match cond {
2079 Condition::Eq => dynasm!(self ; csetm X(reg), eq),
2080 Condition::Ne => dynasm!(self ; csetm X(reg), ne),
2081 Condition::Cs => dynasm!(self ; csetm X(reg), cs),
2082 Condition::Cc => dynasm!(self ; csetm X(reg), cc),
2083 Condition::Mi => dynasm!(self ; csetm X(reg), mi),
2084 Condition::Pl => dynasm!(self ; csetm X(reg), pl),
2085 Condition::Vs => dynasm!(self ; csetm X(reg), vs),
2086 Condition::Vc => dynasm!(self ; csetm X(reg), vc),
2087 Condition::Hi => dynasm!(self ; csetm X(reg), hi),
2088 Condition::Ls => dynasm!(self ; csetm X(reg), ls),
2089 Condition::Ge => dynasm!(self ; csetm X(reg), ge),
2090 Condition::Lt => dynasm!(self ; csetm X(reg), lt),
2091 Condition::Gt => dynasm!(self ; csetm X(reg), gt),
2092 Condition::Le => dynasm!(self ; csetm X(reg), le),
2093 Condition::Al => dynasm!(self ; csetm X(reg), al),
2094 },
2095 _ => codegen_error!("singlepass can't emit CSETM {:?} {:?} {:?}", sz, dst, cond),
2096 }
2097 Ok(())
2098 }
2099 fn emit_cinc(
2100 &mut self,
2101 sz: Size,
2102 src: Location,
2103 dst: Location,
2104 cond: Condition,
2105 ) -> Result<(), CompileError> {
2106 match (sz, src, dst) {
2107 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
2108 match cond {
2109 Condition::Eq => dynasm!(self ; cinc W(dst), W(src), eq),
2110 Condition::Ne => dynasm!(self ; cinc W(dst), W(src), ne),
2111 Condition::Cs => dynasm!(self ; cinc W(dst), W(src), cs),
2112 Condition::Cc => dynasm!(self ; cinc W(dst), W(src), cc),
2113 Condition::Mi => dynasm!(self ; cinc W(dst), W(src), mi),
2114 Condition::Pl => dynasm!(self ; cinc W(dst), W(src), pl),
2115 Condition::Vs => dynasm!(self ; cinc W(dst), W(src), vs),
2116 Condition::Vc => dynasm!(self ; cinc W(dst), W(src), vc),
2117 Condition::Hi => dynasm!(self ; cinc W(dst), W(src), hi),
2118 Condition::Ls => dynasm!(self ; cinc W(dst), W(src), ls),
2119 Condition::Ge => dynasm!(self ; cinc W(dst), W(src), ge),
2120 Condition::Lt => dynasm!(self ; cinc W(dst), W(src), lt),
2121 Condition::Gt => dynasm!(self ; cinc W(dst), W(src), gt),
2122 Condition::Le => dynasm!(self ; cinc W(dst), W(src), le),
2123 Condition::Al => dynasm!(self ; cinc W(dst), W(src), al),
2124 };
2125 }
2126 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
2127 match cond {
2128 Condition::Eq => dynasm!(self ; cinc X(src), X(dst), eq),
2129 Condition::Ne => dynasm!(self ; cinc X(src), X(dst), ne),
2130 Condition::Cs => dynasm!(self ; cinc X(src), X(dst), cs),
2131 Condition::Cc => dynasm!(self ; cinc X(src), X(dst), cc),
2132 Condition::Mi => dynasm!(self ; cinc X(src), X(dst), mi),
2133 Condition::Pl => dynasm!(self ; cinc X(src), X(dst), pl),
2134 Condition::Vs => dynasm!(self ; cinc X(src), X(dst), vs),
2135 Condition::Vc => dynasm!(self ; cinc X(src), X(dst), vc),
2136 Condition::Hi => dynasm!(self ; cinc X(src), X(dst), hi),
2137 Condition::Ls => dynasm!(self ; cinc X(src), X(dst), ls),
2138 Condition::Ge => dynasm!(self ; cinc X(src), X(dst), ge),
2139 Condition::Lt => dynasm!(self ; cinc X(src), X(dst), lt),
2140 Condition::Gt => dynasm!(self ; cinc X(src), X(dst), gt),
2141 Condition::Le => dynasm!(self ; cinc X(src), X(dst), le),
2142 Condition::Al => dynasm!(self ; cinc X(src), X(dst), al),
2143 };
2144 }
2145 _ => codegen_error!("singlepass can't emit CINC"),
2146 }
2147 Ok(())
2148 }
2149
2150 fn emit_clz(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2151 match (sz, src, dst) {
2152 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
2153 dynasm!(self ; clz X(dst), X(src));
2154 }
2155 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
2156 dynasm!(self ; clz W(dst), W(src));
2157 }
2158 _ => codegen_error!("singlepass can't emit CLS {:?} {:?} {:?}", sz, src, dst),
2159 }
2160 Ok(())
2161 }
2162 fn emit_rbit(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2163 match (sz, src, dst) {
2164 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
2165 dynasm!(self ; rbit X(dst), X(src));
2166 }
2167 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
2168 dynasm!(self ; rbit W(dst), W(src));
2169 }
2170 _ => codegen_error!("singlepass can't emit CLS {:?} {:?} {:?}", sz, src, dst),
2171 }
2172 Ok(())
2173 }
2174
2175 fn emit_label(&mut self, label: Label) -> Result<(), CompileError> {
2176 dynasm!(self ; => label);
2177 Ok(())
2178 }
2179 fn emit_load_label(&mut self, reg: GPR, label: Label) -> Result<(), CompileError> {
2180 dynasm!(self ; adr X(reg), =>label);
2181 Ok(())
2182 }
2183
2184 fn emit_b_label(&mut self, label: Label) -> Result<(), CompileError> {
2185 dynasm!(self ; b =>label);
2186 Ok(())
2187 }
2188 fn emit_cbz_label(
2189 &mut self,
2190 sz: Size,
2191 reg: Location,
2192 label: Label,
2193 ) -> Result<(), CompileError> {
2194 match (sz, reg) {
2195 (Size::S32, Location::GPR(reg)) => {
2196 dynasm!(self ; cbz W(reg), =>label);
2197 }
2198 (Size::S64, Location::GPR(reg)) => {
2199 dynasm!(self ; cbz X(reg), =>label);
2200 }
2201 _ => codegen_error!("singlepass can't emit CBZ {:?} {:?} {:?}", sz, reg, label),
2202 }
2203 Ok(())
2204 }
2205 fn emit_cbnz_label(
2206 &mut self,
2207 sz: Size,
2208 reg: Location,
2209 label: Label,
2210 ) -> Result<(), CompileError> {
2211 match (sz, reg) {
2212 (Size::S32, Location::GPR(reg)) => {
2213 dynasm!(self ; cbnz W(reg), =>label);
2214 }
2215 (Size::S64, Location::GPR(reg)) => {
2216 dynasm!(self ; cbnz X(reg), =>label);
2217 }
2218 _ => codegen_error!("singlepass can't emit CBNZ {:?} {:?} {:?}", sz, reg, label),
2219 }
2220 Ok(())
2221 }
2222 fn emit_cbz_label_far(
2223 &mut self,
2224 sz: Size,
2225 reg: Location,
2226 label: Label,
2227 ) -> Result<(), CompileError> {
2228 let near_label: Label = self.get_label();
2229 let continue_label: Label = self.get_label();
2230
2231 match (sz, reg) {
2232 (Size::S32, Location::GPR(reg)) => {
2233 dynasm!(self ; cbz W(reg), => near_label);
2234 dynasm!(self ; b => continue_label);
2235 }
2236 (Size::S64, Location::GPR(reg)) => {
2237 dynasm!(self ; cbz X(reg), => near_label);
2238 dynasm!(self ; b => continue_label);
2239 }
2240 _ => codegen_error!("singlepass can't emit CBZ {:?} {:?} {:?}", sz, reg, label),
2241 }
2242 self.emit_label(near_label)?;
2243 dynasm!(self ; b => label );
2244
2245 self.emit_label(continue_label)?;
2246
2247 Ok(())
2248 }
2249 fn emit_tbz_label(
2250 &mut self,
2251 sz: Size,
2252 reg: Location,
2253 n: u32,
2254 label: Label,
2255 ) -> Result<(), CompileError> {
2256 match (sz, reg) {
2257 (Size::S32, Location::GPR(reg)) => {
2258 dynasm!(self ; tbz W(reg), n, =>label);
2259 }
2260 (Size::S64, Location::GPR(reg)) => {
2261 dynasm!(self ; tbz X(reg), n, =>label);
2262 }
2263 _ => codegen_error!(
2264 "singlepass can't emit TBZ {:?} {:?} {:?} {:?}",
2265 sz,
2266 reg,
2267 n,
2268 label
2269 ),
2270 }
2271 Ok(())
2272 }
2273 fn emit_tbnz_label(
2274 &mut self,
2275 sz: Size,
2276 reg: Location,
2277 n: u32,
2278 label: Label,
2279 ) -> Result<(), CompileError> {
2280 match (sz, reg) {
2281 (Size::S32, Location::GPR(reg)) => {
2282 dynasm!(self ; tbnz W(reg), n, =>label);
2283 }
2284 (Size::S64, Location::GPR(reg)) => {
2285 dynasm!(self ; tbnz X(reg), n, =>label);
2286 }
2287 _ => codegen_error!(
2288 "singlepass can't emit TBNZ {:?} {:?} {:?} {:?}",
2289 sz,
2290 reg,
2291 n,
2292 label
2293 ),
2294 }
2295 Ok(())
2296 }
2297 fn emit_bcond_label(&mut self, condition: Condition, label: Label) -> Result<(), CompileError> {
2298 match condition {
2299 Condition::Eq => dynasm!(self ; b.eq => label),
2300 Condition::Ne => dynasm!(self ; b.ne => label),
2301 Condition::Cs => dynasm!(self ; b.cs => label),
2302 Condition::Cc => dynasm!(self ; b.cc => label),
2303 Condition::Mi => dynasm!(self ; b.mi => label),
2304 Condition::Pl => dynasm!(self ; b.pl => label),
2305 Condition::Vs => dynasm!(self ; b.vs => label),
2306 Condition::Vc => dynasm!(self ; b.vc => label),
2307 Condition::Hi => dynasm!(self ; b.hi => label),
2308 Condition::Ls => dynasm!(self ; b.ls => label),
2309 Condition::Ge => dynasm!(self ; b.ge => label),
2310 Condition::Lt => dynasm!(self ; b.lt => label),
2311 Condition::Gt => dynasm!(self ; b.gt => label),
2312 Condition::Le => dynasm!(self ; b.le => label),
2313 Condition::Al => dynasm!(self ; b => label),
2314 }
2315 Ok(())
2316 }
2317 fn emit_bcond_label_far(
2318 &mut self,
2319 condition: Condition,
2320 label: Label,
2321 ) -> Result<(), CompileError> {
2322 let cont: Label = self.get_label();
2323 match condition {
2324 Condition::Eq => dynasm!(self ; b.ne => cont),
2326 Condition::Ne => dynasm!(self ; b.eq => cont),
2327 Condition::Cs => dynasm!(self ; b.cc => cont),
2328 Condition::Cc => dynasm!(self ; b.cs => cont),
2329 Condition::Mi => dynasm!(self ; b.pl => cont),
2330 Condition::Pl => dynasm!(self ; b.mi => cont),
2331 Condition::Vs => dynasm!(self ; b.vc => cont),
2332 Condition::Vc => dynasm!(self ; b.vs => cont),
2333 Condition::Hi => dynasm!(self ; b.ls => cont),
2334 Condition::Ls => dynasm!(self ; b.hi => cont),
2335 Condition::Ge => dynasm!(self ; b.lt => cont),
2336 Condition::Lt => dynasm!(self ; b.ge => cont),
2337 Condition::Gt => dynasm!(self ; b.le => cont),
2338 Condition::Le => dynasm!(self ; b.gt => cont),
2339 Condition::Al => { }
2340 }
2341 dynasm!(self ; b => label);
2342 self.emit_label(cont)?;
2343 Ok(())
2344 }
2345 fn emit_b_register(&mut self, reg: GPR) -> Result<(), CompileError> {
2346 dynasm!(self ; br X(reg));
2347 Ok(())
2348 }
2349 fn emit_call_label(&mut self, label: Label) -> Result<(), CompileError> {
2350 dynasm!(self ; bl =>label);
2351 Ok(())
2352 }
2353 fn emit_call_register(&mut self, reg: GPR) -> Result<(), CompileError> {
2354 dynasm!(self ; blr X(reg));
2355 Ok(())
2356 }
2357 fn emit_ret(&mut self) -> Result<(), CompileError> {
2358 dynasm!(self ; ret);
2359 Ok(())
2360 }
2361
2362 fn emit_udf(&mut self, payload: u16) -> Result<(), CompileError> {
2363 dynasm!(self ; udf payload as u32);
2364 Ok(())
2365 }
2366 fn emit_dmb(&mut self) -> Result<(), CompileError> {
2367 dynasm!(self ; dmb ish);
2368 Ok(())
2369 }
2370 fn emit_brk(&mut self) -> Result<(), CompileError> {
2371 dynasm!(self ; brk 0);
2372 Ok(())
2373 }
2374
2375 fn emit_fcmp(&mut self, sz: Size, src1: Location, src2: Location) -> Result<(), CompileError> {
2376 match (sz, src1, src2) {
2377 (Size::S32, Location::SIMD(src1), Location::SIMD(src2)) => {
2378 dynasm!(self ; fcmp S(src1), S(src2));
2379 }
2380 (Size::S64, Location::SIMD(src1), Location::SIMD(src2)) => {
2381 dynasm!(self ; fcmp D(src1), D(src2));
2382 }
2383 _ => codegen_error!("singlepass can't emit FCMP {:?} {:?} {:?}", sz, src1, src2),
2384 }
2385 Ok(())
2386 }
2387
2388 fn emit_fneg(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2389 match (sz, src, dst) {
2390 (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2391 dynasm!(self ; fneg S(dst), S(src));
2392 }
2393 (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2394 dynasm!(self ; fneg D(dst), D(src));
2395 }
2396 _ => codegen_error!("singlepass can't emit FNEG {:?} {:?} {:?}", sz, src, dst),
2397 }
2398 Ok(())
2399 }
2400 fn emit_fsqrt(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2401 match (sz, src, dst) {
2402 (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2403 dynasm!(self ; fsqrt S(dst), S(src));
2404 }
2405 (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2406 dynasm!(self ; fsqrt D(dst), D(src));
2407 }
2408 _ => codegen_error!("singlepass can't emit FSQRT {:?} {:?} {:?}", sz, src, dst),
2409 }
2410 Ok(())
2411 }
2412
2413 fn emit_fadd(
2414 &mut self,
2415 sz: Size,
2416 src1: Location,
2417 src2: Location,
2418 dst: Location,
2419 ) -> Result<(), CompileError> {
2420 match (sz, src1, src2, dst) {
2421 (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2422 dynasm!(self ; fadd S(dst), S(src1), S(src2));
2423 }
2424 (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2425 dynasm!(self ; fadd D(dst), D(src1), D(src2));
2426 }
2427 _ => codegen_error!(
2428 "singlepass can't emit FADD {:?} {:?} {:?} {:?}",
2429 sz,
2430 src1,
2431 src2,
2432 dst
2433 ),
2434 }
2435 Ok(())
2436 }
2437 fn emit_fsub(
2438 &mut self,
2439 sz: Size,
2440 src1: Location,
2441 src2: Location,
2442 dst: Location,
2443 ) -> Result<(), CompileError> {
2444 match (sz, src1, src2, dst) {
2445 (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2446 dynasm!(self ; fsub S(dst), S(src1), S(src2));
2447 }
2448 (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2449 dynasm!(self ; fsub D(dst), D(src1), D(src2));
2450 }
2451 _ => codegen_error!(
2452 "singlepass can't emit FSUB {:?} {:?} {:?} {:?}",
2453 sz,
2454 src1,
2455 src2,
2456 dst
2457 ),
2458 }
2459 Ok(())
2460 }
2461 fn emit_fmul(
2462 &mut self,
2463 sz: Size,
2464 src1: Location,
2465 src2: Location,
2466 dst: Location,
2467 ) -> Result<(), CompileError> {
2468 match (sz, src1, src2, dst) {
2469 (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2470 dynasm!(self ; fmul S(dst), S(src1), S(src2));
2471 }
2472 (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2473 dynasm!(self ; fmul D(dst), D(src1), D(src2));
2474 }
2475 _ => codegen_error!(
2476 "singlepass can't emit FMUL {:?} {:?} {:?} {:?}",
2477 sz,
2478 src1,
2479 src2,
2480 dst
2481 ),
2482 }
2483 Ok(())
2484 }
2485 fn emit_fdiv(
2486 &mut self,
2487 sz: Size,
2488 src1: Location,
2489 src2: Location,
2490 dst: Location,
2491 ) -> Result<(), CompileError> {
2492 match (sz, src1, src2, dst) {
2493 (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2494 dynasm!(self ; fdiv S(dst), S(src1), S(src2));
2495 }
2496 (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2497 dynasm!(self ; fdiv D(dst), D(src1), D(src2));
2498 }
2499 _ => codegen_error!(
2500 "singlepass can't emit FDIV {:?} {:?} {:?} {:?}",
2501 sz,
2502 src1,
2503 src2,
2504 dst
2505 ),
2506 }
2507 Ok(())
2508 }
2509
2510 fn emit_fmin(
2511 &mut self,
2512 sz: Size,
2513 src1: Location,
2514 src2: Location,
2515 dst: Location,
2516 ) -> Result<(), CompileError> {
2517 match (sz, src1, src2, dst) {
2518 (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2519 dynasm!(self ; fmin S(dst), S(src1), S(src2));
2520 }
2521 (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2522 dynasm!(self ; fmin D(dst), D(src1), D(src2));
2523 }
2524 _ => codegen_error!(
2525 "singlepass can't emit FMIN {:?} {:?} {:?} {:?}",
2526 sz,
2527 src1,
2528 src2,
2529 dst
2530 ),
2531 }
2532 Ok(())
2533 }
2534 fn emit_fmax(
2535 &mut self,
2536 sz: Size,
2537 src1: Location,
2538 src2: Location,
2539 dst: Location,
2540 ) -> Result<(), CompileError> {
2541 match (sz, src1, src2, dst) {
2542 (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2543 dynasm!(self ; fmax S(dst), S(src1), S(src2));
2544 }
2545 (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2546 dynasm!(self ; fmax D(dst), D(src1), D(src2));
2547 }
2548 _ => codegen_error!(
2549 "singlepass can't emit FMAX {:?} {:?} {:?} {:?}",
2550 sz,
2551 src1,
2552 src2,
2553 dst
2554 ),
2555 }
2556 Ok(())
2557 }
2558
2559 fn emit_frintz(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2560 match (sz, src, dst) {
2561 (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2562 dynasm!(self ; frintz S(dst), S(src));
2563 }
2564 (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2565 dynasm!(self ; frintz D(dst), D(src));
2566 }
2567 _ => codegen_error!("singlepass can't emit FRINTZ {:?} {:?} {:?}", sz, src, dst),
2568 }
2569 Ok(())
2570 }
2571 fn emit_frintn(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2572 match (sz, src, dst) {
2573 (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2574 dynasm!(self ; frintn S(dst), S(src));
2575 }
2576 (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2577 dynasm!(self ; frintn D(dst), D(src));
2578 }
2579 _ => codegen_error!("singlepass can't emit FRINTN {:?} {:?} {:?}", sz, src, dst),
2580 }
2581 Ok(())
2582 }
2583 fn emit_frintm(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2584 match (sz, src, dst) {
2585 (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2586 dynasm!(self ; frintm S(dst), S(src));
2587 }
2588 (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2589 dynasm!(self ; frintm D(dst), D(src));
2590 }
2591 _ => codegen_error!("singlepass can't emit FRINTM {:?} {:?} {:?}", sz, src, dst),
2592 }
2593 Ok(())
2594 }
2595 fn emit_frintp(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2596 match (sz, src, dst) {
2597 (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2598 dynasm!(self ; frintp S(dst), S(src));
2599 }
2600 (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2601 dynasm!(self ; frintp D(dst), D(src));
2602 }
2603 _ => codegen_error!("singlepass can't emit FRINTP {:?} {:?} {:?}", sz, src, dst),
2604 }
2605 Ok(())
2606 }
2607
2608 fn emit_scvtf(
2609 &mut self,
2610 sz_in: Size,
2611 src: Location,
2612 sz_out: Size,
2613 dst: Location,
2614 ) -> Result<(), CompileError> {
2615 match (sz_in, src, sz_out, dst) {
2616 (Size::S32, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
2617 dynasm!(self ; scvtf S(dst), W(src));
2618 }
2619 (Size::S64, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
2620 dynasm!(self ; scvtf S(dst), X(src));
2621 }
2622 (Size::S32, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
2623 dynasm!(self ; scvtf D(dst), W(src));
2624 }
2625 (Size::S64, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
2626 dynasm!(self ; scvtf D(dst), X(src));
2627 }
2628 _ => codegen_error!(
2629 "singlepass can't emit SCVTF {:?} {:?} {:?} {:?}",
2630 sz_in,
2631 src,
2632 sz_out,
2633 dst
2634 ),
2635 }
2636 Ok(())
2637 }
2638 fn emit_ucvtf(
2639 &mut self,
2640 sz_in: Size,
2641 src: Location,
2642 sz_out: Size,
2643 dst: Location,
2644 ) -> Result<(), CompileError> {
2645 match (sz_in, src, sz_out, dst) {
2646 (Size::S32, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
2647 dynasm!(self ; ucvtf S(dst), W(src));
2648 }
2649 (Size::S64, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
2650 dynasm!(self ; ucvtf S(dst), X(src));
2651 }
2652 (Size::S32, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
2653 dynasm!(self ; ucvtf D(dst), W(src));
2654 }
2655 (Size::S64, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
2656 dynasm!(self ; ucvtf D(dst), X(src));
2657 }
2658 _ => codegen_error!(
2659 "singlepass can't emit UCVTF {:?} {:?} {:?} {:?}",
2660 sz_in,
2661 src,
2662 sz_out,
2663 dst
2664 ),
2665 }
2666 Ok(())
2667 }
2668 fn emit_fcvt(&mut self, sz_in: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2669 match (sz_in, src, dst) {
2670 (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2671 dynasm!(self ; fcvt D(dst), S(src));
2672 }
2673 (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2674 dynasm!(self ; fcvt S(dst), D(src));
2675 }
2676 _ => codegen_error!(
2677 "singlepass can't emit UCVTF {:?} {:?} {:?}",
2678 sz_in,
2679 src,
2680 dst
2681 ),
2682 }
2683 Ok(())
2684 }
2685 fn emit_fcvtzs(
2686 &mut self,
2687 sz_in: Size,
2688 src: Location,
2689 sz_out: Size,
2690 dst: Location,
2691 ) -> Result<(), CompileError> {
2692 match (sz_in, src, sz_out, dst) {
2693 (Size::S32, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
2694 dynasm!(self ; fcvtzs W(dst), S(src));
2695 }
2696 (Size::S64, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
2697 dynasm!(self ; fcvtzs W(dst), D(src));
2698 }
2699 (Size::S32, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
2700 dynasm!(self ; fcvtzs X(dst), S(src));
2701 }
2702 (Size::S64, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
2703 dynasm!(self ; fcvtzs X(dst), D(src));
2704 }
2705 _ => codegen_error!(
2706 "singlepass can't emit FCVTZS {:?} {:?} {:?} {:?}",
2707 sz_in,
2708 src,
2709 sz_out,
2710 dst
2711 ),
2712 }
2713 Ok(())
2714 }
2715 fn emit_fcvtzu(
2716 &mut self,
2717 sz_in: Size,
2718 src: Location,
2719 sz_out: Size,
2720 dst: Location,
2721 ) -> Result<(), CompileError> {
2722 match (sz_in, src, sz_out, dst) {
2723 (Size::S32, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
2724 dynasm!(self ; fcvtzu W(dst), S(src));
2725 }
2726 (Size::S64, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
2727 dynasm!(self ; fcvtzu W(dst), D(src));
2728 }
2729 (Size::S32, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
2730 dynasm!(self ; fcvtzu X(dst), S(src));
2731 }
2732 (Size::S64, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
2733 dynasm!(self ; fcvtzu X(dst), D(src));
2734 }
2735 _ => codegen_error!(
2736 "singlepass can't emit FCVTZU {:?} {:?} {:?} {:?}",
2737 sz_in,
2738 src,
2739 sz_out,
2740 dst
2741 ),
2742 }
2743 Ok(())
2744 }
2745
2746 fn emit_read_fpcr(&mut self, reg: GPR) -> Result<(), CompileError> {
2748 dynasm!(self ; mrs X(reg), 0b1_011_0100_0100_000);
2749 Ok(())
2750 }
2751 fn emit_write_fpcr(&mut self, reg: GPR) -> Result<(), CompileError> {
2752 dynasm!(self ; msr 0b1_011_0100_0100_000, X(reg));
2753 Ok(())
2754 }
2755 fn emit_read_fpsr(&mut self, reg: GPR) -> Result<(), CompileError> {
2757 dynasm!(self ; mrs X(reg), 0b1_011_0100_0100_001);
2758 Ok(())
2759 }
2760 fn emit_write_fpsr(&mut self, reg: GPR) -> Result<(), CompileError> {
2761 dynasm!(self ; msr 0b1_011_0100_0100_001, X(reg));
2762 Ok(())
2763 }
2764
2765 fn emit_cnt(&mut self, src: NEON, dst: NEON) -> Result<(), CompileError> {
2766 dynasm!(self ; cnt V(dst).B8, V(src).B8);
2767 Ok(())
2768 }
2769
2770 fn emit_addv(&mut self, src: NEON, dst: NEON) -> Result<(), CompileError> {
2771 dynasm!(self ; addv B(dst), V(src).B8);
2772 Ok(())
2773 }
2774
2775 #[allow(clippy::unit_arg)]
2776 fn emit_fmov(
2777 &mut self,
2778 src_size: Size,
2779 src: Location,
2780 dst_size: Size,
2781 dst: Location,
2782 ) -> Result<(), CompileError> {
2783 let mut err = false;
2784 match (src, dst) {
2785 (AbstractLocation::GPR(src), AbstractLocation::SIMD(dst)) => {
2786 match (src_size, dst_size) {
2787 (Size::S32, Size::S32) => dynasm!(self ; fmov S(dst), W(src)),
2788 (Size::S64, Size::S64) => dynasm!(self ; fmov D(dst), X(src)),
2789 _ => {
2790 err = true;
2791 }
2792 }
2793 }
2794 (AbstractLocation::SIMD(src), AbstractLocation::GPR(dst)) => {
2795 match (src_size, dst_size) {
2796 (Size::S32, Size::S32) => dynasm!(self ; fmov W(dst), S(src)),
2797 (Size::S64, Size::S64) => dynasm!(self ; fmov X(dst), D(src)),
2798 _ => {
2799 err = true;
2800 }
2801 }
2802 }
2803 _ => {
2807 err = true;
2808 }
2809 };
2810
2811 if err {
2812 codegen_error!(
2813 "singlepass can't generate fmov instruction for src_size: {:?}, src: {:?}, dst_size: {:?}, dst: {:?}",
2814 src_size,
2815 src,
2816 dst_size,
2817 dst,
2818 )
2819 }
2820 Ok(())
2821 }
2822}
2823
2824pub fn gen_std_trampoline_arm64(
2825 sig: &FunctionType,
2826 calling_convention: CallingConvention,
2827) -> Result<FunctionBody, CompileError> {
2828 let mut a = Assembler::new(0);
2829
2830 let fptr = GPR::X27;
2831 let args = GPR::X28;
2832
2833 dynasm!(a
2834 ; sub sp, sp, 32
2835 ; stp x29, x30, [sp]
2836 ; stp X(fptr), X(args), [sp, 16]
2837 ; mov x29, sp
2838 ; mov X(fptr), x1
2839 ; mov X(args), x2
2840 );
2841
2842 let stack_args = sig.params().len().saturating_sub(7); let stack_return_slots = sig
2844 .results()
2845 .len()
2846 .saturating_sub(ARM64_RETURN_VALUE_REGISTERS.len());
2847 let mut stack_offset = (stack_args + stack_return_slots) as u32 * 8;
2848 if stack_offset > 0 {
2849 if !stack_offset.is_multiple_of(16) {
2850 stack_offset += 8;
2851 assert!(stack_offset.is_multiple_of(16));
2852 }
2853 if stack_offset < 0x1000 {
2854 dynasm!(a ; sub sp, sp, stack_offset);
2855 } else {
2856 a.emit_mov_imm(Location::GPR(GPR::X26), stack_offset as u64)?;
2857 dynasm!(a ; sub sp, sp, x26);
2858 }
2859 }
2860
2861 let mut caller_stack_offset = stack_return_slots as u32 * 8;
2864 for (i, param) in sig.params().iter().enumerate() {
2865 let sz = match *param {
2866 Type::I32 | Type::F32 => Size::S32,
2867 Type::I64 | Type::F64 => Size::S64,
2868 Type::ExternRef => Size::S64,
2869 Type::FuncRef => Size::S64,
2870 _ => codegen_error!(
2871 "singlepass unsupported param type for trampoline {:?}",
2872 *param
2873 ),
2874 };
2875 match i {
2876 0..=6 => {
2877 a.emit_ldr(
2878 sz,
2879 Location::GPR(GPR::from_index(i + 1).unwrap()),
2880 Location::Memory(args, (i * 16) as i32),
2881 )?;
2882 }
2883 _ => {
2884 #[allow(clippy::single_match)]
2885 match calling_convention {
2886 CallingConvention::AppleAarch64 => {
2887 caller_stack_offset = caller_stack_offset.next_multiple_of(sz.bytes());
2889 }
2890 _ => (),
2891 };
2892 a.emit_ldr(
2894 sz,
2895 Location::GPR(GPR::X16),
2896 Location::Memory(args, (i * 16) as i32),
2897 )?;
2898 a.emit_str(
2899 sz,
2900 Location::GPR(GPR::X16),
2901 Location::Memory(GPR::XzrSp, caller_stack_offset as _),
2902 )?;
2903 match calling_convention {
2904 CallingConvention::AppleAarch64 => {
2905 caller_stack_offset += sz.bytes();
2906 }
2907 _ => {
2908 caller_stack_offset += 8;
2909 }
2910 }
2911 }
2912 }
2913 }
2914
2915 dynasm!(a ; blr X(fptr));
2916
2917 let mut n_stack_return_slots: usize = 0;
2919 for i in 0..sig.results().len() {
2920 let src = if let Some(®) = ARM64_RETURN_VALUE_REGISTERS.get(i) {
2921 reg
2922 } else {
2923 let loc = GPR::X16;
2924 a.emit_ldr(
2925 Size::S64,
2926 Location::GPR(GPR::X16),
2927 Location::Memory(GPR::XzrSp, (n_stack_return_slots as u32 * 8) as _),
2928 )?;
2929 n_stack_return_slots += 1;
2930 loc
2931 };
2932 a.emit_str(
2933 Size::S64,
2934 Location::GPR(src),
2935 Location::Memory(args, (i * 16) as _),
2936 )?;
2937 }
2938
2939 dynasm!(a
2941 ; ldp X(fptr), X(args), [x29, 16]
2942 ; ldp x29, x30, [x29]
2943 );
2944 let restored_stack_offset = 32 + stack_offset;
2945 if restored_stack_offset < 0x1000 {
2946 dynasm!(a; add sp, sp, restored_stack_offset);
2947 } else {
2948 a.emit_mov_imm(Location::GPR(GPR::X26), restored_stack_offset as u64)?;
2949 dynasm!(a; add sp, sp, x26);
2950 }
2951 dynasm!(a; ret);
2952
2953 let mut body = a.finalize().unwrap();
2954 body.shrink_to_fit();
2955 Ok(FunctionBody {
2956 body,
2957 unwind_info: None,
2958 })
2959}
2960pub fn gen_std_dynamic_import_trampoline_arm64(
2962 vmoffsets: &VMOffsets,
2963 sig: &FunctionType,
2964 calling_convention: CallingConvention,
2965) -> Result<FunctionBody, CompileError> {
2966 let mut a = Assembler::new(0);
2967 let stack_offset: usize = 16 * std::cmp::max(sig.params().len(), sig.results().len());
2969 a.emit_stpdb(
2971 Size::S64,
2972 Location::GPR(GPR::X30),
2973 Location::GPR(GPR::X26),
2974 GPR::XzrSp,
2975 16,
2976 )?;
2977
2978 if stack_offset != 0 {
2979 if stack_offset < 0x1000 {
2980 a.emit_sub(
2981 Size::S64,
2982 Location::GPR(GPR::XzrSp),
2983 Location::Imm32(stack_offset as _),
2984 Location::GPR(GPR::XzrSp),
2985 )?;
2986 } else {
2987 a.emit_mov_imm(Location::GPR(GPR::X26), stack_offset as u64)?;
2988 a.emit_sub(
2989 Size::S64,
2990 Location::GPR(GPR::XzrSp),
2991 Location::GPR(GPR::X26),
2992 Location::GPR(GPR::XzrSp),
2993 )?;
2994 }
2995 }
2996
2997 if !sig.params().is_empty() {
2999 let mut argalloc = ArgumentRegisterAllocator::default();
3000 argalloc.next(Type::I64, calling_convention).unwrap(); let mut stack_param_count: usize = 0;
3003
3004 for (i, ty) in sig.params().iter().enumerate() {
3005 let source_loc = match argalloc.next(*ty, calling_convention)? {
3006 Some(ARM64Register::GPR(gpr)) => Location::GPR(gpr),
3007 Some(ARM64Register::NEON(neon)) => Location::SIMD(neon),
3008 None => {
3009 let sz = match calling_convention {
3010 CallingConvention::AppleAarch64 => match *ty {
3011 Type::I32 | Type::F32 => Size::S32,
3012 _ => {
3013 stack_param_count = stack_param_count.next_multiple_of(8);
3014 Size::S64
3015 }
3016 },
3017 _ => Size::S64,
3018 };
3019 a.emit_ldr(
3020 sz,
3021 Location::GPR(GPR::X26),
3022 Location::Memory(GPR::XzrSp, (stack_offset + 16 + stack_param_count) as _),
3023 )?;
3024 stack_param_count += match sz {
3025 Size::S32 => 4,
3026 Size::S64 => 8,
3027 _ => codegen_error!(
3028 "singlepass unreachable in gen_std_dynamic_import_trampoline_arm64"
3029 ),
3030 };
3031 Location::GPR(GPR::X26)
3032 }
3033 };
3034 a.emit_str(
3035 Size::S64,
3036 source_loc,
3037 Location::Memory(GPR::XzrSp, (i * 16) as _),
3038 )?;
3039
3040 a.emit_str(
3042 Size::S64,
3043 Location::GPR(GPR::XzrSp), Location::Memory(GPR::XzrSp, (i * 16 + 8) as _), )?;
3046 }
3047 }
3048
3049 #[allow(clippy::match_single_binding)]
3050 match calling_convention {
3051 _ => {
3052 let offset = vmoffsets.vmdynamicfunction_import_context_address();
3054 a.emit_ldur(Size::S64, Location::GPR(GPR::X26), GPR::X0, offset as i32)?;
3055 a.emit_add(
3057 Size::S64,
3058 Location::GPR(GPR::XzrSp),
3059 Location::Imm8(0),
3060 Location::GPR(GPR::X1),
3061 )?;
3062 }
3063 };
3064
3065 a.emit_call_register(GPR::X26)?;
3067
3068 if !sig.results().is_empty() {
3070 assert_eq!(sig.results().len(), 1);
3071 a.emit_ldr(
3072 Size::S64,
3073 Location::GPR(GPR::X0),
3074 Location::Memory(GPR::XzrSp, 0),
3075 )?;
3076 }
3077
3078 if stack_offset != 0 {
3080 if stack_offset < 0x1000 {
3081 a.emit_add(
3082 Size::S64,
3083 Location::GPR(GPR::XzrSp),
3084 Location::Imm32(stack_offset as _),
3085 Location::GPR(GPR::XzrSp),
3086 )?;
3087 } else {
3088 a.emit_mov_imm(Location::GPR(GPR::X26), stack_offset as u64)?;
3089 a.emit_add(
3090 Size::S64,
3091 Location::GPR(GPR::XzrSp),
3092 Location::GPR(GPR::X26),
3093 Location::GPR(GPR::XzrSp),
3094 )?;
3095 }
3096 }
3097 a.emit_ldpia(
3098 Size::S64,
3099 Location::GPR(GPR::X30),
3100 Location::GPR(GPR::X26),
3101 GPR::XzrSp,
3102 16,
3103 )?;
3104
3105 a.emit_ret()?;
3107
3108 let mut body = a.finalize().unwrap();
3109 body.shrink_to_fit();
3110 Ok(FunctionBody {
3111 body,
3112 unwind_info: None,
3113 })
3114}
3115pub fn gen_import_call_trampoline_arm64(
3117 vmoffsets: &VMOffsets,
3118 index: FunctionIndex,
3119 sig: &FunctionType,
3120 calling_convention: CallingConvention,
3121) -> Result<CustomSection, CompileError> {
3122 let mut a = Assembler::new(0);
3123
3124 if sig
3129 .params()
3130 .iter()
3131 .any(|&x| x == Type::F32 || x == Type::F64)
3132 {
3133 #[allow(clippy::match_single_binding)]
3134 match calling_convention {
3135 _ => {
3136 let mut stack_offset: u32 = if sig.params().len() > 7 {
3138 7 * 8
3139 } else {
3140 (sig.params().len() as u32) * 8
3141 };
3142 stack_offset = stack_offset.next_multiple_of(16);
3143 if stack_offset != 0 {
3144 if stack_offset < 0x1000 {
3145 a.emit_sub(
3146 Size::S64,
3147 Location::GPR(GPR::XzrSp),
3148 Location::Imm32(stack_offset as u32),
3149 Location::GPR(GPR::XzrSp),
3150 )?;
3151 } else {
3152 a.emit_mov_imm(Location::GPR(GPR::X16), stack_offset as u64)?;
3153 a.emit_sub(
3154 Size::S64,
3155 Location::GPR(GPR::XzrSp),
3156 Location::GPR(GPR::X16),
3157 Location::GPR(GPR::XzrSp),
3158 )?;
3159 }
3160 }
3161
3162 static PARAM_REGS: &[GPR] = &[
3164 GPR::X1,
3165 GPR::X2,
3166 GPR::X3,
3167 GPR::X4,
3168 GPR::X5,
3169 GPR::X6,
3170 GPR::X7,
3171 ];
3172 let mut param_locations = vec![];
3173 for (i, param_reg) in PARAM_REGS.iter().enumerate().take(sig.params().len()) {
3174 let loc = match i {
3175 0..=6 => {
3176 let loc = Location::Memory(GPR::XzrSp, (i * 8) as i32);
3177 a.emit_str(Size::S64, Location::GPR(*param_reg), loc)?;
3178 loc
3179 }
3180 _ => {
3181 Location::Memory(GPR::XzrSp, (stack_offset + (i as u32 - 7) * 8) as i32)
3182 }
3183 };
3184 param_locations.push(loc);
3185 }
3186
3187 let mut caller_stack_offset: u32 = 0;
3189 let mut argalloc = ArgumentRegisterAllocator::default();
3190 argalloc.next(Type::I64, calling_convention).unwrap(); for (i, ty) in sig.params().iter().enumerate() {
3192 let prev_loc = param_locations[i];
3193 let targ = match argalloc.next(*ty, calling_convention)? {
3194 Some(ARM64Register::GPR(gpr)) => Location::GPR(gpr),
3195 Some(ARM64Register::NEON(neon)) => Location::SIMD(neon),
3196 None => {
3197 a.emit_ldr(Size::S64, Location::GPR(GPR::X16), prev_loc)?;
3199 a.emit_str(
3200 Size::S64,
3201 Location::GPR(GPR::X16),
3202 Location::Memory(
3203 GPR::XzrSp,
3204 (stack_offset + caller_stack_offset) as i32,
3205 ),
3206 )?;
3207 caller_stack_offset += 8;
3208 continue;
3209 }
3210 };
3211 a.emit_ldr(Size::S64, targ, prev_loc)?;
3212 }
3213
3214 if stack_offset > 0 {
3216 if stack_offset < 0x1000 {
3217 a.emit_add(
3218 Size::S64,
3219 Location::GPR(GPR::XzrSp),
3220 Location::Imm32(stack_offset as u32),
3221 Location::GPR(GPR::XzrSp),
3222 )?;
3223 } else {
3224 a.emit_mov_imm(Location::GPR(GPR::X16), stack_offset as u64)?;
3225 a.emit_add(
3226 Size::S64,
3227 Location::GPR(GPR::XzrSp),
3228 Location::GPR(GPR::X16),
3229 Location::GPR(GPR::XzrSp),
3230 )?;
3231 }
3232 }
3233 }
3234 }
3235 }
3236
3237 let offset = vmoffsets.vmctx_vmfunction_import(index);
3241 let offset =
3244 if (offset > 0) && ((offset < 0xF8) || (offset < 0x7FF8 && offset.is_multiple_of(8))) {
3245 offset
3246 } else {
3247 a.emit_mov_imm(Location::GPR(GPR::X16), (offset as i64) as u64)?;
3248 a.emit_add(
3249 Size::S64,
3250 Location::GPR(GPR::X0),
3251 Location::GPR(GPR::X16),
3252 Location::GPR(GPR::X0),
3253 )?;
3254 0
3255 };
3256 #[allow(clippy::match_single_binding)]
3257 match calling_convention {
3258 _ => {
3259 if offset.is_multiple_of(8) {
3260 a.emit_ldr(
3261 Size::S64,
3262 Location::GPR(GPR::X16),
3263 Location::Memory(GPR::X0, offset as i32), )?;
3265 a.emit_ldr(
3266 Size::S64,
3267 Location::GPR(GPR::X0),
3268 Location::Memory(GPR::X0, offset as i32 + 8), )?;
3270 } else {
3271 a.emit_ldur(
3272 Size::S64,
3273 Location::GPR(GPR::X16),
3274 GPR::X0,
3275 offset as i32, )?;
3277 a.emit_ldur(
3278 Size::S64,
3279 Location::GPR(GPR::X0),
3280 GPR::X0,
3281 offset as i32 + 8, )?;
3283 }
3284 }
3285 }
3286 a.emit_b_register(GPR::X16)?;
3287
3288 let mut contents = a.finalize().unwrap();
3289 contents.shrink_to_fit();
3290 let section_body = SectionBody::new_with_vec(contents);
3291
3292 Ok(CustomSection {
3293 protection: CustomSectionProtection::ReadExecute,
3294 alignment: None,
3295 bytes: section_body,
3296 relocations: vec![],
3297 })
3298}