1pub use crate::{
2 arm64_decl::{ARM64Register, ArgumentRegisterAllocator, GPR, NEON},
3 location::{Multiplier, Reg},
4 machine::{Label, Offset},
5};
6use crate::{codegen_error, common_decl::Size, location::Location as AbstractLocation};
7use dynasm::dynasm;
8pub use dynasmrt::aarch64::{encode_logical_immediate_32bit, encode_logical_immediate_64bit};
9use dynasmrt::{
10 AssemblyOffset, DynamicLabel, DynasmApi, DynasmLabelApi, VecAssembler,
11 aarch64::Aarch64Relocation,
12};
13use wasmer_compiler::types::{
14 function::FunctionBody,
15 section::{CustomSection, CustomSectionProtection, SectionBody},
16};
17use wasmer_types::{
18 CompileError, FunctionIndex, FunctionType, Type, VMOffsets, target::CallingConvention,
19};
20
21type Assembler = VecAssembler<Aarch64Relocation>;
22
23macro_rules! dynasm {
28 ($a:expr_2021 ; $($tt:tt)*) => {
29 dynasm::dynasm!(
30 $a
31 ; .arch aarch64
32 ; $($tt)*
33 )
34 };
35}
36
37pub type Location = AbstractLocation<GPR, NEON>;
38
39#[derive(Copy, Clone, Debug, Eq, PartialEq)]
40#[allow(dead_code)]
41#[repr(u8)]
42pub enum Condition {
43 Eq = 0,
46 Ne = 1,
48 Cs = 2,
50 Cc = 3,
52 Mi = 4,
54 Pl = 5,
56 Vs = 6,
58 Vc = 7,
60 Hi = 8,
62 Ls = 9,
64 Ge = 10,
66 Lt = 11,
68 Gt = 12,
70 Le = 13,
72 Al = 14,
74}
75
76#[derive(Copy, Clone, Debug, Eq, PartialEq)]
77#[allow(dead_code, clippy::upper_case_acronyms)]
78pub enum NeonOrMemory {
79 NEON(NEON),
80 Memory(GPR, i32),
81}
82
83#[derive(Copy, Clone, Debug)]
84#[allow(dead_code, clippy::upper_case_acronyms)]
85pub enum GPROrMemory {
86 GPR(GPR),
87 Memory(GPR, i32),
88}
89
90#[allow(unused)]
91pub trait EmitterARM64 {
92 fn get_label(&mut self) -> Label;
93 fn get_offset(&self) -> Offset;
94 fn get_jmp_instr_size(&self) -> u8;
95
96 fn finalize_function(&mut self);
97
98 fn emit_str(&mut self, sz: Size, reg: Location, addr: Location) -> Result<(), CompileError>;
99 fn emit_ldr(&mut self, sz: Size, reg: Location, addr: Location) -> Result<(), CompileError>;
100 fn emit_stur(
101 &mut self,
102 sz: Size,
103 reg: Location,
104 addr: GPR,
105 offset: i32,
106 ) -> Result<(), CompileError>;
107 fn emit_ldur(
108 &mut self,
109 sz: Size,
110 reg: Location,
111 addr: GPR,
112 offset: i32,
113 ) -> Result<(), CompileError>;
114 fn emit_strdb(
115 &mut self,
116 sz: Size,
117 reg: Location,
118 addr: GPR,
119 offset: u32,
120 ) -> Result<(), CompileError>;
121 fn emit_stria(
122 &mut self,
123 sz: Size,
124 reg: Location,
125 addr: GPR,
126 offset: u32,
127 ) -> Result<(), CompileError>;
128 fn emit_ldria(
129 &mut self,
130 sz: Size,
131 reg: Location,
132 addr: GPR,
133 offset: u32,
134 ) -> Result<(), CompileError>;
135 fn emit_stpdb(
136 &mut self,
137 sz: Size,
138 reg1: Location,
139 reg2: Location,
140 addr: GPR,
141 offset: u32,
142 ) -> Result<(), CompileError>;
143 fn emit_ldpia(
144 &mut self,
145 sz: Size,
146 reg1: Location,
147 reg2: Location,
148 addr: GPR,
149 offset: u32,
150 ) -> Result<(), CompileError>;
151
152 fn emit_ldrb(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
153 fn emit_ldrh(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
154 fn emit_ldrsb(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
155 fn emit_ldrsh(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
156 fn emit_ldrsw(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
157 fn emit_strb(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
158 fn emit_strh(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
159
160 fn emit_ldaxr(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
161 fn emit_ldaxrb(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
162 fn emit_ldaxrh(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
163 fn emit_stlxr(
164 &mut self,
165 sz: Size,
166 status: Location,
167 reg: Location,
168 dst: Location,
169 ) -> Result<(), CompileError>;
170 fn emit_stlxrb(
171 &mut self,
172 sz: Size,
173 status: Location,
174 reg: Location,
175 dst: Location,
176 ) -> Result<(), CompileError>;
177 fn emit_stlxrh(
178 &mut self,
179 sz: Size,
180 status: Location,
181 reg: Location,
182 dst: Location,
183 ) -> Result<(), CompileError>;
184
185 fn emit_mov(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
186
187 fn emit_movn(&mut self, sz: Size, reg: Location, val: u32) -> Result<(), CompileError>;
188 fn emit_movz(&mut self, reg: Location, val: u32) -> Result<(), CompileError>;
189 fn emit_movk(&mut self, reg: Location, val: u32, shift: u32) -> Result<(), CompileError>;
190
191 fn emit_mov_imm(&mut self, dst: Location, val: u64) -> Result<(), CompileError>;
192
193 fn emit_add(
194 &mut self,
195 sz: Size,
196 src1: Location,
197 src2: Location,
198 dst: Location,
199 ) -> Result<(), CompileError>;
200 fn emit_sub(
201 &mut self,
202 sz: Size,
203 src1: Location,
204 src2: Location,
205 dst: Location,
206 ) -> Result<(), CompileError>;
207 fn emit_mul(
208 &mut self,
209 sz: Size,
210 src1: Location,
211 src2: Location,
212 dst: Location,
213 ) -> Result<(), CompileError>;
214 fn emit_adds(
215 &mut self,
216 sz: Size,
217 src1: Location,
218 src2: Location,
219 dst: Location,
220 ) -> Result<(), CompileError>;
221 fn emit_subs(
222 &mut self,
223 sz: Size,
224 src1: Location,
225 src2: Location,
226 dst: Location,
227 ) -> Result<(), CompileError>;
228
229 fn emit_add_lsl(
230 &mut self,
231 sz: Size,
232 src1: Location,
233 src2: Location,
234 lsl: u32,
235 dst: Location,
236 ) -> Result<(), CompileError>;
237
238 fn emit_cmp(&mut self, sz: Size, left: Location, right: Location) -> Result<(), CompileError>;
239 fn emit_tst(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
240
241 fn emit_lsl(
242 &mut self,
243 sz: Size,
244 src1: Location,
245 src2: Location,
246 dst: Location,
247 ) -> Result<(), CompileError>;
248 fn emit_lsr(
249 &mut self,
250 sz: Size,
251 src1: Location,
252 src2: Location,
253 dst: Location,
254 ) -> Result<(), CompileError>;
255 fn emit_asr(
256 &mut self,
257 sz: Size,
258 src1: Location,
259 src2: Location,
260 dst: Location,
261 ) -> Result<(), CompileError>;
262 fn emit_ror(
263 &mut self,
264 sz: Size,
265 src1: Location,
266 src2: Location,
267 dst: Location,
268 ) -> Result<(), CompileError>;
269
270 fn emit_or(
271 &mut self,
272 sz: Size,
273 src1: Location,
274 src2: Location,
275 dst: Location,
276 ) -> Result<(), CompileError>;
277 fn emit_and(
278 &mut self,
279 sz: Size,
280 src1: Location,
281 src2: Location,
282 dst: Location,
283 ) -> Result<(), CompileError>;
284 fn emit_eor(
285 &mut self,
286 sz: Size,
287 src1: Location,
288 src2: Location,
289 dst: Location,
290 ) -> Result<(), CompileError>;
291
292 fn emit_bfc(
293 &mut self,
294 se: Size,
295 lsb: u32,
296 width: u32,
297 dst: Location,
298 ) -> Result<(), CompileError>;
299 fn emit_bfi(
300 &mut self,
301 se: Size,
302 src: Location,
303 lsb: u32,
304 width: u32,
305 dst: Location,
306 ) -> Result<(), CompileError>;
307
308 fn emit_udiv(
309 &mut self,
310 sz: Size,
311 src1: Location,
312 src2: Location,
313 dst: Location,
314 ) -> Result<(), CompileError>;
315 fn emit_sdiv(
316 &mut self,
317 sz: Size,
318 src1: Location,
319 src2: Location,
320 dst: Location,
321 ) -> Result<(), CompileError>;
322 fn emit_msub(
324 &mut self,
325 sz: Size,
326 a: Location,
327 b: Location,
328 c: Location,
329 dst: Location,
330 ) -> Result<(), CompileError>;
331
332 fn emit_sxtb(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
333 fn emit_sxth(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
334 fn emit_sxtw(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
335 fn emit_uxtb(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
336 fn emit_uxth(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
337
338 fn emit_cset(&mut self, sz: Size, dst: Location, cond: Condition) -> Result<(), CompileError>;
339 fn emit_csetm(&mut self, sz: Size, dst: Location, cond: Condition) -> Result<(), CompileError>;
340 fn emit_cinc(
341 &mut self,
342 sz: Size,
343 src: Location,
344 dst: Location,
345 cond: Condition,
346 ) -> Result<(), CompileError>;
347 fn emit_clz(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
348 fn emit_rbit(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
349
350 fn emit_label(&mut self, label: Label) -> Result<(), CompileError>;
351 fn emit_load_label(&mut self, reg: GPR, label: Label) -> Result<(), CompileError>;
352 fn emit_b_label(&mut self, label: Label) -> Result<(), CompileError>;
353 fn emit_cbz_label(&mut self, sz: Size, reg: Location, label: Label)
354 -> Result<(), CompileError>;
355 fn emit_cbnz_label(
356 &mut self,
357 sz: Size,
358 reg: Location,
359 label: Label,
360 ) -> Result<(), CompileError>;
361 fn emit_cbz_label_far(
362 &mut self,
363 sz: Size,
364 reg: Location,
365 label: Label,
366 ) -> Result<(), CompileError>;
367 fn emit_tbz_label(
368 &mut self,
369 sz: Size,
370 reg: Location,
371 n: u32,
372 label: Label,
373 ) -> Result<(), CompileError>;
374 fn emit_tbnz_label(
375 &mut self,
376 sz: Size,
377 reg: Location,
378 n: u32,
379 label: Label,
380 ) -> Result<(), CompileError>;
381 fn emit_bcond_label(&mut self, condition: Condition, label: Label) -> Result<(), CompileError>;
382 fn emit_bcond_label_far(
383 &mut self,
384 condition: Condition,
385 label: Label,
386 ) -> Result<(), CompileError>;
387 fn emit_b_register(&mut self, reg: GPR) -> Result<(), CompileError>;
388 fn emit_call_label(&mut self, label: Label) -> Result<(), CompileError>;
389 fn emit_call_register(&mut self, reg: GPR) -> Result<(), CompileError>;
390 fn emit_ret(&mut self) -> Result<(), CompileError>;
391
392 fn emit_udf(&mut self, payload: u16) -> Result<(), CompileError>;
393 fn emit_dmb(&mut self) -> Result<(), CompileError>;
394 fn emit_brk(&mut self) -> Result<(), CompileError>;
395
396 fn emit_fcmp(&mut self, sz: Size, src1: Location, src2: Location) -> Result<(), CompileError>;
397 fn emit_fneg(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
398 fn emit_fsqrt(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
399
400 fn emit_fadd(
401 &mut self,
402 sz: Size,
403 src1: Location,
404 src2: Location,
405 dst: Location,
406 ) -> Result<(), CompileError>;
407 fn emit_fsub(
408 &mut self,
409 sz: Size,
410 src1: Location,
411 src2: Location,
412 dst: Location,
413 ) -> Result<(), CompileError>;
414 fn emit_fmul(
415 &mut self,
416 sz: Size,
417 src1: Location,
418 src2: Location,
419 dst: Location,
420 ) -> Result<(), CompileError>;
421 fn emit_fdiv(
422 &mut self,
423 sz: Size,
424 src1: Location,
425 src2: Location,
426 dst: Location,
427 ) -> Result<(), CompileError>;
428
429 fn emit_fmin(
430 &mut self,
431 sz: Size,
432 src1: Location,
433 src2: Location,
434 dst: Location,
435 ) -> Result<(), CompileError>;
436 fn emit_fmax(
437 &mut self,
438 sz: Size,
439 src1: Location,
440 src2: Location,
441 dst: Location,
442 ) -> Result<(), CompileError>;
443
444 fn emit_frintz(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
445 fn emit_frintn(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
446 fn emit_frintm(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
447 fn emit_frintp(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
448
449 fn emit_scvtf(
450 &mut self,
451 sz_in: Size,
452 src: Location,
453 sz_out: Size,
454 dst: Location,
455 ) -> Result<(), CompileError>;
456 fn emit_ucvtf(
457 &mut self,
458 sz_in: Size,
459 src: Location,
460 sz_out: Size,
461 dst: Location,
462 ) -> Result<(), CompileError>;
463 fn emit_fcvt(&mut self, sz_in: Size, src: Location, dst: Location) -> Result<(), CompileError>;
464 fn emit_fcvtzs(
465 &mut self,
466 sz_in: Size,
467 src: Location,
468 sz_out: Size,
469 dst: Location,
470 ) -> Result<(), CompileError>;
471 fn emit_fcvtzu(
472 &mut self,
473 sz_in: Size,
474 src: Location,
475 sz_out: Size,
476 dst: Location,
477 ) -> Result<(), CompileError>;
478
479 fn emit_fmov(
480 &mut self,
481 src_size: Size,
482 src: Location,
483 dst_size: Size,
484 dst: Location,
485 ) -> Result<(), CompileError>;
486 fn emit_cnt(&mut self, src: NEON, dst: NEON) -> Result<(), CompileError>;
487 fn emit_addv(&mut self, src: NEON, dst: NEON) -> Result<(), CompileError>;
488 fn emit_read_fpcr(&mut self, reg: GPR) -> Result<(), CompileError>;
489 fn emit_write_fpcr(&mut self, reg: GPR) -> Result<(), CompileError>;
490 fn emit_read_fpsr(&mut self, reg: GPR) -> Result<(), CompileError>;
491 fn emit_write_fpsr(&mut self, reg: GPR) -> Result<(), CompileError>;
492
493 fn arch_supports_canonicalize_nan(&self) -> bool {
494 true
495 }
496
497 fn arch_requires_indirect_call_trampoline(&self) -> bool {
498 false
499 }
500
501 fn arch_emit_indirect_call_with_trampoline(
502 &mut self,
503 _loc: Location,
504 ) -> Result<(), CompileError> {
505 codegen_error!("singlepass arch_emit_indirect_call_with_trampoline unimplemented")
506 }
507}
508
509impl EmitterARM64 for Assembler {
510 fn get_label(&mut self) -> DynamicLabel {
511 self.new_dynamic_label()
512 }
513
514 fn get_offset(&self) -> AssemblyOffset {
515 self.offset()
516 }
517
518 fn get_jmp_instr_size(&self) -> u8 {
519 4 }
521
522 fn finalize_function(&mut self) {
523 dynasm!(
524 self
525 ; const_neg_one_32:
526 ; .i32 -1
527 ; const_zero_32:
528 ; .i32 0
529 ; const_pos_one_32:
530 ; .i32 1
531 );
532 }
533
534 fn emit_str(&mut self, sz: Size, reg: Location, addr: Location) -> Result<(), CompileError> {
535 match (sz, reg, addr) {
536 (Size::S64, Location::GPR(reg), Location::Memory(addr, disp)) => {
537 let disp = disp as u32;
538 assert!((disp & 0x7) == 0 && (disp < 0x8000));
539 dynasm!(self ; str X(reg), [X(addr), disp]);
540 }
541 (Size::S32, Location::GPR(reg), Location::Memory(addr, disp)) => {
542 let disp = disp as u32;
543 assert!((disp & 0x3) == 0 && (disp < 0x4000));
544 dynasm!(self ; str W(reg), [X(addr), disp]);
545 }
546 (Size::S16, Location::GPR(reg), Location::Memory(addr, disp)) => {
547 let disp = disp as u32;
548 assert!((disp & 0x1) == 0 && (disp < 0x2000));
549 dynasm!(self ; strh W(reg), [X(addr), disp]);
550 }
551 (Size::S8, Location::GPR(reg), Location::Memory(addr, disp)) => {
552 let disp = disp as u32;
553 assert!(disp < 0x1000);
554 dynasm!(self ; strb W(reg), [X(addr), disp]);
555 }
556 (Size::S64, Location::SIMD(reg), Location::Memory(addr, disp)) => {
557 let disp = disp as u32;
558 assert!((disp & 0x7) == 0 && (disp < 0x8000));
559 dynasm!(self ; str D(reg), [X(addr), disp]);
560 }
561 (Size::S32, Location::SIMD(reg), Location::Memory(addr, disp)) => {
562 let disp = disp as u32;
563 assert!((disp & 0x3) == 0 && (disp < 0x4000));
564 dynasm!(self ; str S(reg), [X(addr), disp]);
565 }
566 (Size::S64, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
567 assert!(offs == 0);
568 let mult = mult as u32;
569 match mult {
570 0 => dynasm!(self ; str X(reg), [X(addr)]),
571 1 => dynasm!(self ; str X(reg), [X(addr), X(r2)]),
572 _ => dynasm!(self ; str X(reg), [X(addr), X(r2), LSL mult]),
573 };
574 }
575 (Size::S32, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
576 assert!(offs == 0);
577 let mult = mult as u32;
578 match mult {
579 0 => dynasm!(self ; str W(reg), [X(addr)]),
580 1 => dynasm!(self ; str W(reg), [X(addr), X(r2)]),
581 _ => dynasm!(self ; str W(reg), [X(addr), X(r2), LSL mult]),
582 };
583 }
584 _ => codegen_error!("singlepass can't emit STR {:?}, {:?}, {:?}", sz, reg, addr),
585 }
586 Ok(())
587 }
588 fn emit_ldr(&mut self, sz: Size, reg: Location, addr: Location) -> Result<(), CompileError> {
589 match (sz, reg, addr) {
590 (Size::S64, Location::GPR(reg), Location::Memory(addr, disp)) => {
591 assert!((disp & 0x7) == 0 && (disp < 0x8000));
592 let disp = disp as u32;
593 dynasm!(self ; ldr X(reg), [X(addr), disp]);
594 }
595 (Size::S32, Location::GPR(reg), Location::Memory(addr, disp)) => {
596 assert!((disp & 0x3) == 0 && (disp < 0x4000));
597 let disp = disp as u32;
598 dynasm!(self ; ldr W(reg), [X(addr), disp]);
599 }
600 (Size::S16, Location::GPR(reg), Location::Memory(addr, disp)) => {
601 assert!((disp & 0x1 == 0) && (disp < 0x2000));
602 let disp = disp as u32;
603 dynasm!(self ; ldrh W(reg), [X(addr), disp]);
604 }
605 (Size::S8, Location::GPR(reg), Location::Memory(addr, disp)) => {
606 assert!(disp < 0x1000);
607 let disp = disp as u32;
608 dynasm!(self ; ldrb W(reg), [X(addr), disp]);
609 }
610 (Size::S64, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
611 assert!(offs == 0);
612 let mult = mult as u32;
613 match mult {
614 0 => dynasm!(self ; ldr X(reg), [X(addr)]),
615 1 => dynasm!(self ; ldr X(reg), [X(addr), X(r2)]),
616 _ => dynasm!(self ; ldr X(reg), [X(addr), X(r2), LSL mult]),
617 };
618 }
619 (Size::S32, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
620 assert!(offs == 0);
621 let mult = mult as u32;
622 match mult {
623 0 => dynasm!(self ; ldr W(reg), [X(addr)]),
624 1 => dynasm!(self ; ldr W(reg), [X(addr), X(r2)]),
625 _ => dynasm!(self ; ldr W(reg), [X(addr), X(r2), LSL mult]),
626 };
627 }
628 (Size::S64, Location::SIMD(reg), Location::Memory(addr, disp)) => {
629 let disp = disp as u32;
630 assert!((disp & 0x7) == 0 && (disp < 0x8000));
631 dynasm!(self ; ldr D(reg), [X(addr), disp]);
632 }
633 (Size::S32, Location::SIMD(reg), Location::Memory(addr, disp)) => {
634 let disp = disp as u32;
635 assert!((disp & 0x3) == 0 && (disp < 0x4000));
636 dynasm!(self ; ldr S(reg), [X(addr), disp]);
637 }
638 (Size::S64, Location::SIMD(reg), Location::Memory2(addr, r2, mult, offs)) => {
639 assert!(offs == 0);
640 let mult = mult as u32;
641 match mult {
642 0 => dynasm!(self ; ldr D(reg), [X(addr)]),
643 1 => dynasm!(self ; ldr D(reg), [X(addr), X(r2)]),
644 _ => dynasm!(self ; ldr D(reg), [X(addr), X(r2), LSL mult]),
645 };
646 }
647 (Size::S32, Location::SIMD(reg), Location::Memory2(addr, r2, mult, offs)) => {
648 assert!(offs == 0);
649 let mult = mult as u32;
650 match mult {
651 0 => dynasm!(self ; ldr S(reg), [X(addr)]),
652 1 => dynasm!(self ; ldr S(reg), [X(addr), X(r2)]),
653 _ => dynasm!(self ; ldr S(reg), [X(addr), X(r2), LSL mult]),
654 };
655 }
656 _ => codegen_error!("singlepass can't emit LDR {:?}, {:?}, {:?}", sz, reg, addr),
657 }
658 Ok(())
659 }
660 fn emit_stur(
661 &mut self,
662 sz: Size,
663 reg: Location,
664 addr: GPR,
665 offset: i32,
666 ) -> Result<(), CompileError> {
667 assert!((-255..=255).contains(&offset));
668 match (sz, reg) {
669 (Size::S64, Location::GPR(reg)) => {
670 dynasm!(self ; stur X(reg), [X(addr), offset]);
671 }
672 (Size::S32, Location::GPR(reg)) => {
673 dynasm!(self ; stur W(reg), [X(addr), offset]);
674 }
675 (Size::S64, Location::SIMD(reg)) => {
676 dynasm!(self ; stur D(reg), [X(addr), offset]);
677 }
678 (Size::S32, Location::SIMD(reg)) => {
679 dynasm!(self ; stur S(reg), [X(addr), offset]);
680 }
681 _ => codegen_error!(
682 "singlepass can't emit STUR {:?}, {:?}, {:?}, {:?}",
683 sz,
684 reg,
685 addr,
686 offset
687 ),
688 }
689 Ok(())
690 }
691 fn emit_ldur(
692 &mut self,
693 sz: Size,
694 reg: Location,
695 addr: GPR,
696 offset: i32,
697 ) -> Result<(), CompileError> {
698 assert!((-255..=255).contains(&offset));
699 match (sz, reg) {
700 (Size::S64, Location::GPR(reg)) => {
701 dynasm!(self ; ldur X(reg), [X(addr), offset]);
702 }
703 (Size::S32, Location::GPR(reg)) => {
704 dynasm!(self ; ldur W(reg), [X(addr), offset]);
705 }
706 (Size::S64, Location::SIMD(reg)) => {
707 dynasm!(self ; ldur D(reg), [X(addr), offset]);
708 }
709 (Size::S32, Location::SIMD(reg)) => {
710 dynasm!(self ; ldur S(reg), [X(addr), offset]);
711 }
712 _ => codegen_error!(
713 "singlepass can't emit LDUR {:?}, {:?}, {:?}, {:?}",
714 sz,
715 reg,
716 addr,
717 offset
718 ),
719 }
720 Ok(())
721 }
722
723 fn emit_strdb(
724 &mut self,
725 sz: Size,
726 reg: Location,
727 addr: GPR,
728 offset: u32,
729 ) -> Result<(), CompileError> {
730 assert!(offset <= 255);
731 match (sz, reg) {
732 (Size::S64, Location::GPR(reg)) => {
733 dynasm!(self ; str X(reg), [X(addr), -(offset as i32)]!);
734 }
735 (Size::S64, Location::SIMD(reg)) => {
736 dynasm!(self ; str D(reg), [X(addr), -(offset as i32)]!);
737 }
738 _ => codegen_error!("singlepass can't emit STRDB"),
739 }
740 Ok(())
741 }
742 fn emit_stria(
743 &mut self,
744 sz: Size,
745 reg: Location,
746 addr: GPR,
747 offset: u32,
748 ) -> Result<(), CompileError> {
749 assert!(offset <= 255);
750 match (sz, reg) {
751 (Size::S64, Location::GPR(reg)) => {
752 dynasm!(self ; str X(reg), [X(addr)], offset as i32);
753 }
754 (Size::S64, Location::SIMD(reg)) => {
755 dynasm!(self ; str D(reg), [X(addr)], offset as i32);
756 }
757 _ => codegen_error!("singlepass can't emit STRIA"),
758 }
759 Ok(())
760 }
761 fn emit_ldria(
762 &mut self,
763 sz: Size,
764 reg: Location,
765 addr: GPR,
766 offset: u32,
767 ) -> Result<(), CompileError> {
768 assert!(offset <= 255);
769 match (sz, reg) {
770 (Size::S64, Location::GPR(reg)) => {
771 dynasm!(self ; ldr X(reg), [X(addr)], offset as _);
772 }
773 (Size::S64, Location::SIMD(reg)) => {
774 dynasm!(self ; ldr D(reg), [X(addr)], offset as _);
775 }
776 _ => codegen_error!("singlepass can't emit LDRIA"),
777 }
778 Ok(())
779 }
780
781 fn emit_stpdb(
782 &mut self,
783 sz: Size,
784 reg1: Location,
785 reg2: Location,
786 addr: GPR,
787 offset: u32,
788 ) -> Result<(), CompileError> {
789 assert!(offset <= 255);
790 match (sz, reg1, reg2) {
791 (Size::S64, Location::GPR(reg1), Location::GPR(reg2)) => {
792 dynasm!(self ; stp X(reg1), X(reg2), [X(addr), -(offset as i32)]!);
793 }
794 _ => codegen_error!("singlepass can't emit STPDB"),
795 }
796 Ok(())
797 }
798 fn emit_ldpia(
799 &mut self,
800 sz: Size,
801 reg1: Location,
802 reg2: Location,
803 addr: GPR,
804 offset: u32,
805 ) -> Result<(), CompileError> {
806 assert!(offset <= 255);
807 match (sz, reg1, reg2) {
808 (Size::S64, Location::GPR(reg1), Location::GPR(reg2)) => {
809 dynasm!(self ; ldp X(reg1), X(reg2), [X(addr)], offset as _);
810 }
811 _ => codegen_error!("singlepass can't emit LDPIA"),
812 }
813 Ok(())
814 }
815
816 fn emit_ldrb(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
817 match (reg, dst) {
818 (Location::GPR(reg), Location::Memory(addr, offset)) => {
819 let offset = offset as u32;
820 assert!(offset < 0x1000);
821 dynasm!(self ; ldrb W(reg), [X(addr), offset]);
822 }
823 (Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
824 assert!(offs == 0);
825 let mult = mult as u32;
826 match mult {
827 0 => dynasm!(self ; ldrb W(reg), [X(addr)]),
828 1 => dynasm!(self ; ldrb W(reg), [X(addr), X(r2)]),
829 _ => dynasm!(self ; ldrb W(reg), [X(addr), X(r2), LSL mult]),
830 };
831 }
832 _ => codegen_error!("singlepass can't emit LDRB {:?}, {:?}", reg, dst),
833 }
834 Ok(())
835 }
836 fn emit_ldrh(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
837 match (reg, dst) {
838 (Location::GPR(reg), Location::Memory(addr, offset)) => {
839 let offset = offset as u32;
840 assert!((offset & 1 == 0) && (offset < 0x2000));
841 dynasm!(self ; ldrh W(reg), [X(addr), offset]);
842 }
843 (Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
844 assert!(offs == 0);
845 let mult = mult as u32;
846 match mult {
847 0 => dynasm!(self ; ldrh W(reg), [X(addr)]),
848 1 => dynasm!(self ; ldrh W(reg), [X(addr), X(r2)]),
849 _ => dynasm!(self ; ldrh W(reg), [X(addr), X(r2), LSL mult]),
850 };
851 }
852 _ => codegen_error!("singlepass can't emit LDRH {:?}, {:?}", reg, dst),
853 }
854 Ok(())
855 }
856 fn emit_ldrsb(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
857 match (sz, reg, dst) {
858 (Size::S64, Location::GPR(reg), Location::Memory(addr, offset)) => {
859 let offset = offset as u32;
860 assert!(offset < 0x1000);
861 dynasm!(self ; ldrsb X(reg), [X(addr), offset]);
862 }
863 (Size::S32, Location::GPR(reg), Location::Memory(addr, offset)) => {
864 let offset = offset as u32;
865 assert!(offset < 0x1000);
866 dynasm!(self ; ldrsb W(reg), [X(addr), offset]);
867 }
868 (Size::S64, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
869 assert!(offs == 0);
870 let mult = mult as u32;
871 match mult {
872 0 => dynasm!(self ; ldrsb X(reg), [X(addr)]),
873 1 => dynasm!(self ; ldrsb X(reg), [X(addr), X(r2)]),
874 _ => dynasm!(self ; ldrsb X(reg), [X(addr), X(r2), LSL mult]),
875 };
876 }
877 (Size::S32, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
878 assert!(offs == 0);
879 let mult = mult as u32;
880 match mult {
881 0 => dynasm!(self ; ldrsb W(reg), [X(addr)]),
882 1 => dynasm!(self ; ldrsb W(reg), [X(addr), X(r2)]),
883 _ => dynasm!(self ; ldrsb W(reg), [X(addr), X(r2), LSL mult]),
884 };
885 }
886 _ => codegen_error!("singlepass can't emit LDRSB {:?}, {:?}, {:?}", sz, reg, dst),
887 }
888 Ok(())
889 }
890 fn emit_ldrsh(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
891 match (sz, reg, dst) {
892 (Size::S64, Location::GPR(reg), Location::Memory(addr, offset)) => {
893 let offset = offset as u32;
894 assert!((offset & 1 == 0) && (offset < 0x2000));
895 dynasm!(self ; ldrsh X(reg), [X(addr), offset]);
896 }
897 (Size::S32, Location::GPR(reg), Location::Memory(addr, offset)) => {
898 let offset = offset as u32;
899 assert!((offset & 1 == 0) && (offset < 0x2000));
900 dynasm!(self ; ldrsh W(reg), [X(addr), offset]);
901 }
902 (Size::S64, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
903 assert!(offs == 0);
904 let mult = mult as u32;
905 match mult {
906 0 => dynasm!(self ; ldrsh X(reg), [X(addr)]),
907 1 => dynasm!(self ; ldrsh X(reg), [X(addr), X(r2)]),
908 _ => dynasm!(self ; ldrsh X(reg), [X(addr), X(r2), LSL mult]),
909 };
910 }
911 (Size::S32, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
912 assert!(offs == 0);
913 let mult = mult as u32;
914 match mult {
915 0 => dynasm!(self ; ldrsh W(reg), [X(addr)]),
916 1 => dynasm!(self ; ldrsh W(reg), [X(addr), X(r2)]),
917 _ => dynasm!(self ; ldrsh W(reg), [X(addr), X(r2), LSL mult]),
918 };
919 }
920 _ => codegen_error!("singlepass can't emit LDRSH {:?}, {:?}, {:?}", sz, reg, dst),
921 }
922 Ok(())
923 }
924 fn emit_ldrsw(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
925 match (sz, reg, dst) {
926 (Size::S64, Location::GPR(reg), Location::Memory(addr, offset)) => {
927 let offset = offset as u32;
928 assert!((offset & 3 == 0) && (offset < 0x4000));
929 dynasm!(self ; ldrsw X(reg), [X(addr), offset]);
930 }
931 (Size::S64, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
932 assert!(offs == 0);
933 let mult = mult as u32;
934 match mult {
935 0 => dynasm!(self ; ldrsw X(reg), [X(addr)]),
936 1 => dynasm!(self ; ldrsw X(reg), [X(addr), X(r2)]),
937 _ => dynasm!(self ; ldrsw X(reg), [X(addr), X(r2), LSL mult]),
938 };
939 }
940 _ => codegen_error!("singlepass can't emit LDRSW {:?}, {:?}, {:?}", sz, reg, dst),
941 }
942 Ok(())
943 }
944 fn emit_strb(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
945 match (reg, dst) {
946 (Location::GPR(reg), Location::Memory(addr, offset)) => {
947 let offset = offset as u32;
948 assert!(offset < 0x1000);
949 dynasm!(self ; strb W(reg), [X(addr), offset]);
950 }
951 (Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
952 assert!(offs == 0);
953 let mult = mult as u32;
954 match mult {
955 0 => dynasm!(self ; strb W(reg), [X(addr)]),
956 1 => dynasm!(self ; strb W(reg), [X(addr), X(r2)]),
957 _ => dynasm!(self ; strb W(reg), [X(addr), X(r2), LSL mult]),
958 };
959 }
960 _ => codegen_error!("singlepass can't emit STRB {:?}, {:?}", reg, dst),
961 }
962 Ok(())
963 }
964 fn emit_strh(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
965 match (reg, dst) {
966 (Location::GPR(reg), Location::Memory(addr, offset)) => {
967 let offset = offset as u32;
968 assert!((offset & 1 == 0) && (offset < 0x2000));
969 dynasm!(self ; strh W(reg), [X(addr), offset]);
970 }
971 (Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
972 assert!(offs == 0);
973 let mult = mult as u32;
974 match mult {
975 0 => dynasm!(self ; strh W(reg), [X(addr)]),
976 1 => dynasm!(self ; strh W(reg), [X(addr), X(r2)]),
977 _ => dynasm!(self ; strh W(reg), [X(addr), X(r2), LSL mult]),
978 };
979 }
980 _ => codegen_error!("singlepass can't emit STRH {:?}, {:?}", reg, dst),
981 }
982 Ok(())
983 }
984
985 fn emit_ldaxr(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
986 match (sz, reg, dst) {
987 (Size::S32, Location::GPR(reg), Location::GPR(dst)) => {
988 dynasm!(self ; ldaxr W(reg), [X(dst)]);
989 }
990 (Size::S64, Location::GPR(reg), Location::GPR(dst)) => {
991 dynasm!(self ; ldaxr X(reg), [X(dst)]);
992 }
993 _ => codegen_error!("singlepass can't emit LDAXR {:?}, {:?}", reg, dst),
994 }
995 Ok(())
996 }
997 fn emit_ldaxrb(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
998 match (reg, dst) {
999 (Location::GPR(reg), Location::GPR(dst)) => {
1000 dynasm!(self ; ldaxrb W(reg), [X(dst)]);
1001 }
1002 _ => codegen_error!("singlepass can't emit LDAXRB {:?}, {:?}", reg, dst),
1003 }
1004 Ok(())
1005 }
1006 fn emit_ldaxrh(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
1007 match (reg, dst) {
1008 (Location::GPR(reg), Location::GPR(dst)) => {
1009 dynasm!(self ; ldaxrh W(reg), [X(dst)]);
1010 }
1011 _ => codegen_error!("singlepass can't emit LDAXRH {:?}, {:?}", reg, dst),
1012 }
1013 Ok(())
1014 }
1015 fn emit_stlxr(
1016 &mut self,
1017 sz: Size,
1018 status: Location,
1019 reg: Location,
1020 dst: Location,
1021 ) -> Result<(), CompileError> {
1022 match (sz, status, reg, dst) {
1023 (Size::S32, Location::GPR(status), Location::GPR(reg), Location::GPR(dst)) => {
1024 dynasm!(self ; stlxr W(status), W(reg), [X(dst)]);
1025 }
1026 (Size::S64, Location::GPR(status), Location::GPR(reg), Location::GPR(dst)) => {
1027 dynasm!(self ; stlxr W(status), X(reg), [X(dst)]);
1028 }
1029 _ => codegen_error!("singlepass can't emit STLXR {:?}, {:?}", reg, dst),
1030 }
1031 Ok(())
1032 }
1033 fn emit_stlxrb(
1034 &mut self,
1035 _sz: Size,
1036 status: Location,
1037 reg: Location,
1038 dst: Location,
1039 ) -> Result<(), CompileError> {
1040 match (status, reg, dst) {
1041 (Location::GPR(status), Location::GPR(reg), Location::GPR(dst)) => {
1042 dynasm!(self ; stlxrb W(status), W(reg), [X(dst)]);
1043 }
1044 _ => codegen_error!("singlepass can't emit STLXRB {:?}, {:?}", reg, dst),
1045 }
1046 Ok(())
1047 }
1048 fn emit_stlxrh(
1049 &mut self,
1050 _sz: Size,
1051 status: Location,
1052 reg: Location,
1053 dst: Location,
1054 ) -> Result<(), CompileError> {
1055 match (status, reg, dst) {
1056 (Location::GPR(status), Location::GPR(reg), Location::GPR(dst)) => {
1057 dynasm!(self ; stlxrh W(status), W(reg), [X(dst)]);
1058 }
1059 _ => codegen_error!("singlepass can't emit STLXRH {:?}, {:?}", reg, dst),
1060 }
1061 Ok(())
1062 }
1063
1064 fn emit_mov(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1065 match (sz, src, dst) {
1066 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1067 dynasm!(self ; mov X(dst), X(src));
1068 }
1069 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1070 dynasm!(self ; mov W(dst), W(src));
1071 }
1072 (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
1073 dynasm!(self ; mov V(dst).D[0], V(src).D[0]);
1074 }
1075 (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
1076 dynasm!(self ; mov V(dst).S[0], V(src).S[0]);
1077 }
1078 (Size::S64, Location::GPR(src), Location::SIMD(dst)) => {
1079 dynasm!(self ; mov V(dst).D[0], X(src));
1080 }
1081 (Size::S32, Location::GPR(src), Location::SIMD(dst)) => {
1082 dynasm!(self ; mov V(dst).S[0], W(src));
1083 }
1084 (Size::S64, Location::SIMD(src), Location::GPR(dst)) => {
1085 dynasm!(self ; mov X(dst), V(src).D[0]);
1086 }
1087 (Size::S32, Location::SIMD(src), Location::GPR(dst)) => {
1088 dynasm!(self ; mov W(dst), V(src).S[0]);
1089 }
1090 (Size::S32, Location::Imm32(val), Location::GPR(dst)) => {
1091 if val < 0x1000 || encode_logical_immediate_32bit(val as _).is_some() {
1092 dynasm!(self ; mov W(dst), val);
1093 } else {
1094 dynasm!(self
1095 ; movz W(dst), val
1096 ; movk W(dst), val, lsl #16);
1097 }
1098 }
1099 (Size::S64, Location::Imm32(val), Location::GPR(dst)) => {
1100 if val < 0x1000 {
1101 dynasm!(self ; mov W(dst), val);
1102 } else if encode_logical_immediate_64bit(val as _).is_some() {
1103 dynasm!(self ; mov X(dst), val as _);
1104 } else {
1105 dynasm!(self
1106 ; movz X(dst), val
1107 ; movk X(dst), val, lsl #16);
1108 }
1109 }
1110 (Size::S64, Location::Imm64(val), Location::GPR(dst)) => {
1111 if val < 0x1000 || encode_logical_immediate_64bit(val as _).is_some() {
1112 dynasm!(self ; mov X(dst), val as _);
1113 } else {
1114 dynasm!(self
1115 ; movz X(dst), val as _
1116 ; movk X(dst), val as _, lsl #16
1117 ; movk X(dst), val as _, lsl #32
1118 ; movk X(dst), val as _, lsl #48);
1119 }
1120 }
1121 _ => codegen_error!("singlepass can't emit MOV {:?}, {:?}, {:?}", sz, src, dst),
1122 }
1123 Ok(())
1124 }
1125
1126 fn emit_movn(&mut self, sz: Size, reg: Location, val: u32) -> Result<(), CompileError> {
1127 match (sz, reg) {
1128 (Size::S32, Location::GPR(reg)) => {
1129 dynasm!(self ; movn W(reg), val);
1130 }
1131 (Size::S64, Location::GPR(reg)) => {
1132 dynasm!(self ; movn X(reg), val);
1133 }
1134 _ => codegen_error!("singlepass can't emit MOVN"),
1135 }
1136 Ok(())
1137 }
1138 fn emit_movz(&mut self, reg: Location, val: u32) -> Result<(), CompileError> {
1139 match reg {
1140 Location::GPR(reg) => {
1141 dynasm!(self ; movz W(reg), val);
1142 }
1143 _ => codegen_error!("singlepass can't emit MOVZ"),
1144 }
1145 Ok(())
1146 }
1147 fn emit_movk(&mut self, reg: Location, val: u32, shift: u32) -> Result<(), CompileError> {
1148 match reg {
1149 Location::GPR(reg) => {
1150 dynasm!(self ; movk X(reg), val, LSL shift);
1151 }
1152 _ => codegen_error!("singlepass can't emit MOVK"),
1153 }
1154 Ok(())
1155 }
1156
1157 fn emit_mov_imm(&mut self, dst: Location, val: u64) -> Result<(), CompileError> {
1158 match dst {
1159 Location::GPR(dst) => {
1160 let offset = val.trailing_zeros() & 48;
1161 let masked = 0xffff & (val >> offset);
1162 if (masked << offset) == val {
1163 dynasm!(self ; movz X(dst), masked as u32, LSL offset);
1164 } else if val >> 16 == 0xffff_ffff_ffff {
1165 let val: u16 = !((val & 0xffff) as u16);
1166 dynasm!(self ; movn X(dst), val as u32);
1167 } else if val >> 16 == 0xffff {
1168 let val: u16 = !((val & 0xffff) as u16);
1169 dynasm!(self ; movn W(dst), val as u32);
1170 } else {
1171 dynasm!(self ; movz W(dst), (val&0xffff) as u32);
1172 let val = val >> 16;
1173 if val != 0 {
1174 dynasm!(self ; movk X(dst), (val&0xffff) as u32, LSL 16);
1175 let val = val >> 16;
1176 if val != 0 {
1177 dynasm!(self ; movk X(dst), (val&0xffff) as u32, LSL 32);
1178 let val = val >> 16;
1179 if val != 0 {
1180 dynasm!(self ; movk X(dst), (val&0xffff) as u32, LSL 48);
1181 }
1182 }
1183 }
1184 }
1185 }
1186 _ => codegen_error!("singlepass can't emit MOVW {:?}", dst),
1187 }
1188 Ok(())
1189 }
1190
1191 fn emit_add(
1192 &mut self,
1193 sz: Size,
1194 src1: Location,
1195 src2: Location,
1196 dst: Location,
1197 ) -> Result<(), CompileError> {
1198 match (sz, src1, src2, dst) {
1199 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1200 dynasm!(self ; add XSP(dst), XSP(src1), X(src2), UXTX);
1201 }
1202 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1203 dynasm!(self ; add WSP(dst), WSP(src1), W(src2), UXTX);
1204 }
1205 (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1206 | (Size::S64, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1207 dynasm!(self ; add XSP(dst), XSP(src1), imm as _);
1208 }
1209 (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
1210 | (Size::S64, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
1211 if imm >= 0x1000 {
1212 unreachable!();
1213 }
1214 dynasm!(self ; add XSP(dst), XSP(src1), imm);
1215 }
1216 (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst))
1217 | (Size::S64, Location::Imm64(imm), Location::GPR(src1), Location::GPR(dst)) => {
1218 if imm >= 0x1000 {
1219 unreachable!();
1220 }
1221 let imm = imm as u32;
1222 dynasm!(self ; add XSP(dst), XSP(src1), imm);
1223 }
1224 (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1225 | (Size::S32, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1226 dynasm!(self ; add WSP(dst), WSP(src1), imm as u32);
1227 }
1228 (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
1229 | (Size::S32, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
1230 if imm >= 0x1000 {
1231 unreachable!();
1232 }
1233 dynasm!(self ; add WSP(dst), WSP(src1), imm);
1234 }
1235 _ => codegen_error!(
1236 "singlepass can't emit ADD {:?} {:?} {:?} {:?}",
1237 sz,
1238 src1,
1239 src2,
1240 dst
1241 ),
1242 }
1243 Ok(())
1244 }
1245 fn emit_sub(
1246 &mut self,
1247 sz: Size,
1248 src1: Location,
1249 src2: Location,
1250 dst: Location,
1251 ) -> Result<(), CompileError> {
1252 match (sz, src1, src2, dst) {
1253 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1254 dynasm!(self; sub XSP(dst), XSP(src1), X(src2), UXTX 0);
1255 }
1256 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1257 dynasm!(self ; sub WSP(dst), WSP(src1), W(src2), UXTX 0);
1258 }
1259 (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1260 dynasm!(self ; sub XSP(dst), XSP(src1), imm as u32);
1261 }
1262 (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1263 dynasm!(self ; sub WSP(dst), WSP(src1), imm as u32);
1264 }
1265 (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1266 if imm >= 0x1000 {
1267 unreachable!();
1268 }
1269 dynasm!(self ; sub WSP(dst), WSP(src1), imm);
1270 }
1271 (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1272 if imm >= 0x1000 {
1273 unreachable!();
1274 }
1275 dynasm!(self ; sub XSP(dst), XSP(src1), imm);
1276 }
1277 (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
1278 if imm >= 0x1000 {
1279 unreachable!();
1280 }
1281 dynasm!(self ; sub XSP(dst), XSP(src1), imm as u32);
1282 }
1283 _ => codegen_error!(
1284 "singlepass can't emit SUB {:?} {:?} {:?} {:?}",
1285 sz,
1286 src1,
1287 src2,
1288 dst
1289 ),
1290 }
1291 Ok(())
1292 }
1293 fn emit_mul(
1294 &mut self,
1295 sz: Size,
1296 src1: Location,
1297 src2: Location,
1298 dst: Location,
1299 ) -> Result<(), CompileError> {
1300 match (sz, src1, src2, dst) {
1301 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1302 dynasm!(self ; mul X(dst), X(src1), X(src2));
1303 }
1304 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1305 dynasm!(self ; mul W(dst), W(src1), W(src2));
1306 }
1307 _ => codegen_error!(
1308 "singlepass can't emit MUL {:?} {:?} {:?} {:?}",
1309 sz,
1310 src1,
1311 src2,
1312 dst
1313 ),
1314 }
1315 Ok(())
1316 }
1317 fn emit_adds(
1318 &mut self,
1319 sz: Size,
1320 src1: Location,
1321 src2: Location,
1322 dst: Location,
1323 ) -> Result<(), CompileError> {
1324 match (sz, src1, src2, dst) {
1325 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1326 dynasm!(self ; adds X(dst), X(src1), X(src2));
1327 }
1328 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1329 dynasm!(self ; adds W(dst), W(src1), W(src2));
1330 }
1331 (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1332 | (Size::S64, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1333 dynasm!(self ; adds X(dst), XSP(src1), imm as u32);
1334 }
1335 (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
1336 | (Size::S64, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
1337 if imm >= 0x1000 {
1338 codegen_error!("singlepass ADD.S with imm too large {}", imm);
1339 }
1340 dynasm!(self ; adds X(dst), XSP(src1), imm);
1341 }
1342 (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1343 | (Size::S32, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1344 dynasm!(self ; adds W(dst), WSP(src1), imm as u32);
1345 }
1346 (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
1347 | (Size::S32, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
1348 if imm >= 0x1000 {
1349 codegen_error!("singlepass ADD.S with imm too large {}", imm);
1350 }
1351 dynasm!(self ; adds W(dst), WSP(src1), imm);
1352 }
1353 _ => codegen_error!(
1354 "singlepass can't emit ADD.S {:?} {:?} {:?} {:?}",
1355 sz,
1356 src1,
1357 src2,
1358 dst
1359 ),
1360 }
1361 Ok(())
1362 }
1363 fn emit_subs(
1364 &mut self,
1365 sz: Size,
1366 src1: Location,
1367 src2: Location,
1368 dst: Location,
1369 ) -> Result<(), CompileError> {
1370 match (sz, src1, src2, dst) {
1371 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1372 dynasm!(self ; subs X(dst), X(src1), X(src2));
1373 }
1374 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1375 dynasm!(self ; subs W(dst), W(src1), W(src2));
1376 }
1377 (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1378 dynasm!(self ; subs X(dst), XSP(src1), imm as u32);
1379 }
1380 (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1381 dynasm!(self ; subs W(dst), WSP(src1), imm as u32);
1382 }
1383 _ => codegen_error!(
1384 "singlepass can't emit SUB.S {:?} {:?} {:?} {:?}",
1385 sz,
1386 src1,
1387 src2,
1388 dst
1389 ),
1390 }
1391 Ok(())
1392 }
1393 fn emit_add_lsl(
1394 &mut self,
1395 sz: Size,
1396 src1: Location,
1397 src2: Location,
1398 lsl: u32,
1399 dst: Location,
1400 ) -> Result<(), CompileError> {
1401 match (sz, src1, src2, dst) {
1402 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1403 dynasm!(self ; add X(dst), X(src1), X(src2), LSL lsl);
1404 }
1405 _ => codegen_error!(
1406 "singlepass can't emit LSL {:?} {:?} {:?} {:?} LSL {:?}",
1407 sz,
1408 src1,
1409 src2,
1410 dst,
1411 lsl
1412 ),
1413 }
1414 Ok(())
1415 }
1416
1417 fn emit_cmp(&mut self, sz: Size, left: Location, right: Location) -> Result<(), CompileError> {
1423 match (sz, left, right) {
1424 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1425 dynasm!(self ; cmp X(dst), X(src));
1426 }
1427 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1428 dynasm!(self ; cmp W(dst), W(src));
1429 }
1430 (Size::S64, Location::Imm8(imm), Location::GPR(dst)) => {
1431 dynasm!(self ; cmp XSP(dst), imm as u32);
1432 }
1433 (Size::S64, Location::Imm32(imm), Location::GPR(dst)) => {
1434 if imm >= 0x1000 {
1435 codegen_error!("singlepass CMP with imm too large {}", imm);
1436 }
1437 dynasm!(self ; cmp XSP(dst), imm as u32);
1438 }
1439 (Size::S64, Location::Imm64(imm), Location::GPR(dst)) => {
1440 if imm >= 0x1000 {
1441 codegen_error!("singlepass CMP with imm too large {}", imm);
1442 }
1443 dynasm!(self ; cmp XSP(dst), imm as u32);
1444 }
1445 (Size::S32, Location::Imm8(imm), Location::GPR(dst)) => {
1446 dynasm!(self ; cmp WSP(dst), imm as u32);
1447 }
1448 (Size::S32, Location::Imm32(imm), Location::GPR(dst)) => {
1449 if imm >= 0x1000 {
1450 codegen_error!("singlepass CMP with imm too large {}", imm);
1451 }
1452 dynasm!(self ; cmp WSP(dst), imm as u32);
1453 }
1454 _ => codegen_error!("singlepass can't emit CMP {:?} {:?} {:?}", sz, left, right),
1455 }
1456 Ok(())
1457 }
1458
1459 fn emit_tst(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1460 match (sz, src, dst) {
1461 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1462 dynasm!(self ; tst X(dst), X(src));
1463 }
1464 (Size::S64, Location::Imm32(imm), Location::GPR(dst)) => {
1465 if encode_logical_immediate_64bit(imm as u64).is_none() {
1466 codegen_error!("singlepass TST with incompatible imm {}", imm);
1467 }
1468 dynasm!(self ; tst X(dst), imm as u64);
1469 }
1470 (Size::S64, Location::Imm64(imm), Location::GPR(dst)) => {
1471 if encode_logical_immediate_64bit(imm as u64).is_none() {
1472 codegen_error!("singlepass TST with incompatible imm {}", imm);
1473 }
1474 dynasm!(self ; tst X(dst), imm as u64);
1475 }
1476 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1477 dynasm!(self ; tst W(dst), W(src));
1478 }
1479 (Size::S32, Location::Imm32(imm), Location::GPR(dst)) => {
1480 if encode_logical_immediate_64bit(imm as u64).is_none() {
1481 codegen_error!("singlepass TST with incompatible imm {}", imm);
1482 }
1483 dynasm!(self ; tst W(dst), imm);
1484 }
1485 _ => codegen_error!("singlepass can't emit TST"),
1486 }
1487 Ok(())
1488 }
1489
1490 fn emit_lsl(
1491 &mut self,
1492 sz: Size,
1493 src1: Location,
1494 src2: Location,
1495 dst: Location,
1496 ) -> Result<(), CompileError> {
1497 match (sz, src1, src2, dst) {
1498 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1499 dynasm!(self ; lsl X(dst), X(src1), X(src2));
1500 }
1501 (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1502 if imm > 63 {
1503 codegen_error!("singlepass LSL with incompatible imm {}", imm);
1504 }
1505 let imm = imm as u32;
1506
1507 dynasm!(self ; lsl X(dst), X(src1), imm);
1508 }
1509 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1510 dynasm!(self ; lsl W(dst), W(src1), W(src2));
1511 }
1512 (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1513 if imm > 63 {
1514 codegen_error!("singlepass LSL with incompatible imm {}", imm);
1515 }
1516 dynasm!(self ; lsl X(dst), X(src1), imm as u32);
1517 }
1518 (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
1519 if imm > 63 {
1520 codegen_error!("singlepass LSL with incompatible imm {}", imm);
1521 }
1522 dynasm!(self ; lsl X(dst), X(src1), imm as u32);
1523 }
1524 (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1525 if imm > 31 {
1526 codegen_error!("singlepass LSL with incompatible imm {}", imm);
1527 }
1528 dynasm!(self ; lsl W(dst), W(src1), imm as u32);
1529 }
1530 (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1531 if imm > 31 {
1532 codegen_error!("singlepass LSL with incompatible imm {}", imm);
1533 }
1534 dynasm!(self ; lsl W(dst), W(src1), imm as u32);
1535 }
1536 _ => codegen_error!(
1537 "singlepass can't emit LSL {:?} {:?} {:?} {:?}",
1538 sz,
1539 src1,
1540 src2,
1541 dst
1542 ),
1543 }
1544 Ok(())
1545 }
1546 fn emit_asr(
1547 &mut self,
1548 sz: Size,
1549 src1: Location,
1550 src2: Location,
1551 dst: Location,
1552 ) -> Result<(), CompileError> {
1553 match (sz, src1, src2, dst) {
1554 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1555 dynasm!(self ; asr X(dst), X(src1), X(src2));
1556 }
1557 (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1558 let imm = imm as u32;
1559
1560 if imm == 0 || imm > 63 {
1561 codegen_error!("singlepass ASR with incompatible imm {}", imm);
1562 }
1563 dynasm!(self ; asr X(dst), X(src1), imm);
1564 }
1565 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1566 dynasm!(self ; asr W(dst), W(src1), W(src2));
1567 }
1568 (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1569 if imm == 0 || imm > 63 {
1570 codegen_error!("singlepass ASR with incompatible imm {}", imm);
1571 }
1572 dynasm!(self ; asr X(dst), X(src1), imm as u32);
1573 }
1574 (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
1575 if imm == 0 || imm > 63 {
1576 codegen_error!("singlepass ASR with incompatible imm {}", imm);
1577 }
1578 dynasm!(self ; asr X(dst), X(src1), imm as u32);
1579 }
1580 (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1581 if imm == 0 || imm > 31 {
1582 codegen_error!("singlepass ASR with incompatible imm {}", imm);
1583 }
1584 dynasm!(self ; asr W(dst), W(src1), imm as u32);
1585 }
1586 (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1587 if imm == 0 || imm > 31 {
1588 codegen_error!("singlepass ASR with incompatible imm {}", imm);
1589 }
1590 dynasm!(self ; asr W(dst), W(src1), imm as u32);
1591 }
1592 _ => codegen_error!(
1593 "singlepass can't emit ASR {:?} {:?} {:?} {:?}",
1594 sz,
1595 src1,
1596 src2,
1597 dst
1598 ),
1599 }
1600 Ok(())
1601 }
1602 fn emit_lsr(
1603 &mut self,
1604 sz: Size,
1605 src1: Location,
1606 src2: Location,
1607 dst: Location,
1608 ) -> Result<(), CompileError> {
1609 match (sz, src1, src2, dst) {
1610 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1611 dynasm!(self ; lsr X(dst), X(src1), X(src2));
1612 }
1613 (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1614 let imm = imm as u32;
1615
1616 if imm == 0 || imm > 63 {
1617 codegen_error!("singlepass LSR with incompatible imm {}", imm);
1618 }
1619 dynasm!(self ; lsr X(dst), X(src1), imm);
1620 }
1621 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1622 dynasm!(self ; lsr W(dst), W(src1), W(src2));
1623 }
1624 (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1625 if imm == 0 || imm > 63 {
1626 codegen_error!("singlepass LSR with incompatible imm {}", imm);
1627 }
1628 dynasm!(self ; lsr X(dst), X(src1), imm as u32);
1629 }
1630 (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
1631 if imm == 0 || imm > 63 {
1632 codegen_error!("singlepass LSR with incompatible imm {}", imm);
1633 }
1634 dynasm!(self ; lsr X(dst), X(src1), imm as u32);
1635 }
1636 (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1637 if imm == 0 || imm > 31 {
1638 codegen_error!("singlepass LSR with incompatible imm {}", imm);
1639 }
1640 dynasm!(self ; lsr W(dst), W(src1), imm as u32);
1641 }
1642 (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1643 if imm == 0 || imm > 31 {
1644 codegen_error!("singlepass LSR with incompatible imm {}", imm);
1645 }
1646 dynasm!(self ; lsr W(dst), W(src1), imm as u32);
1647 }
1648 _ => codegen_error!(
1649 "singlepass can't emit LSR {:?} {:?} {:?} {:?}",
1650 sz,
1651 src1,
1652 src2,
1653 dst
1654 ),
1655 }
1656 Ok(())
1657 }
1658 fn emit_ror(
1659 &mut self,
1660 sz: Size,
1661 src1: Location,
1662 src2: Location,
1663 dst: Location,
1664 ) -> Result<(), CompileError> {
1665 match (sz, src1, src2, dst) {
1666 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1667 dynasm!(self ; ror X(dst), X(src1), X(src2));
1668 }
1669 (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1670 let imm = imm as u32;
1671
1672 if imm == 0 || imm > 63 {
1673 codegen_error!("singlepass ROR with incompatible imm {}", imm);
1674 }
1675 dynasm!(self ; ror X(dst), X(src1), imm);
1676 }
1677 (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
1678 let imm = imm as u32;
1679
1680 if imm == 0 || imm > 63 {
1681 codegen_error!("singlepass ROR with incompatible imm {}", imm);
1682 }
1683 dynasm!(self ; ror X(dst), X(src1), imm);
1684 }
1685 (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1686 if imm == 0 || imm > 31 {
1687 codegen_error!("singlepass ROR with incompatible imm {}", imm);
1688 }
1689 dynasm!(self ; ror W(dst), W(src1), imm as u32);
1690 }
1691 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1692 dynasm!(self ; ror W(dst), W(src1), W(src2));
1693 }
1694 (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1695 if imm == 0 || imm > 63 {
1696 codegen_error!("singlepass ROR with incompatible imm {}", imm);
1697 }
1698 dynasm!(self ; ror X(dst), X(src1), imm as u32);
1699 }
1700 (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1701 if imm == 0 || imm > 31 {
1702 codegen_error!("singlepass ROR with incompatible imm {}", imm);
1703 }
1704 dynasm!(self ; ror W(dst), W(src1), imm as u32);
1705 }
1706 _ => codegen_error!(
1707 "singlepass can't emit ROR {:?} {:?} {:?} {:?}",
1708 sz,
1709 src1,
1710 src2,
1711 dst
1712 ),
1713 }
1714 Ok(())
1715 }
1716
1717 fn emit_or(
1718 &mut self,
1719 sz: Size,
1720 src1: Location,
1721 src2: Location,
1722 dst: Location,
1723 ) -> Result<(), CompileError> {
1724 match (sz, src1, src2, dst) {
1725 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1726 dynasm!(self ; orr X(dst), X(src1), X(src2));
1727 }
1728 (Size::S64, Location::GPR(src1), Location::Imm64(src2), Location::GPR(dst)) => {
1729 let src2 = src2 as u64;
1730
1731 if encode_logical_immediate_64bit(src2 as u64).is_none() {
1732 codegen_error!("singlepass OR with incompatible imm {}", src2);
1733 }
1734 dynasm!(self ; orr XSP(dst), X(src1), src2);
1735 }
1736 (Size::S32, Location::GPR(src1), Location::Imm32(src2), Location::GPR(dst)) => {
1737 let src2 = src2 as u32;
1738
1739 if encode_logical_immediate_32bit(src2).is_none() {
1740 codegen_error!("singlepass OR with incompatible imm {}", src2);
1741 }
1742 dynasm!(self ; orr WSP(dst), W(src1), src2);
1743 }
1744 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1745 dynasm!(self ; orr W(dst), W(src1), W(src2));
1746 }
1747 _ => codegen_error!(
1748 "singlepass can't emit OR {:?} {:?} {:?} {:?}",
1749 sz,
1750 src1,
1751 src2,
1752 dst
1753 ),
1754 }
1755 Ok(())
1756 }
1757 fn emit_and(
1758 &mut self,
1759 sz: Size,
1760 src1: Location,
1761 src2: Location,
1762 dst: Location,
1763 ) -> Result<(), CompileError> {
1764 match (sz, src1, src2, dst) {
1765 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1766 dynasm!(self ; and X(dst), X(src1), X(src2));
1767 }
1768 (Size::S64, Location::GPR(src1), Location::Imm64(src2), Location::GPR(dst)) => {
1769 let src2 = src2 as u64;
1770
1771 if encode_logical_immediate_64bit(src2 as u64).is_none() {
1772 codegen_error!("singlepass AND with incompatible imm {}", src2);
1773 }
1774 dynasm!(self ; and XSP(dst), X(src1), src2);
1775 }
1776 (Size::S32, Location::GPR(src1), Location::Imm32(src2), Location::GPR(dst)) => {
1777 let src2 = src2 as u32;
1778
1779 if encode_logical_immediate_32bit(src2).is_none() {
1780 codegen_error!("singlepass AND with incompatible imm {}", src2);
1781 }
1782 dynasm!(self ; and WSP(dst), W(src1), src2);
1783 }
1784 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1785 dynasm!(self ; and W(dst), W(src1), W(src2));
1786 }
1787 _ => codegen_error!(
1788 "singlepass can't emit AND {:?} {:?} {:?} {:?}",
1789 sz,
1790 src1,
1791 src2,
1792 dst
1793 ),
1794 }
1795 Ok(())
1796 }
1797 fn emit_eor(
1798 &mut self,
1799 sz: Size,
1800 src1: Location,
1801 src2: Location,
1802 dst: Location,
1803 ) -> Result<(), CompileError> {
1804 match (sz, src1, src2, dst) {
1805 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1806 dynasm!(self ; eor X(dst), X(src1), X(src2));
1807 }
1808 (Size::S64, Location::GPR(src1), Location::Imm64(src2), Location::GPR(dst)) => {
1809 let src2 = src2 as u64;
1810
1811 if encode_logical_immediate_64bit(src2 as u64).is_none() {
1812 codegen_error!("singlepass EOR with incompatible imm {}", src2);
1813 }
1814 dynasm!(self ; eor XSP(dst), X(src1), src2);
1815 }
1816 (Size::S32, Location::GPR(src1), Location::Imm32(src2), Location::GPR(dst)) => {
1817 let src2 = src2 as u32;
1818
1819 if encode_logical_immediate_32bit(src2).is_none() {
1820 codegen_error!("singlepass EOR with incompatible imm {}", src2);
1821 }
1822 dynasm!(self ; eor WSP(dst), W(src1), src2);
1823 }
1824 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1825 dynasm!(self ; eor W(dst), W(src1), W(src2));
1826 }
1827 _ => codegen_error!(
1828 "singlepass can't emit EOR {:?} {:?} {:?} {:?}",
1829 sz,
1830 src1,
1831 src2,
1832 dst
1833 ),
1834 }
1835 Ok(())
1836 }
1837
1838 fn emit_bfc(
1839 &mut self,
1840 sz: Size,
1841 lsb: u32,
1842 width: u32,
1843 dst: Location,
1844 ) -> Result<(), CompileError> {
1845 match (sz, dst) {
1846 (Size::S32, Location::GPR(dst)) => {
1847 dynasm!(self ; bfc W(dst), lsb, width);
1848 }
1849 (Size::S64, Location::GPR(dst)) => {
1850 dynasm!(self ; bfc X(dst), lsb, width);
1851 }
1852 _ => codegen_error!("singlepass can't emit BFC"),
1853 }
1854 Ok(())
1855 }
1856 fn emit_bfi(
1857 &mut self,
1858 sz: Size,
1859 src: Location,
1860 lsb: u32,
1861 width: u32,
1862 dst: Location,
1863 ) -> Result<(), CompileError> {
1864 match (sz, src, dst) {
1865 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1866 dynasm!(self ; bfi W(dst), W(src), lsb, width);
1867 }
1868 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1869 dynasm!(self ; bfi X(dst), X(src), lsb, width);
1870 }
1871 _ => codegen_error!("singlepass can't emit BFI"),
1872 }
1873 Ok(())
1874 }
1875
1876 fn emit_udiv(
1877 &mut self,
1878 sz: Size,
1879 src1: Location,
1880 src2: Location,
1881 dst: Location,
1882 ) -> Result<(), CompileError> {
1883 match (sz, src1, src2, dst) {
1884 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1885 dynasm!(self ; udiv W(dst), W(src1), W(src2));
1886 }
1887 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1888 dynasm!(self ; udiv X(dst), X(src1), X(src2));
1889 }
1890 _ => codegen_error!(
1891 "singlepass can't emit UDIV {:?} {:?} {:?} {:?}",
1892 sz,
1893 src1,
1894 src2,
1895 dst
1896 ),
1897 }
1898 Ok(())
1899 }
1900 fn emit_sdiv(
1901 &mut self,
1902 sz: Size,
1903 src1: Location,
1904 src2: Location,
1905 dst: Location,
1906 ) -> Result<(), CompileError> {
1907 match (sz, src1, src2, dst) {
1908 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1909 dynasm!(self ; sdiv W(dst), W(src1), W(src2));
1910 }
1911 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1912 dynasm!(self ; sdiv X(dst), X(src1), X(src2));
1913 }
1914 _ => codegen_error!(
1915 "singlepass can't emit SDIV {:?} {:?} {:?} {:?}",
1916 sz,
1917 src1,
1918 src2,
1919 dst
1920 ),
1921 }
1922 Ok(())
1923 }
1924
1925 fn emit_msub(
1927 &mut self,
1928 sz: Size,
1929 a: Location,
1930 b: Location,
1931 c: Location,
1932 dst: Location,
1933 ) -> Result<(), CompileError> {
1934 match (sz, a, b, c, dst) {
1935 (
1936 Size::S32,
1937 Location::GPR(a),
1938 Location::GPR(b),
1939 Location::GPR(c),
1940 Location::GPR(dst),
1941 ) => {
1942 dynasm!(self ; msub W(dst), W(a), W(b), W(c));
1943 }
1944 (
1945 Size::S64,
1946 Location::GPR(a),
1947 Location::GPR(b),
1948 Location::GPR(c),
1949 Location::GPR(dst),
1950 ) => {
1951 dynasm!(self ; msub X(dst), X(a), X(b), X(c));
1952 }
1953 _ => codegen_error!(
1954 "singlepass can't emit msub {:?} {:?} {:?} {:?} {:?}",
1955 sz,
1956 a,
1957 b,
1958 c,
1959 dst
1960 ),
1961 }
1962 Ok(())
1963 }
1964
1965 fn emit_sxtb(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1966 match (sz, src, dst) {
1967 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1968 dynasm!(self ; sxtb W(dst), W(src));
1969 }
1970 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1971 dynasm!(self ; sxtb X(dst), W(src));
1972 }
1973 _ => codegen_error!("singlepass can't emit SXTB {:?} {:?} {:?}", sz, src, dst),
1974 }
1975 Ok(())
1976 }
1977 fn emit_sxth(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1978 match (sz, src, dst) {
1979 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1980 dynasm!(self ; sxth W(dst), W(src));
1981 }
1982 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1983 dynasm!(self ; sxth X(dst), W(src));
1984 }
1985 _ => codegen_error!("singlepass can't emit SXTH {:?} {:?} {:?}", sz, src, dst),
1986 }
1987 Ok(())
1988 }
1989 fn emit_sxtw(&mut self, _sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1990 match (src, dst) {
1991 (Location::GPR(src), Location::GPR(dst)) => {
1992 dynasm!(self ; sxtw X(dst), W(src));
1993 }
1994 _ => codegen_error!("singlepass can't emit SXTW {:?} {:?}", src, dst),
1995 }
1996 Ok(())
1997 }
1998 fn emit_uxtb(&mut self, _sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1999 match (src, dst) {
2000 (Location::GPR(src), Location::GPR(dst)) => {
2001 dynasm!(self ; uxtb W(dst), W(src));
2002 }
2003 _ => codegen_error!("singlepass can't emit UXTB {:?} {:?}", src, dst),
2004 }
2005 Ok(())
2006 }
2007 fn emit_uxth(&mut self, _sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2008 match (src, dst) {
2009 (Location::GPR(src), Location::GPR(dst)) => {
2010 dynasm!(self ; uxth W(dst), W(src));
2011 }
2012 _ => codegen_error!("singlepass can't emit UXTH {:?} {:?}", src, dst),
2013 }
2014 Ok(())
2015 }
2016
2017 fn emit_cset(&mut self, sz: Size, dst: Location, cond: Condition) -> Result<(), CompileError> {
2018 match (sz, dst) {
2019 (Size::S32, Location::GPR(reg)) => match cond {
2020 Condition::Eq => dynasm!(self ; cset W(reg), eq),
2021 Condition::Ne => dynasm!(self ; cset W(reg), ne),
2022 Condition::Cs => dynasm!(self ; cset W(reg), cs),
2023 Condition::Cc => dynasm!(self ; cset W(reg), cc),
2024 Condition::Mi => dynasm!(self ; cset W(reg), mi),
2025 Condition::Pl => dynasm!(self ; cset W(reg), pl),
2026 Condition::Vs => dynasm!(self ; cset W(reg), vs),
2027 Condition::Vc => dynasm!(self ; cset W(reg), vc),
2028 Condition::Hi => dynasm!(self ; cset W(reg), hi),
2029 Condition::Ls => dynasm!(self ; cset W(reg), ls),
2030 Condition::Ge => dynasm!(self ; cset W(reg), ge),
2031 Condition::Lt => dynasm!(self ; cset W(reg), lt),
2032 Condition::Gt => dynasm!(self ; cset W(reg), gt),
2033 Condition::Le => dynasm!(self ; cset W(reg), le),
2034 Condition::Al => dynasm!(self ; cset W(reg), al),
2035 },
2036 (Size::S64, Location::GPR(reg)) => match cond {
2037 Condition::Eq => dynasm!(self ; cset X(reg), eq),
2038 Condition::Ne => dynasm!(self ; cset X(reg), ne),
2039 Condition::Cs => dynasm!(self ; cset X(reg), cs),
2040 Condition::Cc => dynasm!(self ; cset X(reg), cc),
2041 Condition::Mi => dynasm!(self ; cset X(reg), mi),
2042 Condition::Pl => dynasm!(self ; cset X(reg), pl),
2043 Condition::Vs => dynasm!(self ; cset X(reg), vs),
2044 Condition::Vc => dynasm!(self ; cset X(reg), vc),
2045 Condition::Hi => dynasm!(self ; cset X(reg), hi),
2046 Condition::Ls => dynasm!(self ; cset X(reg), ls),
2047 Condition::Ge => dynasm!(self ; cset X(reg), ge),
2048 Condition::Lt => dynasm!(self ; cset X(reg), lt),
2049 Condition::Gt => dynasm!(self ; cset X(reg), gt),
2050 Condition::Le => dynasm!(self ; cset X(reg), le),
2051 Condition::Al => dynasm!(self ; cset X(reg), al),
2052 },
2053 _ => codegen_error!("singlepass can't emit CSET {:?} {:?} {:?}", sz, dst, cond),
2054 }
2055 Ok(())
2056 }
2057 fn emit_csetm(&mut self, sz: Size, dst: Location, cond: Condition) -> Result<(), CompileError> {
2058 match (sz, dst) {
2059 (Size::S32, Location::GPR(reg)) => match cond {
2060 Condition::Eq => dynasm!(self ; csetm W(reg), eq),
2061 Condition::Ne => dynasm!(self ; csetm W(reg), ne),
2062 Condition::Cs => dynasm!(self ; csetm W(reg), cs),
2063 Condition::Cc => dynasm!(self ; csetm W(reg), cc),
2064 Condition::Mi => dynasm!(self ; csetm W(reg), mi),
2065 Condition::Pl => dynasm!(self ; csetm W(reg), pl),
2066 Condition::Vs => dynasm!(self ; csetm W(reg), vs),
2067 Condition::Vc => dynasm!(self ; csetm W(reg), vc),
2068 Condition::Hi => dynasm!(self ; csetm W(reg), hi),
2069 Condition::Ls => dynasm!(self ; csetm W(reg), ls),
2070 Condition::Ge => dynasm!(self ; csetm W(reg), ge),
2071 Condition::Lt => dynasm!(self ; csetm W(reg), lt),
2072 Condition::Gt => dynasm!(self ; csetm W(reg), gt),
2073 Condition::Le => dynasm!(self ; csetm W(reg), le),
2074 Condition::Al => dynasm!(self ; csetm W(reg), al),
2075 },
2076 (Size::S64, Location::GPR(reg)) => match cond {
2077 Condition::Eq => dynasm!(self ; csetm X(reg), eq),
2078 Condition::Ne => dynasm!(self ; csetm X(reg), ne),
2079 Condition::Cs => dynasm!(self ; csetm X(reg), cs),
2080 Condition::Cc => dynasm!(self ; csetm X(reg), cc),
2081 Condition::Mi => dynasm!(self ; csetm X(reg), mi),
2082 Condition::Pl => dynasm!(self ; csetm X(reg), pl),
2083 Condition::Vs => dynasm!(self ; csetm X(reg), vs),
2084 Condition::Vc => dynasm!(self ; csetm X(reg), vc),
2085 Condition::Hi => dynasm!(self ; csetm X(reg), hi),
2086 Condition::Ls => dynasm!(self ; csetm X(reg), ls),
2087 Condition::Ge => dynasm!(self ; csetm X(reg), ge),
2088 Condition::Lt => dynasm!(self ; csetm X(reg), lt),
2089 Condition::Gt => dynasm!(self ; csetm X(reg), gt),
2090 Condition::Le => dynasm!(self ; csetm X(reg), le),
2091 Condition::Al => dynasm!(self ; csetm X(reg), al),
2092 },
2093 _ => codegen_error!("singlepass can't emit CSETM {:?} {:?} {:?}", sz, dst, cond),
2094 }
2095 Ok(())
2096 }
2097 fn emit_cinc(
2098 &mut self,
2099 sz: Size,
2100 src: Location,
2101 dst: Location,
2102 cond: Condition,
2103 ) -> Result<(), CompileError> {
2104 match (sz, src, dst) {
2105 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
2106 match cond {
2107 Condition::Eq => dynasm!(self ; cinc W(dst), W(src), eq),
2108 Condition::Ne => dynasm!(self ; cinc W(dst), W(src), ne),
2109 Condition::Cs => dynasm!(self ; cinc W(dst), W(src), cs),
2110 Condition::Cc => dynasm!(self ; cinc W(dst), W(src), cc),
2111 Condition::Mi => dynasm!(self ; cinc W(dst), W(src), mi),
2112 Condition::Pl => dynasm!(self ; cinc W(dst), W(src), pl),
2113 Condition::Vs => dynasm!(self ; cinc W(dst), W(src), vs),
2114 Condition::Vc => dynasm!(self ; cinc W(dst), W(src), vc),
2115 Condition::Hi => dynasm!(self ; cinc W(dst), W(src), hi),
2116 Condition::Ls => dynasm!(self ; cinc W(dst), W(src), ls),
2117 Condition::Ge => dynasm!(self ; cinc W(dst), W(src), ge),
2118 Condition::Lt => dynasm!(self ; cinc W(dst), W(src), lt),
2119 Condition::Gt => dynasm!(self ; cinc W(dst), W(src), gt),
2120 Condition::Le => dynasm!(self ; cinc W(dst), W(src), le),
2121 Condition::Al => dynasm!(self ; cinc W(dst), W(src), al),
2122 };
2123 }
2124 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
2125 match cond {
2126 Condition::Eq => dynasm!(self ; cinc X(src), X(dst), eq),
2127 Condition::Ne => dynasm!(self ; cinc X(src), X(dst), ne),
2128 Condition::Cs => dynasm!(self ; cinc X(src), X(dst), cs),
2129 Condition::Cc => dynasm!(self ; cinc X(src), X(dst), cc),
2130 Condition::Mi => dynasm!(self ; cinc X(src), X(dst), mi),
2131 Condition::Pl => dynasm!(self ; cinc X(src), X(dst), pl),
2132 Condition::Vs => dynasm!(self ; cinc X(src), X(dst), vs),
2133 Condition::Vc => dynasm!(self ; cinc X(src), X(dst), vc),
2134 Condition::Hi => dynasm!(self ; cinc X(src), X(dst), hi),
2135 Condition::Ls => dynasm!(self ; cinc X(src), X(dst), ls),
2136 Condition::Ge => dynasm!(self ; cinc X(src), X(dst), ge),
2137 Condition::Lt => dynasm!(self ; cinc X(src), X(dst), lt),
2138 Condition::Gt => dynasm!(self ; cinc X(src), X(dst), gt),
2139 Condition::Le => dynasm!(self ; cinc X(src), X(dst), le),
2140 Condition::Al => dynasm!(self ; cinc X(src), X(dst), al),
2141 };
2142 }
2143 _ => codegen_error!("singlepass can't emit CINC"),
2144 }
2145 Ok(())
2146 }
2147
2148 fn emit_clz(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2149 match (sz, src, dst) {
2150 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
2151 dynasm!(self ; clz X(dst), X(src));
2152 }
2153 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
2154 dynasm!(self ; clz W(dst), W(src));
2155 }
2156 _ => codegen_error!("singlepass can't emit CLS {:?} {:?} {:?}", sz, src, dst),
2157 }
2158 Ok(())
2159 }
2160 fn emit_rbit(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2161 match (sz, src, dst) {
2162 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
2163 dynasm!(self ; rbit X(dst), X(src));
2164 }
2165 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
2166 dynasm!(self ; rbit W(dst), W(src));
2167 }
2168 _ => codegen_error!("singlepass can't emit CLS {:?} {:?} {:?}", sz, src, dst),
2169 }
2170 Ok(())
2171 }
2172
2173 fn emit_label(&mut self, label: Label) -> Result<(), CompileError> {
2174 dynasm!(self ; => label);
2175 Ok(())
2176 }
2177 fn emit_load_label(&mut self, reg: GPR, label: Label) -> Result<(), CompileError> {
2178 dynasm!(self ; adr X(reg), =>label);
2179 Ok(())
2180 }
2181
2182 fn emit_b_label(&mut self, label: Label) -> Result<(), CompileError> {
2183 dynasm!(self ; b =>label);
2184 Ok(())
2185 }
2186 fn emit_cbz_label(
2187 &mut self,
2188 sz: Size,
2189 reg: Location,
2190 label: Label,
2191 ) -> Result<(), CompileError> {
2192 match (sz, reg) {
2193 (Size::S32, Location::GPR(reg)) => {
2194 dynasm!(self ; cbz W(reg), =>label);
2195 }
2196 (Size::S64, Location::GPR(reg)) => {
2197 dynasm!(self ; cbz X(reg), =>label);
2198 }
2199 _ => codegen_error!("singlepass can't emit CBZ {:?} {:?} {:?}", sz, reg, label),
2200 }
2201 Ok(())
2202 }
2203 fn emit_cbnz_label(
2204 &mut self,
2205 sz: Size,
2206 reg: Location,
2207 label: Label,
2208 ) -> Result<(), CompileError> {
2209 match (sz, reg) {
2210 (Size::S32, Location::GPR(reg)) => {
2211 dynasm!(self ; cbnz W(reg), =>label);
2212 }
2213 (Size::S64, Location::GPR(reg)) => {
2214 dynasm!(self ; cbnz X(reg), =>label);
2215 }
2216 _ => codegen_error!("singlepass can't emit CBNZ {:?} {:?} {:?}", sz, reg, label),
2217 }
2218 Ok(())
2219 }
2220 fn emit_cbz_label_far(
2221 &mut self,
2222 sz: Size,
2223 reg: Location,
2224 label: Label,
2225 ) -> Result<(), CompileError> {
2226 let near_label: Label = self.get_label();
2227 let continue_label: Label = self.get_label();
2228
2229 match (sz, reg) {
2230 (Size::S32, Location::GPR(reg)) => {
2231 dynasm!(self ; cbz W(reg), => near_label);
2232 dynasm!(self ; b => continue_label);
2233 }
2234 (Size::S64, Location::GPR(reg)) => {
2235 dynasm!(self ; cbz X(reg), => near_label);
2236 dynasm!(self ; b => continue_label);
2237 }
2238 _ => codegen_error!("singlepass can't emit CBZ {:?} {:?} {:?}", sz, reg, label),
2239 }
2240 self.emit_label(near_label)?;
2241 dynasm!(self ; b => label );
2242
2243 self.emit_label(continue_label)?;
2244
2245 Ok(())
2246 }
2247 fn emit_tbz_label(
2248 &mut self,
2249 sz: Size,
2250 reg: Location,
2251 n: u32,
2252 label: Label,
2253 ) -> Result<(), CompileError> {
2254 match (sz, reg) {
2255 (Size::S32, Location::GPR(reg)) => {
2256 dynasm!(self ; tbz W(reg), n, =>label);
2257 }
2258 (Size::S64, Location::GPR(reg)) => {
2259 dynasm!(self ; tbz X(reg), n, =>label);
2260 }
2261 _ => codegen_error!(
2262 "singlepass can't emit TBZ {:?} {:?} {:?} {:?}",
2263 sz,
2264 reg,
2265 n,
2266 label
2267 ),
2268 }
2269 Ok(())
2270 }
2271 fn emit_tbnz_label(
2272 &mut self,
2273 sz: Size,
2274 reg: Location,
2275 n: u32,
2276 label: Label,
2277 ) -> Result<(), CompileError> {
2278 match (sz, reg) {
2279 (Size::S32, Location::GPR(reg)) => {
2280 dynasm!(self ; tbnz W(reg), n, =>label);
2281 }
2282 (Size::S64, Location::GPR(reg)) => {
2283 dynasm!(self ; tbnz X(reg), n, =>label);
2284 }
2285 _ => codegen_error!(
2286 "singlepass can't emit TBNZ {:?} {:?} {:?} {:?}",
2287 sz,
2288 reg,
2289 n,
2290 label
2291 ),
2292 }
2293 Ok(())
2294 }
2295 fn emit_bcond_label(&mut self, condition: Condition, label: Label) -> Result<(), CompileError> {
2296 match condition {
2297 Condition::Eq => dynasm!(self ; b.eq => label),
2298 Condition::Ne => dynasm!(self ; b.ne => label),
2299 Condition::Cs => dynasm!(self ; b.cs => label),
2300 Condition::Cc => dynasm!(self ; b.cc => label),
2301 Condition::Mi => dynasm!(self ; b.mi => label),
2302 Condition::Pl => dynasm!(self ; b.pl => label),
2303 Condition::Vs => dynasm!(self ; b.vs => label),
2304 Condition::Vc => dynasm!(self ; b.vc => label),
2305 Condition::Hi => dynasm!(self ; b.hi => label),
2306 Condition::Ls => dynasm!(self ; b.ls => label),
2307 Condition::Ge => dynasm!(self ; b.ge => label),
2308 Condition::Lt => dynasm!(self ; b.lt => label),
2309 Condition::Gt => dynasm!(self ; b.gt => label),
2310 Condition::Le => dynasm!(self ; b.le => label),
2311 Condition::Al => dynasm!(self ; b => label),
2312 }
2313 Ok(())
2314 }
2315 fn emit_bcond_label_far(
2316 &mut self,
2317 condition: Condition,
2318 label: Label,
2319 ) -> Result<(), CompileError> {
2320 let cont: Label = self.get_label();
2321 match condition {
2322 Condition::Eq => dynasm!(self ; b.ne => cont),
2324 Condition::Ne => dynasm!(self ; b.eq => cont),
2325 Condition::Cs => dynasm!(self ; b.cc => cont),
2326 Condition::Cc => dynasm!(self ; b.cs => cont),
2327 Condition::Mi => dynasm!(self ; b.pl => cont),
2328 Condition::Pl => dynasm!(self ; b.mi => cont),
2329 Condition::Vs => dynasm!(self ; b.vc => cont),
2330 Condition::Vc => dynasm!(self ; b.vs => cont),
2331 Condition::Hi => dynasm!(self ; b.ls => cont),
2332 Condition::Ls => dynasm!(self ; b.hi => cont),
2333 Condition::Ge => dynasm!(self ; b.lt => cont),
2334 Condition::Lt => dynasm!(self ; b.ge => cont),
2335 Condition::Gt => dynasm!(self ; b.le => cont),
2336 Condition::Le => dynasm!(self ; b.gt => cont),
2337 Condition::Al => { }
2338 }
2339 dynasm!(self ; b => label);
2340 self.emit_label(cont)?;
2341 Ok(())
2342 }
2343 fn emit_b_register(&mut self, reg: GPR) -> Result<(), CompileError> {
2344 dynasm!(self ; br X(reg));
2345 Ok(())
2346 }
2347 fn emit_call_label(&mut self, label: Label) -> Result<(), CompileError> {
2348 dynasm!(self ; bl =>label);
2349 Ok(())
2350 }
2351 fn emit_call_register(&mut self, reg: GPR) -> Result<(), CompileError> {
2352 dynasm!(self ; blr X(reg));
2353 Ok(())
2354 }
2355 fn emit_ret(&mut self) -> Result<(), CompileError> {
2356 dynasm!(self ; ret);
2357 Ok(())
2358 }
2359
2360 fn emit_udf(&mut self, payload: u16) -> Result<(), CompileError> {
2361 dynasm!(self ; udf payload as u32);
2362 Ok(())
2363 }
2364 fn emit_dmb(&mut self) -> Result<(), CompileError> {
2365 dynasm!(self ; dmb ish);
2366 Ok(())
2367 }
2368 fn emit_brk(&mut self) -> Result<(), CompileError> {
2369 dynasm!(self ; brk 0);
2370 Ok(())
2371 }
2372
2373 fn emit_fcmp(&mut self, sz: Size, src1: Location, src2: Location) -> Result<(), CompileError> {
2374 match (sz, src1, src2) {
2375 (Size::S32, Location::SIMD(src1), Location::SIMD(src2)) => {
2376 dynasm!(self ; fcmp S(src1), S(src2));
2377 }
2378 (Size::S64, Location::SIMD(src1), Location::SIMD(src2)) => {
2379 dynasm!(self ; fcmp D(src1), D(src2));
2380 }
2381 _ => codegen_error!("singlepass can't emit FCMP {:?} {:?} {:?}", sz, src1, src2),
2382 }
2383 Ok(())
2384 }
2385
2386 fn emit_fneg(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2387 match (sz, src, dst) {
2388 (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2389 dynasm!(self ; fneg S(dst), S(src));
2390 }
2391 (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2392 dynasm!(self ; fneg D(dst), D(src));
2393 }
2394 _ => codegen_error!("singlepass can't emit FNEG {:?} {:?} {:?}", sz, src, dst),
2395 }
2396 Ok(())
2397 }
2398 fn emit_fsqrt(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2399 match (sz, src, dst) {
2400 (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2401 dynasm!(self ; fsqrt S(dst), S(src));
2402 }
2403 (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2404 dynasm!(self ; fsqrt D(dst), D(src));
2405 }
2406 _ => codegen_error!("singlepass can't emit FSQRT {:?} {:?} {:?}", sz, src, dst),
2407 }
2408 Ok(())
2409 }
2410
2411 fn emit_fadd(
2412 &mut self,
2413 sz: Size,
2414 src1: Location,
2415 src2: Location,
2416 dst: Location,
2417 ) -> Result<(), CompileError> {
2418 match (sz, src1, src2, dst) {
2419 (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2420 dynasm!(self ; fadd S(dst), S(src1), S(src2));
2421 }
2422 (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2423 dynasm!(self ; fadd D(dst), D(src1), D(src2));
2424 }
2425 _ => codegen_error!(
2426 "singlepass can't emit FADD {:?} {:?} {:?} {:?}",
2427 sz,
2428 src1,
2429 src2,
2430 dst
2431 ),
2432 }
2433 Ok(())
2434 }
2435 fn emit_fsub(
2436 &mut self,
2437 sz: Size,
2438 src1: Location,
2439 src2: Location,
2440 dst: Location,
2441 ) -> Result<(), CompileError> {
2442 match (sz, src1, src2, dst) {
2443 (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2444 dynasm!(self ; fsub S(dst), S(src1), S(src2));
2445 }
2446 (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2447 dynasm!(self ; fsub D(dst), D(src1), D(src2));
2448 }
2449 _ => codegen_error!(
2450 "singlepass can't emit FSUB {:?} {:?} {:?} {:?}",
2451 sz,
2452 src1,
2453 src2,
2454 dst
2455 ),
2456 }
2457 Ok(())
2458 }
2459 fn emit_fmul(
2460 &mut self,
2461 sz: Size,
2462 src1: Location,
2463 src2: Location,
2464 dst: Location,
2465 ) -> Result<(), CompileError> {
2466 match (sz, src1, src2, dst) {
2467 (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2468 dynasm!(self ; fmul S(dst), S(src1), S(src2));
2469 }
2470 (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2471 dynasm!(self ; fmul D(dst), D(src1), D(src2));
2472 }
2473 _ => codegen_error!(
2474 "singlepass can't emit FMUL {:?} {:?} {:?} {:?}",
2475 sz,
2476 src1,
2477 src2,
2478 dst
2479 ),
2480 }
2481 Ok(())
2482 }
2483 fn emit_fdiv(
2484 &mut self,
2485 sz: Size,
2486 src1: Location,
2487 src2: Location,
2488 dst: Location,
2489 ) -> Result<(), CompileError> {
2490 match (sz, src1, src2, dst) {
2491 (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2492 dynasm!(self ; fdiv S(dst), S(src1), S(src2));
2493 }
2494 (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2495 dynasm!(self ; fdiv D(dst), D(src1), D(src2));
2496 }
2497 _ => codegen_error!(
2498 "singlepass can't emit FDIV {:?} {:?} {:?} {:?}",
2499 sz,
2500 src1,
2501 src2,
2502 dst
2503 ),
2504 }
2505 Ok(())
2506 }
2507
2508 fn emit_fmin(
2509 &mut self,
2510 sz: Size,
2511 src1: Location,
2512 src2: Location,
2513 dst: Location,
2514 ) -> Result<(), CompileError> {
2515 match (sz, src1, src2, dst) {
2516 (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2517 dynasm!(self ; fmin S(dst), S(src1), S(src2));
2518 }
2519 (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2520 dynasm!(self ; fmin D(dst), D(src1), D(src2));
2521 }
2522 _ => codegen_error!(
2523 "singlepass can't emit FMIN {:?} {:?} {:?} {:?}",
2524 sz,
2525 src1,
2526 src2,
2527 dst
2528 ),
2529 }
2530 Ok(())
2531 }
2532 fn emit_fmax(
2533 &mut self,
2534 sz: Size,
2535 src1: Location,
2536 src2: Location,
2537 dst: Location,
2538 ) -> Result<(), CompileError> {
2539 match (sz, src1, src2, dst) {
2540 (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2541 dynasm!(self ; fmax S(dst), S(src1), S(src2));
2542 }
2543 (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2544 dynasm!(self ; fmax D(dst), D(src1), D(src2));
2545 }
2546 _ => codegen_error!(
2547 "singlepass can't emit FMAX {:?} {:?} {:?} {:?}",
2548 sz,
2549 src1,
2550 src2,
2551 dst
2552 ),
2553 }
2554 Ok(())
2555 }
2556
2557 fn emit_frintz(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2558 match (sz, src, dst) {
2559 (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2560 dynasm!(self ; frintz S(dst), S(src));
2561 }
2562 (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2563 dynasm!(self ; frintz D(dst), D(src));
2564 }
2565 _ => codegen_error!("singlepass can't emit FRINTZ {:?} {:?} {:?}", sz, src, dst),
2566 }
2567 Ok(())
2568 }
2569 fn emit_frintn(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2570 match (sz, src, dst) {
2571 (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2572 dynasm!(self ; frintn S(dst), S(src));
2573 }
2574 (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2575 dynasm!(self ; frintn D(dst), D(src));
2576 }
2577 _ => codegen_error!("singlepass can't emit FRINTN {:?} {:?} {:?}", sz, src, dst),
2578 }
2579 Ok(())
2580 }
2581 fn emit_frintm(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2582 match (sz, src, dst) {
2583 (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2584 dynasm!(self ; frintm S(dst), S(src));
2585 }
2586 (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2587 dynasm!(self ; frintm D(dst), D(src));
2588 }
2589 _ => codegen_error!("singlepass can't emit FRINTM {:?} {:?} {:?}", sz, src, dst),
2590 }
2591 Ok(())
2592 }
2593 fn emit_frintp(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2594 match (sz, src, dst) {
2595 (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2596 dynasm!(self ; frintp S(dst), S(src));
2597 }
2598 (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2599 dynasm!(self ; frintp D(dst), D(src));
2600 }
2601 _ => codegen_error!("singlepass can't emit FRINTP {:?} {:?} {:?}", sz, src, dst),
2602 }
2603 Ok(())
2604 }
2605
2606 fn emit_scvtf(
2607 &mut self,
2608 sz_in: Size,
2609 src: Location,
2610 sz_out: Size,
2611 dst: Location,
2612 ) -> Result<(), CompileError> {
2613 match (sz_in, src, sz_out, dst) {
2614 (Size::S32, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
2615 dynasm!(self ; scvtf S(dst), W(src));
2616 }
2617 (Size::S64, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
2618 dynasm!(self ; scvtf S(dst), X(src));
2619 }
2620 (Size::S32, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
2621 dynasm!(self ; scvtf D(dst), W(src));
2622 }
2623 (Size::S64, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
2624 dynasm!(self ; scvtf D(dst), X(src));
2625 }
2626 _ => codegen_error!(
2627 "singlepass can't emit SCVTF {:?} {:?} {:?} {:?}",
2628 sz_in,
2629 src,
2630 sz_out,
2631 dst
2632 ),
2633 }
2634 Ok(())
2635 }
2636 fn emit_ucvtf(
2637 &mut self,
2638 sz_in: Size,
2639 src: Location,
2640 sz_out: Size,
2641 dst: Location,
2642 ) -> Result<(), CompileError> {
2643 match (sz_in, src, sz_out, dst) {
2644 (Size::S32, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
2645 dynasm!(self ; ucvtf S(dst), W(src));
2646 }
2647 (Size::S64, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
2648 dynasm!(self ; ucvtf S(dst), X(src));
2649 }
2650 (Size::S32, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
2651 dynasm!(self ; ucvtf D(dst), W(src));
2652 }
2653 (Size::S64, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
2654 dynasm!(self ; ucvtf D(dst), X(src));
2655 }
2656 _ => codegen_error!(
2657 "singlepass can't emit UCVTF {:?} {:?} {:?} {:?}",
2658 sz_in,
2659 src,
2660 sz_out,
2661 dst
2662 ),
2663 }
2664 Ok(())
2665 }
2666 fn emit_fcvt(&mut self, sz_in: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2667 match (sz_in, src, dst) {
2668 (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2669 dynasm!(self ; fcvt D(dst), S(src));
2670 }
2671 (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2672 dynasm!(self ; fcvt S(dst), D(src));
2673 }
2674 _ => codegen_error!(
2675 "singlepass can't emit UCVTF {:?} {:?} {:?}",
2676 sz_in,
2677 src,
2678 dst
2679 ),
2680 }
2681 Ok(())
2682 }
2683 fn emit_fcvtzs(
2684 &mut self,
2685 sz_in: Size,
2686 src: Location,
2687 sz_out: Size,
2688 dst: Location,
2689 ) -> Result<(), CompileError> {
2690 match (sz_in, src, sz_out, dst) {
2691 (Size::S32, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
2692 dynasm!(self ; fcvtzs W(dst), S(src));
2693 }
2694 (Size::S64, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
2695 dynasm!(self ; fcvtzs W(dst), D(src));
2696 }
2697 (Size::S32, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
2698 dynasm!(self ; fcvtzs X(dst), S(src));
2699 }
2700 (Size::S64, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
2701 dynasm!(self ; fcvtzs X(dst), D(src));
2702 }
2703 _ => codegen_error!(
2704 "singlepass can't emit FCVTZS {:?} {:?} {:?} {:?}",
2705 sz_in,
2706 src,
2707 sz_out,
2708 dst
2709 ),
2710 }
2711 Ok(())
2712 }
2713 fn emit_fcvtzu(
2714 &mut self,
2715 sz_in: Size,
2716 src: Location,
2717 sz_out: Size,
2718 dst: Location,
2719 ) -> Result<(), CompileError> {
2720 match (sz_in, src, sz_out, dst) {
2721 (Size::S32, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
2722 dynasm!(self ; fcvtzu W(dst), S(src));
2723 }
2724 (Size::S64, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
2725 dynasm!(self ; fcvtzu W(dst), D(src));
2726 }
2727 (Size::S32, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
2728 dynasm!(self ; fcvtzu X(dst), S(src));
2729 }
2730 (Size::S64, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
2731 dynasm!(self ; fcvtzu X(dst), D(src));
2732 }
2733 _ => codegen_error!(
2734 "singlepass can't emit FCVTZU {:?} {:?} {:?} {:?}",
2735 sz_in,
2736 src,
2737 sz_out,
2738 dst
2739 ),
2740 }
2741 Ok(())
2742 }
2743
2744 fn emit_read_fpcr(&mut self, reg: GPR) -> Result<(), CompileError> {
2746 dynasm!(self ; mrs X(reg), 0b1_011_0100_0100_000);
2747 Ok(())
2748 }
2749 fn emit_write_fpcr(&mut self, reg: GPR) -> Result<(), CompileError> {
2750 dynasm!(self ; msr 0b1_011_0100_0100_000, X(reg));
2751 Ok(())
2752 }
2753 fn emit_read_fpsr(&mut self, reg: GPR) -> Result<(), CompileError> {
2755 dynasm!(self ; mrs X(reg), 0b1_011_0100_0100_001);
2756 Ok(())
2757 }
2758 fn emit_write_fpsr(&mut self, reg: GPR) -> Result<(), CompileError> {
2759 dynasm!(self ; msr 0b1_011_0100_0100_001, X(reg));
2760 Ok(())
2761 }
2762
2763 fn emit_cnt(&mut self, src: NEON, dst: NEON) -> Result<(), CompileError> {
2764 dynasm!(self ; cnt V(dst).B8, V(src).B8);
2765 Ok(())
2766 }
2767
2768 fn emit_addv(&mut self, src: NEON, dst: NEON) -> Result<(), CompileError> {
2769 dynasm!(self ; addv B(dst), V(src).B8);
2770 Ok(())
2771 }
2772
2773 #[allow(clippy::unit_arg)]
2774 fn emit_fmov(
2775 &mut self,
2776 src_size: Size,
2777 src: Location,
2778 dst_size: Size,
2779 dst: Location,
2780 ) -> Result<(), CompileError> {
2781 let mut err = false;
2782 match (src, dst) {
2783 (AbstractLocation::GPR(src), AbstractLocation::SIMD(dst)) => {
2784 match (src_size, dst_size) {
2785 (Size::S32, Size::S32) => dynasm!(self ; fmov S(dst), W(src)),
2786 (Size::S64, Size::S64) => dynasm!(self ; fmov D(dst), X(src)),
2787 _ => {
2788 err = true;
2789 }
2790 }
2791 }
2792 (AbstractLocation::SIMD(src), AbstractLocation::GPR(dst)) => {
2793 match (src_size, dst_size) {
2794 (Size::S32, Size::S32) => dynasm!(self ; fmov W(dst), S(src)),
2795 (Size::S64, Size::S64) => dynasm!(self ; fmov X(dst), D(src)),
2796 _ => {
2797 err = true;
2798 }
2799 }
2800 }
2801 _ => {
2805 err = true;
2806 }
2807 };
2808
2809 if err {
2810 codegen_error!(
2811 "singlepass can't generate fmov instruction for src_size: {:?}, src: {:?}, dst_size: {:?}, dst: {:?}",
2812 src_size,
2813 src,
2814 dst_size,
2815 dst,
2816 )
2817 }
2818 Ok(())
2819 }
2820}
2821
2822pub fn gen_std_trampoline_arm64(
2823 sig: &FunctionType,
2824 calling_convention: CallingConvention,
2825) -> Result<FunctionBody, CompileError> {
2826 let mut a = Assembler::new(0);
2827
2828 let fptr = GPR::X27;
2829 let args = GPR::X28;
2830
2831 dynasm!(a
2832 ; sub sp, sp, 32
2833 ; stp x29, x30, [sp]
2834 ; stp X(fptr), X(args), [sp, 16]
2835 ; mov x29, sp
2836 ; mov X(fptr), x1
2837 ; mov X(args), x2
2838 );
2839
2840 let stack_args = sig.params().len().saturating_sub(7); let mut stack_offset = stack_args as u32 * 8;
2842 if stack_args > 0 {
2843 if stack_offset % 16 != 0 {
2844 stack_offset += 8;
2845 assert!(stack_offset % 16 == 0);
2846 }
2847 if stack_offset < 0x1000 {
2848 dynasm!(a ; sub sp, sp, stack_offset);
2849 } else {
2850 a.emit_mov_imm(Location::GPR(GPR::X26), stack_offset as u64)?;
2851 dynasm!(a ; sub sp, sp, x26);
2852 }
2853 }
2854
2855 let mut caller_stack_offset: i32 = 0;
2858 for (i, param) in sig.params().iter().enumerate() {
2859 let sz = match *param {
2860 Type::I32 | Type::F32 => Size::S32,
2861 Type::I64 | Type::F64 => Size::S64,
2862 Type::ExternRef => Size::S64,
2863 Type::FuncRef => Size::S64,
2864 _ => codegen_error!(
2865 "singlepass unsupported param type for trampoline {:?}",
2866 *param
2867 ),
2868 };
2869 match i {
2870 0..=6 => {
2871 a.emit_ldr(
2872 sz,
2873 Location::GPR(GPR::from_index(i + 1).unwrap()),
2874 Location::Memory(args, (i * 16) as i32),
2875 )?;
2876 }
2877 _ => {
2878 #[allow(clippy::single_match)]
2879 match calling_convention {
2880 CallingConvention::AppleAarch64 => {
2881 let sz = 1
2882 << match sz {
2883 Size::S8 => 0,
2884 Size::S16 => 1,
2885 Size::S32 => 2,
2886 Size::S64 => 3,
2887 };
2888 if sz > 1 && caller_stack_offset & (sz - 1) != 0 {
2890 caller_stack_offset = (caller_stack_offset + (sz - 1)) & !(sz - 1);
2891 }
2892 }
2893 _ => (),
2894 };
2895 a.emit_ldr(
2897 sz,
2898 Location::GPR(GPR::X16),
2899 Location::Memory(args, (i * 16) as i32),
2900 )?;
2901 a.emit_str(
2902 sz,
2903 Location::GPR(GPR::X16),
2904 Location::Memory(GPR::XzrSp, caller_stack_offset),
2905 )?;
2906 match calling_convention {
2907 CallingConvention::AppleAarch64 => {
2908 caller_stack_offset += 1
2909 << match sz {
2910 Size::S8 => 0,
2911 Size::S16 => 1,
2912 Size::S32 => 2,
2913 Size::S64 => 3,
2914 };
2915 }
2916 _ => {
2917 caller_stack_offset += 8;
2918 }
2919 }
2920 }
2921 }
2922 }
2923
2924 dynasm!(a ; blr X(fptr));
2925
2926 if !sig.results().is_empty() {
2928 a.emit_str(Size::S64, Location::GPR(GPR::X0), Location::Memory(args, 0))?;
2929 }
2930
2931 dynasm!(a
2933 ; ldp X(fptr), X(args), [x29, 16]
2934 ; ldp x29, x30, [x29]
2935 );
2936 let restored_stack_offset = 32 + stack_offset;
2937 if restored_stack_offset < 0x1000 {
2938 dynasm!(a; add sp, sp, restored_stack_offset);
2939 } else {
2940 a.emit_mov_imm(Location::GPR(GPR::X26), restored_stack_offset as u64)?;
2941 dynasm!(a; add sp, sp, x26);
2942 }
2943 dynasm!(a; ret);
2944
2945 let mut body = a.finalize().unwrap();
2946 body.shrink_to_fit();
2947 Ok(FunctionBody {
2948 body,
2949 unwind_info: None,
2950 })
2951}
2952pub fn gen_std_dynamic_import_trampoline_arm64(
2954 vmoffsets: &VMOffsets,
2955 sig: &FunctionType,
2956 calling_convention: CallingConvention,
2957) -> Result<FunctionBody, CompileError> {
2958 let mut a = Assembler::new(0);
2959 let stack_offset: usize = 16 * std::cmp::max(sig.params().len(), sig.results().len());
2961 a.emit_stpdb(
2963 Size::S64,
2964 Location::GPR(GPR::X30),
2965 Location::GPR(GPR::X26),
2966 GPR::XzrSp,
2967 16,
2968 )?;
2969
2970 if stack_offset != 0 {
2971 if stack_offset < 0x1000 {
2972 a.emit_sub(
2973 Size::S64,
2974 Location::GPR(GPR::XzrSp),
2975 Location::Imm32(stack_offset as _),
2976 Location::GPR(GPR::XzrSp),
2977 )?;
2978 } else {
2979 a.emit_mov_imm(Location::GPR(GPR::X26), stack_offset as u64)?;
2980 a.emit_sub(
2981 Size::S64,
2982 Location::GPR(GPR::XzrSp),
2983 Location::GPR(GPR::X26),
2984 Location::GPR(GPR::XzrSp),
2985 )?;
2986 }
2987 }
2988
2989 if !sig.params().is_empty() {
2991 let mut argalloc = ArgumentRegisterAllocator::default();
2992 argalloc.next(Type::I64, calling_convention).unwrap(); let mut stack_param_count: usize = 0;
2995
2996 for (i, ty) in sig.params().iter().enumerate() {
2997 let source_loc = match argalloc.next(*ty, calling_convention)? {
2998 Some(ARM64Register::GPR(gpr)) => Location::GPR(gpr),
2999 Some(ARM64Register::NEON(neon)) => Location::SIMD(neon),
3000 None => {
3001 let sz = match calling_convention {
3002 CallingConvention::AppleAarch64 => match *ty {
3003 Type::I32 | Type::F32 => Size::S32,
3004 _ => {
3005 if stack_param_count & 7 != 0 {
3006 stack_param_count = (stack_param_count + 7) & !7;
3007 };
3008 Size::S64
3009 }
3010 },
3011 _ => Size::S64,
3012 };
3013 a.emit_ldr(
3014 sz,
3015 Location::GPR(GPR::X26),
3016 Location::Memory(GPR::XzrSp, (stack_offset + 16 + stack_param_count) as _),
3017 )?;
3018 stack_param_count += match sz {
3019 Size::S32 => 4,
3020 Size::S64 => 8,
3021 _ => codegen_error!(
3022 "singlepass unreachable in gen_std_dynamic_import_trampoline_arm64"
3023 ),
3024 };
3025 Location::GPR(GPR::X26)
3026 }
3027 };
3028 a.emit_str(
3029 Size::S64,
3030 source_loc,
3031 Location::Memory(GPR::XzrSp, (i * 16) as _),
3032 )?;
3033
3034 a.emit_str(
3036 Size::S64,
3037 Location::GPR(GPR::XzrSp), Location::Memory(GPR::XzrSp, (i * 16 + 8) as _), )?;
3040 }
3041 }
3042
3043 #[allow(clippy::match_single_binding)]
3044 match calling_convention {
3045 _ => {
3046 let offset = vmoffsets.vmdynamicfunction_import_context_address();
3048 a.emit_ldur(Size::S64, Location::GPR(GPR::X26), GPR::X0, offset as i32)?;
3049 a.emit_add(
3051 Size::S64,
3052 Location::GPR(GPR::XzrSp),
3053 Location::Imm8(0),
3054 Location::GPR(GPR::X1),
3055 )?;
3056 }
3057 };
3058
3059 a.emit_call_register(GPR::X26)?;
3061
3062 if !sig.results().is_empty() {
3064 assert_eq!(sig.results().len(), 1);
3065 a.emit_ldr(
3066 Size::S64,
3067 Location::GPR(GPR::X0),
3068 Location::Memory(GPR::XzrSp, 0),
3069 )?;
3070 }
3071
3072 if stack_offset != 0 {
3074 if stack_offset < 0x1000 {
3075 a.emit_add(
3076 Size::S64,
3077 Location::GPR(GPR::XzrSp),
3078 Location::Imm32(stack_offset as _),
3079 Location::GPR(GPR::XzrSp),
3080 )?;
3081 } else {
3082 a.emit_mov_imm(Location::GPR(GPR::X26), stack_offset as u64)?;
3083 a.emit_add(
3084 Size::S64,
3085 Location::GPR(GPR::XzrSp),
3086 Location::GPR(GPR::X26),
3087 Location::GPR(GPR::XzrSp),
3088 )?;
3089 }
3090 }
3091 a.emit_ldpia(
3092 Size::S64,
3093 Location::GPR(GPR::X30),
3094 Location::GPR(GPR::X26),
3095 GPR::XzrSp,
3096 16,
3097 )?;
3098
3099 a.emit_ret()?;
3101
3102 let mut body = a.finalize().unwrap();
3103 body.shrink_to_fit();
3104 Ok(FunctionBody {
3105 body,
3106 unwind_info: None,
3107 })
3108}
3109pub fn gen_import_call_trampoline_arm64(
3111 vmoffsets: &VMOffsets,
3112 index: FunctionIndex,
3113 sig: &FunctionType,
3114 calling_convention: CallingConvention,
3115) -> Result<CustomSection, CompileError> {
3116 let mut a = Assembler::new(0);
3117
3118 if sig
3123 .params()
3124 .iter()
3125 .any(|&x| x == Type::F32 || x == Type::F64)
3126 {
3127 #[allow(clippy::match_single_binding)]
3128 match calling_convention {
3129 _ => {
3130 let stack_offset: i32 = if sig.params().len() > 7 {
3132 7 * 8
3133 } else {
3134 (sig.params().len() as i32) * 8
3135 };
3136 let stack_offset = if stack_offset & 15 != 0 {
3137 stack_offset + 8
3138 } else {
3139 stack_offset
3140 };
3141 if stack_offset > 0 {
3142 if stack_offset < 0x1000 {
3143 a.emit_sub(
3144 Size::S64,
3145 Location::GPR(GPR::XzrSp),
3146 Location::Imm32(stack_offset as u32),
3147 Location::GPR(GPR::XzrSp),
3148 )?;
3149 } else {
3150 a.emit_mov_imm(Location::GPR(GPR::X16), stack_offset as u64)?;
3151 a.emit_sub(
3152 Size::S64,
3153 Location::GPR(GPR::XzrSp),
3154 Location::GPR(GPR::X16),
3155 Location::GPR(GPR::XzrSp),
3156 )?;
3157 }
3158 }
3159
3160 static PARAM_REGS: &[GPR] = &[
3162 GPR::X1,
3163 GPR::X2,
3164 GPR::X3,
3165 GPR::X4,
3166 GPR::X5,
3167 GPR::X6,
3168 GPR::X7,
3169 ];
3170 let mut param_locations = vec![];
3171 for (i, param_reg) in PARAM_REGS.iter().enumerate().take(sig.params().len()) {
3172 let loc = match i {
3173 0..=6 => {
3174 let loc = Location::Memory(GPR::XzrSp, (i * 8) as i32);
3175 a.emit_str(Size::S64, Location::GPR(*param_reg), loc)?;
3176 loc
3177 }
3178 _ => Location::Memory(GPR::XzrSp, stack_offset + ((i - 7) * 8) as i32),
3179 };
3180 param_locations.push(loc);
3181 }
3182
3183 let mut caller_stack_offset: i32 = 0;
3185 let mut argalloc = ArgumentRegisterAllocator::default();
3186 argalloc.next(Type::I64, calling_convention).unwrap(); for (i, ty) in sig.params().iter().enumerate() {
3188 let prev_loc = param_locations[i];
3189 let targ = match argalloc.next(*ty, calling_convention)? {
3190 Some(ARM64Register::GPR(gpr)) => Location::GPR(gpr),
3191 Some(ARM64Register::NEON(neon)) => Location::SIMD(neon),
3192 None => {
3193 a.emit_ldr(Size::S64, Location::GPR(GPR::X16), prev_loc)?;
3195 a.emit_str(
3196 Size::S64,
3197 Location::GPR(GPR::X16),
3198 Location::Memory(GPR::XzrSp, stack_offset + caller_stack_offset),
3199 )?;
3200 caller_stack_offset += 8;
3201 continue;
3202 }
3203 };
3204 a.emit_ldr(Size::S64, targ, prev_loc)?;
3205 }
3206
3207 if stack_offset > 0 {
3209 if stack_offset < 0x1000 {
3210 a.emit_add(
3211 Size::S64,
3212 Location::GPR(GPR::XzrSp),
3213 Location::Imm32(stack_offset as u32),
3214 Location::GPR(GPR::XzrSp),
3215 )?;
3216 } else {
3217 a.emit_mov_imm(Location::GPR(GPR::X16), stack_offset as u64)?;
3218 a.emit_add(
3219 Size::S64,
3220 Location::GPR(GPR::XzrSp),
3221 Location::GPR(GPR::X16),
3222 Location::GPR(GPR::XzrSp),
3223 )?;
3224 }
3225 }
3226 }
3227 }
3228 }
3229
3230 let offset = vmoffsets.vmctx_vmfunction_import(index);
3234 let offset =
3237 if (offset > 0 && offset < 0xF8) || (offset > 0 && offset < 0x7FF8 && (offset & 7) == 0) {
3238 offset
3239 } else {
3240 a.emit_mov_imm(Location::GPR(GPR::X16), (offset as i64) as u64)?;
3241 a.emit_add(
3242 Size::S64,
3243 Location::GPR(GPR::X0),
3244 Location::GPR(GPR::X16),
3245 Location::GPR(GPR::X0),
3246 )?;
3247 0
3248 };
3249 #[allow(clippy::match_single_binding)]
3250 match calling_convention {
3251 _ => {
3252 if (offset & 7) == 0 {
3253 a.emit_ldr(
3254 Size::S64,
3255 Location::GPR(GPR::X16),
3256 Location::Memory(GPR::X0, offset as i32), )?;
3258 a.emit_ldr(
3259 Size::S64,
3260 Location::GPR(GPR::X0),
3261 Location::Memory(GPR::X0, offset as i32 + 8), )?;
3263 } else {
3264 a.emit_ldur(
3265 Size::S64,
3266 Location::GPR(GPR::X16),
3267 GPR::X0,
3268 offset as i32, )?;
3270 a.emit_ldur(
3271 Size::S64,
3272 Location::GPR(GPR::X0),
3273 GPR::X0,
3274 offset as i32 + 8, )?;
3276 }
3277 }
3278 }
3279 a.emit_b_register(GPR::X16)?;
3280
3281 let mut contents = a.finalize().unwrap();
3282 contents.shrink_to_fit();
3283 let section_body = SectionBody::new_with_vec(contents);
3284
3285 Ok(CustomSection {
3286 protection: CustomSectionProtection::ReadExecute,
3287 alignment: None,
3288 bytes: section_body,
3289 relocations: vec![],
3290 })
3291}