1pub use crate::{
2 arm64_decl::{ARM64Register, ArgumentRegisterAllocator, GPR, NEON},
3 location::{Multiplier, Reg},
4 machine::{Label, Offset},
5};
6use crate::{codegen_error, common_decl::Size, location::Location as AbstractLocation};
7use dynasm::dynasm;
8pub use dynasmrt::aarch64::{encode_logical_immediate_32bit, encode_logical_immediate_64bit};
9use dynasmrt::{
10 AssemblyOffset, DynamicLabel, DynasmApi, DynasmLabelApi, VecAssembler,
11 aarch64::Aarch64Relocation,
12};
13use wasmer_compiler::types::{
14 function::FunctionBody,
15 section::{CustomSection, CustomSectionProtection, SectionBody},
16};
17use wasmer_types::{
18 CompileError, FunctionIndex, FunctionType, Type, VMOffsets, target::CallingConvention,
19};
20
21type Assembler = VecAssembler<Aarch64Relocation>;
22
23macro_rules! dynasm {
28 ($a:expr_2021 ; $($tt:tt)*) => {
29 dynasm::dynasm!(
30 $a
31 ; .arch aarch64
32 ; $($tt)*
33 )
34 };
35}
36
37pub type Location = AbstractLocation<GPR, NEON>;
38
39#[derive(Copy, Clone, Debug, Eq, PartialEq)]
40#[allow(dead_code)]
41#[repr(u8)]
42pub enum Condition {
43 Eq = 0,
46 Ne = 1,
48 Cs = 2,
50 Cc = 3,
52 Mi = 4,
54 Pl = 5,
56 Vs = 6,
58 Vc = 7,
60 Hi = 8,
62 Ls = 9,
64 Ge = 10,
66 Lt = 11,
68 Gt = 12,
70 Le = 13,
72 Al = 14,
74}
75
76#[derive(Copy, Clone, Debug, Eq, PartialEq)]
77#[allow(dead_code, clippy::upper_case_acronyms)]
78pub enum NeonOrMemory {
79 NEON(NEON),
80 Memory(GPR, i32),
81}
82
83#[derive(Copy, Clone, Debug)]
84#[allow(dead_code, clippy::upper_case_acronyms)]
85pub enum GPROrMemory {
86 GPR(GPR),
87 Memory(GPR, i32),
88}
89
90#[allow(unused)]
91pub trait EmitterARM64 {
92 fn get_label(&mut self) -> Label;
93 fn get_offset(&self) -> Offset;
94 fn get_jmp_instr_size(&self) -> u8;
95
96 fn finalize_function(&mut self);
97
98 fn emit_str(&mut self, sz: Size, reg: Location, addr: Location) -> Result<(), CompileError>;
99 fn emit_ldr(&mut self, sz: Size, reg: Location, addr: Location) -> Result<(), CompileError>;
100 fn emit_stur(
101 &mut self,
102 sz: Size,
103 reg: Location,
104 addr: GPR,
105 offset: i32,
106 ) -> Result<(), CompileError>;
107 fn emit_ldur(
108 &mut self,
109 sz: Size,
110 reg: Location,
111 addr: GPR,
112 offset: i32,
113 ) -> Result<(), CompileError>;
114 fn emit_strdb(
115 &mut self,
116 sz: Size,
117 reg: Location,
118 addr: GPR,
119 offset: u32,
120 ) -> Result<(), CompileError>;
121 fn emit_stria(
122 &mut self,
123 sz: Size,
124 reg: Location,
125 addr: GPR,
126 offset: u32,
127 ) -> Result<(), CompileError>;
128 fn emit_ldria(
129 &mut self,
130 sz: Size,
131 reg: Location,
132 addr: GPR,
133 offset: u32,
134 ) -> Result<(), CompileError>;
135 fn emit_stpdb(
136 &mut self,
137 sz: Size,
138 reg1: Location,
139 reg2: Location,
140 addr: GPR,
141 offset: u32,
142 ) -> Result<(), CompileError>;
143 fn emit_ldpia(
144 &mut self,
145 sz: Size,
146 reg1: Location,
147 reg2: Location,
148 addr: GPR,
149 offset: u32,
150 ) -> Result<(), CompileError>;
151
152 fn emit_ldrb(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
153 fn emit_ldrh(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
154 fn emit_ldrsb(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
155 fn emit_ldrsh(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
156 fn emit_ldrsw(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
157 fn emit_strb(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
158 fn emit_strh(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
159
160 fn emit_ldaxr(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
161 fn emit_ldaxrb(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
162 fn emit_ldaxrh(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
163 fn emit_stlxr(
164 &mut self,
165 sz: Size,
166 status: Location,
167 reg: Location,
168 dst: Location,
169 ) -> Result<(), CompileError>;
170 fn emit_stlxrb(
171 &mut self,
172 sz: Size,
173 status: Location,
174 reg: Location,
175 dst: Location,
176 ) -> Result<(), CompileError>;
177 fn emit_stlxrh(
178 &mut self,
179 sz: Size,
180 status: Location,
181 reg: Location,
182 dst: Location,
183 ) -> Result<(), CompileError>;
184
185 fn emit_mov(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
186
187 fn emit_movn(&mut self, sz: Size, reg: Location, val: u32) -> Result<(), CompileError>;
188 fn emit_movz(&mut self, reg: Location, val: u32) -> Result<(), CompileError>;
189 fn emit_movk(&mut self, reg: Location, val: u32, shift: u32) -> Result<(), CompileError>;
190
191 fn emit_mov_imm(&mut self, dst: Location, val: u64) -> Result<(), CompileError>;
192
193 fn emit_add(
194 &mut self,
195 sz: Size,
196 src1: Location,
197 src2: Location,
198 dst: Location,
199 ) -> Result<(), CompileError>;
200 fn emit_sub(
201 &mut self,
202 sz: Size,
203 src1: Location,
204 src2: Location,
205 dst: Location,
206 ) -> Result<(), CompileError>;
207 fn emit_mul(
208 &mut self,
209 sz: Size,
210 src1: Location,
211 src2: Location,
212 dst: Location,
213 ) -> Result<(), CompileError>;
214 fn emit_adds(
215 &mut self,
216 sz: Size,
217 src1: Location,
218 src2: Location,
219 dst: Location,
220 ) -> Result<(), CompileError>;
221 fn emit_subs(
222 &mut self,
223 sz: Size,
224 src1: Location,
225 src2: Location,
226 dst: Location,
227 ) -> Result<(), CompileError>;
228
229 fn emit_add_lsl(
230 &mut self,
231 sz: Size,
232 src1: Location,
233 src2: Location,
234 lsl: u32,
235 dst: Location,
236 ) -> Result<(), CompileError>;
237
238 fn emit_cmp(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
239 fn emit_tst(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
240
241 fn emit_lsl(
242 &mut self,
243 sz: Size,
244 src1: Location,
245 src2: Location,
246 dst: Location,
247 ) -> Result<(), CompileError>;
248 fn emit_lsr(
249 &mut self,
250 sz: Size,
251 src1: Location,
252 src2: Location,
253 dst: Location,
254 ) -> Result<(), CompileError>;
255 fn emit_asr(
256 &mut self,
257 sz: Size,
258 src1: Location,
259 src2: Location,
260 dst: Location,
261 ) -> Result<(), CompileError>;
262 fn emit_ror(
263 &mut self,
264 sz: Size,
265 src1: Location,
266 src2: Location,
267 dst: Location,
268 ) -> Result<(), CompileError>;
269
270 fn emit_or(
271 &mut self,
272 sz: Size,
273 src1: Location,
274 src2: Location,
275 dst: Location,
276 ) -> Result<(), CompileError>;
277 fn emit_and(
278 &mut self,
279 sz: Size,
280 src1: Location,
281 src2: Location,
282 dst: Location,
283 ) -> Result<(), CompileError>;
284 fn emit_eor(
285 &mut self,
286 sz: Size,
287 src1: Location,
288 src2: Location,
289 dst: Location,
290 ) -> Result<(), CompileError>;
291
292 fn emit_bfc(
293 &mut self,
294 se: Size,
295 lsb: u32,
296 width: u32,
297 dst: Location,
298 ) -> Result<(), CompileError>;
299 fn emit_bfi(
300 &mut self,
301 se: Size,
302 src: Location,
303 lsb: u32,
304 width: u32,
305 dst: Location,
306 ) -> Result<(), CompileError>;
307
308 fn emit_udiv(
309 &mut self,
310 sz: Size,
311 src1: Location,
312 src2: Location,
313 dst: Location,
314 ) -> Result<(), CompileError>;
315 fn emit_sdiv(
316 &mut self,
317 sz: Size,
318 src1: Location,
319 src2: Location,
320 dst: Location,
321 ) -> Result<(), CompileError>;
322 fn emit_msub(
324 &mut self,
325 sz: Size,
326 a: Location,
327 b: Location,
328 c: Location,
329 dst: Location,
330 ) -> Result<(), CompileError>;
331
332 fn emit_sxtb(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
333 fn emit_sxth(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
334 fn emit_sxtw(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
335 fn emit_uxtb(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
336 fn emit_uxth(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
337
338 fn emit_cset(&mut self, sz: Size, dst: Location, cond: Condition) -> Result<(), CompileError>;
339 fn emit_csetm(&mut self, sz: Size, dst: Location, cond: Condition) -> Result<(), CompileError>;
340 fn emit_cinc(
341 &mut self,
342 sz: Size,
343 src: Location,
344 dst: Location,
345 cond: Condition,
346 ) -> Result<(), CompileError>;
347 fn emit_clz(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
348 fn emit_rbit(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
349
350 fn emit_label(&mut self, label: Label) -> Result<(), CompileError>;
351 fn emit_load_label(&mut self, reg: GPR, label: Label) -> Result<(), CompileError>;
352 fn emit_b_label(&mut self, label: Label) -> Result<(), CompileError>;
353 fn emit_cbz_label(&mut self, sz: Size, reg: Location, label: Label)
354 -> Result<(), CompileError>;
355 fn emit_cbnz_label(
356 &mut self,
357 sz: Size,
358 reg: Location,
359 label: Label,
360 ) -> Result<(), CompileError>;
361 fn emit_cbz_label_far(
362 &mut self,
363 sz: Size,
364 reg: Location,
365 label: Label,
366 ) -> Result<(), CompileError>;
367 fn emit_tbz_label(
368 &mut self,
369 sz: Size,
370 reg: Location,
371 n: u32,
372 label: Label,
373 ) -> Result<(), CompileError>;
374 fn emit_tbnz_label(
375 &mut self,
376 sz: Size,
377 reg: Location,
378 n: u32,
379 label: Label,
380 ) -> Result<(), CompileError>;
381 fn emit_bcond_label(&mut self, condition: Condition, label: Label) -> Result<(), CompileError>;
382 fn emit_bcond_label_far(
383 &mut self,
384 condition: Condition,
385 label: Label,
386 ) -> Result<(), CompileError>;
387 fn emit_b_register(&mut self, reg: GPR) -> Result<(), CompileError>;
388 fn emit_call_label(&mut self, label: Label) -> Result<(), CompileError>;
389 fn emit_call_register(&mut self, reg: GPR) -> Result<(), CompileError>;
390 fn emit_ret(&mut self) -> Result<(), CompileError>;
391
392 fn emit_udf(&mut self, payload: u16) -> Result<(), CompileError>;
393 fn emit_dmb(&mut self) -> Result<(), CompileError>;
394 fn emit_brk(&mut self) -> Result<(), CompileError>;
395
396 fn emit_fcmp(&mut self, sz: Size, src1: Location, src2: Location) -> Result<(), CompileError>;
397 fn emit_fneg(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
398 fn emit_fsqrt(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
399
400 fn emit_fadd(
401 &mut self,
402 sz: Size,
403 src1: Location,
404 src2: Location,
405 dst: Location,
406 ) -> Result<(), CompileError>;
407 fn emit_fsub(
408 &mut self,
409 sz: Size,
410 src1: Location,
411 src2: Location,
412 dst: Location,
413 ) -> Result<(), CompileError>;
414 fn emit_fmul(
415 &mut self,
416 sz: Size,
417 src1: Location,
418 src2: Location,
419 dst: Location,
420 ) -> Result<(), CompileError>;
421 fn emit_fdiv(
422 &mut self,
423 sz: Size,
424 src1: Location,
425 src2: Location,
426 dst: Location,
427 ) -> Result<(), CompileError>;
428
429 fn emit_fmin(
430 &mut self,
431 sz: Size,
432 src1: Location,
433 src2: Location,
434 dst: Location,
435 ) -> Result<(), CompileError>;
436 fn emit_fmax(
437 &mut self,
438 sz: Size,
439 src1: Location,
440 src2: Location,
441 dst: Location,
442 ) -> Result<(), CompileError>;
443
444 fn emit_frintz(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
445 fn emit_frintn(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
446 fn emit_frintm(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
447 fn emit_frintp(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
448
449 fn emit_scvtf(
450 &mut self,
451 sz_in: Size,
452 src: Location,
453 sz_out: Size,
454 dst: Location,
455 ) -> Result<(), CompileError>;
456 fn emit_ucvtf(
457 &mut self,
458 sz_in: Size,
459 src: Location,
460 sz_out: Size,
461 dst: Location,
462 ) -> Result<(), CompileError>;
463 fn emit_fcvt(&mut self, sz_in: Size, src: Location, dst: Location) -> Result<(), CompileError>;
464 fn emit_fcvtzs(
465 &mut self,
466 sz_in: Size,
467 src: Location,
468 sz_out: Size,
469 dst: Location,
470 ) -> Result<(), CompileError>;
471 fn emit_fcvtzu(
472 &mut self,
473 sz_in: Size,
474 src: Location,
475 sz_out: Size,
476 dst: Location,
477 ) -> Result<(), CompileError>;
478
479 fn emit_fmov(
480 &mut self,
481 src_size: Size,
482 src: Location,
483 dst_size: Size,
484 dst: Location,
485 ) -> Result<(), CompileError>;
486 fn emit_cnt(&mut self, src: NEON, dst: NEON) -> Result<(), CompileError>;
487 fn emit_addv(&mut self, src: NEON, dst: NEON) -> Result<(), CompileError>;
488 fn emit_read_fpcr(&mut self, reg: GPR) -> Result<(), CompileError>;
489 fn emit_write_fpcr(&mut self, reg: GPR) -> Result<(), CompileError>;
490 fn emit_read_fpsr(&mut self, reg: GPR) -> Result<(), CompileError>;
491 fn emit_write_fpsr(&mut self, reg: GPR) -> Result<(), CompileError>;
492
493 fn arch_supports_canonicalize_nan(&self) -> bool {
494 true
495 }
496
497 fn arch_requires_indirect_call_trampoline(&self) -> bool {
498 false
499 }
500
501 fn arch_emit_indirect_call_with_trampoline(
502 &mut self,
503 _loc: Location,
504 ) -> Result<(), CompileError> {
505 codegen_error!("singlepass arch_emit_indirect_call_with_trampoline unimplemented")
506 }
507}
508
509impl EmitterARM64 for Assembler {
510 fn get_label(&mut self) -> DynamicLabel {
511 self.new_dynamic_label()
512 }
513
514 fn get_offset(&self) -> AssemblyOffset {
515 self.offset()
516 }
517
518 fn get_jmp_instr_size(&self) -> u8 {
519 4 }
521
522 fn finalize_function(&mut self) {
523 dynasm!(
524 self
525 ; const_neg_one_32:
526 ; .i32 -1
527 ; const_zero_32:
528 ; .i32 0
529 ; const_pos_one_32:
530 ; .i32 1
531 );
532 }
533
534 fn emit_str(&mut self, sz: Size, reg: Location, addr: Location) -> Result<(), CompileError> {
535 match (sz, reg, addr) {
536 (Size::S64, Location::GPR(reg), Location::Memory(addr, disp)) => {
537 let disp = disp as u32;
538 assert!((disp & 0x7) == 0 && (disp < 0x8000));
539 dynasm!(self ; str X(reg), [X(addr), disp]);
540 }
541 (Size::S32, Location::GPR(reg), Location::Memory(addr, disp)) => {
542 let disp = disp as u32;
543 assert!((disp & 0x3) == 0 && (disp < 0x4000));
544 dynasm!(self ; str W(reg), [X(addr), disp]);
545 }
546 (Size::S16, Location::GPR(reg), Location::Memory(addr, disp)) => {
547 let disp = disp as u32;
548 assert!((disp & 0x1) == 0 && (disp < 0x2000));
549 dynasm!(self ; strh W(reg), [X(addr), disp]);
550 }
551 (Size::S8, Location::GPR(reg), Location::Memory(addr, disp)) => {
552 let disp = disp as u32;
553 assert!(disp < 0x1000);
554 dynasm!(self ; strb W(reg), [X(addr), disp]);
555 }
556 (Size::S64, Location::SIMD(reg), Location::Memory(addr, disp)) => {
557 let disp = disp as u32;
558 assert!((disp & 0x7) == 0 && (disp < 0x8000));
559 dynasm!(self ; str D(reg), [X(addr), disp]);
560 }
561 (Size::S32, Location::SIMD(reg), Location::Memory(addr, disp)) => {
562 let disp = disp as u32;
563 assert!((disp & 0x3) == 0 && (disp < 0x4000));
564 dynasm!(self ; str S(reg), [X(addr), disp]);
565 }
566 (Size::S64, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
567 assert!(offs == 0);
568 let mult = mult as u32;
569 match mult {
570 0 => dynasm!(self ; str X(reg), [X(addr)]),
571 1 => dynasm!(self ; str X(reg), [X(addr), X(r2)]),
572 _ => dynasm!(self ; str X(reg), [X(addr), X(r2), LSL mult]),
573 };
574 }
575 (Size::S32, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
576 assert!(offs == 0);
577 let mult = mult as u32;
578 match mult {
579 0 => dynasm!(self ; str W(reg), [X(addr)]),
580 1 => dynasm!(self ; str W(reg), [X(addr), X(r2)]),
581 _ => dynasm!(self ; str W(reg), [X(addr), X(r2), LSL mult]),
582 };
583 }
584 _ => codegen_error!("singlepass can't emit STR {:?}, {:?}, {:?}", sz, reg, addr),
585 }
586 Ok(())
587 }
588 fn emit_ldr(&mut self, sz: Size, reg: Location, addr: Location) -> Result<(), CompileError> {
589 match (sz, reg, addr) {
590 (Size::S64, Location::GPR(reg), Location::Memory(addr, disp)) => {
591 assert!((disp & 0x7) == 0 && (disp < 0x8000));
592 let disp = disp as u32;
593 dynasm!(self ; ldr X(reg), [X(addr), disp]);
594 }
595 (Size::S32, Location::GPR(reg), Location::Memory(addr, disp)) => {
596 assert!((disp & 0x3) == 0 && (disp < 0x4000));
597 let disp = disp as u32;
598 dynasm!(self ; ldr W(reg), [X(addr), disp]);
599 }
600 (Size::S16, Location::GPR(reg), Location::Memory(addr, disp)) => {
601 assert!((disp & 0x1 == 0) && (disp < 0x2000));
602 let disp = disp as u32;
603 dynasm!(self ; ldrh W(reg), [X(addr), disp]);
604 }
605 (Size::S8, Location::GPR(reg), Location::Memory(addr, disp)) => {
606 assert!(disp < 0x1000);
607 let disp = disp as u32;
608 dynasm!(self ; ldrb W(reg), [X(addr), disp]);
609 }
610 (Size::S64, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
611 assert!(offs == 0);
612 let mult = mult as u32;
613 match mult {
614 0 => dynasm!(self ; ldr X(reg), [X(addr)]),
615 1 => dynasm!(self ; ldr X(reg), [X(addr), X(r2)]),
616 _ => dynasm!(self ; ldr X(reg), [X(addr), X(r2), LSL mult]),
617 };
618 }
619 (Size::S32, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
620 assert!(offs == 0);
621 let mult = mult as u32;
622 match mult {
623 0 => dynasm!(self ; ldr W(reg), [X(addr)]),
624 1 => dynasm!(self ; ldr W(reg), [X(addr), X(r2)]),
625 _ => dynasm!(self ; ldr W(reg), [X(addr), X(r2), LSL mult]),
626 };
627 }
628 (Size::S64, Location::SIMD(reg), Location::Memory(addr, disp)) => {
629 let disp = disp as u32;
630 assert!((disp & 0x7) == 0 && (disp < 0x8000));
631 dynasm!(self ; ldr D(reg), [X(addr), disp]);
632 }
633 (Size::S32, Location::SIMD(reg), Location::Memory(addr, disp)) => {
634 let disp = disp as u32;
635 assert!((disp & 0x3) == 0 && (disp < 0x4000));
636 dynasm!(self ; ldr S(reg), [X(addr), disp]);
637 }
638 (Size::S64, Location::SIMD(reg), Location::Memory2(addr, r2, mult, offs)) => {
639 assert!(offs == 0);
640 let mult = mult as u32;
641 match mult {
642 0 => dynasm!(self ; ldr D(reg), [X(addr)]),
643 1 => dynasm!(self ; ldr D(reg), [X(addr), X(r2)]),
644 _ => dynasm!(self ; ldr D(reg), [X(addr), X(r2), LSL mult]),
645 };
646 }
647 (Size::S32, Location::SIMD(reg), Location::Memory2(addr, r2, mult, offs)) => {
648 assert!(offs == 0);
649 let mult = mult as u32;
650 match mult {
651 0 => dynasm!(self ; ldr S(reg), [X(addr)]),
652 1 => dynasm!(self ; ldr S(reg), [X(addr), X(r2)]),
653 _ => dynasm!(self ; ldr S(reg), [X(addr), X(r2), LSL mult]),
654 };
655 }
656 _ => codegen_error!("singlepass can't emit LDR {:?}, {:?}, {:?}", sz, reg, addr),
657 }
658 Ok(())
659 }
660 fn emit_stur(
661 &mut self,
662 sz: Size,
663 reg: Location,
664 addr: GPR,
665 offset: i32,
666 ) -> Result<(), CompileError> {
667 assert!((-255..=255).contains(&offset));
668 match (sz, reg) {
669 (Size::S64, Location::GPR(reg)) => {
670 dynasm!(self ; stur X(reg), [X(addr), offset]);
671 }
672 (Size::S32, Location::GPR(reg)) => {
673 dynasm!(self ; stur W(reg), [X(addr), offset]);
674 }
675 (Size::S64, Location::SIMD(reg)) => {
676 dynasm!(self ; stur D(reg), [X(addr), offset]);
677 }
678 (Size::S32, Location::SIMD(reg)) => {
679 dynasm!(self ; stur S(reg), [X(addr), offset]);
680 }
681 _ => codegen_error!(
682 "singlepass can't emit STUR {:?}, {:?}, {:?}, {:?}",
683 sz,
684 reg,
685 addr,
686 offset
687 ),
688 }
689 Ok(())
690 }
691 fn emit_ldur(
692 &mut self,
693 sz: Size,
694 reg: Location,
695 addr: GPR,
696 offset: i32,
697 ) -> Result<(), CompileError> {
698 assert!((-255..=255).contains(&offset));
699 match (sz, reg) {
700 (Size::S64, Location::GPR(reg)) => {
701 dynasm!(self ; ldur X(reg), [X(addr), offset]);
702 }
703 (Size::S32, Location::GPR(reg)) => {
704 dynasm!(self ; ldur W(reg), [X(addr), offset]);
705 }
706 (Size::S64, Location::SIMD(reg)) => {
707 dynasm!(self ; ldur D(reg), [X(addr), offset]);
708 }
709 (Size::S32, Location::SIMD(reg)) => {
710 dynasm!(self ; ldur S(reg), [X(addr), offset]);
711 }
712 _ => codegen_error!(
713 "singlepass can't emit LDUR {:?}, {:?}, {:?}, {:?}",
714 sz,
715 reg,
716 addr,
717 offset
718 ),
719 }
720 Ok(())
721 }
722
723 fn emit_strdb(
724 &mut self,
725 sz: Size,
726 reg: Location,
727 addr: GPR,
728 offset: u32,
729 ) -> Result<(), CompileError> {
730 assert!(offset <= 255);
731 match (sz, reg) {
732 (Size::S64, Location::GPR(reg)) => {
733 dynasm!(self ; str X(reg), [X(addr), -(offset as i32)]!);
734 }
735 (Size::S64, Location::SIMD(reg)) => {
736 dynasm!(self ; str D(reg), [X(addr), -(offset as i32)]!);
737 }
738 _ => codegen_error!("singlepass can't emit STRDB"),
739 }
740 Ok(())
741 }
742 fn emit_stria(
743 &mut self,
744 sz: Size,
745 reg: Location,
746 addr: GPR,
747 offset: u32,
748 ) -> Result<(), CompileError> {
749 assert!(offset <= 255);
750 match (sz, reg) {
751 (Size::S64, Location::GPR(reg)) => {
752 dynasm!(self ; str X(reg), [X(addr)], offset as i32);
753 }
754 (Size::S64, Location::SIMD(reg)) => {
755 dynasm!(self ; str D(reg), [X(addr)], offset as i32);
756 }
757 _ => codegen_error!("singlepass can't emit STRIA"),
758 }
759 Ok(())
760 }
761 fn emit_ldria(
762 &mut self,
763 sz: Size,
764 reg: Location,
765 addr: GPR,
766 offset: u32,
767 ) -> Result<(), CompileError> {
768 assert!(offset <= 255);
769 match (sz, reg) {
770 (Size::S64, Location::GPR(reg)) => {
771 dynasm!(self ; ldr X(reg), [X(addr)], offset as _);
772 }
773 (Size::S64, Location::SIMD(reg)) => {
774 dynasm!(self ; ldr D(reg), [X(addr)], offset as _);
775 }
776 _ => codegen_error!("singlepass can't emit LDRIA"),
777 }
778 Ok(())
779 }
780
781 fn emit_stpdb(
782 &mut self,
783 sz: Size,
784 reg1: Location,
785 reg2: Location,
786 addr: GPR,
787 offset: u32,
788 ) -> Result<(), CompileError> {
789 assert!(offset <= 255);
790 match (sz, reg1, reg2) {
791 (Size::S64, Location::GPR(reg1), Location::GPR(reg2)) => {
792 dynasm!(self ; stp X(reg1), X(reg2), [X(addr), -(offset as i32)]!);
793 }
794 _ => codegen_error!("singlepass can't emit STPDB"),
795 }
796 Ok(())
797 }
798 fn emit_ldpia(
799 &mut self,
800 sz: Size,
801 reg1: Location,
802 reg2: Location,
803 addr: GPR,
804 offset: u32,
805 ) -> Result<(), CompileError> {
806 assert!(offset <= 255);
807 match (sz, reg1, reg2) {
808 (Size::S64, Location::GPR(reg1), Location::GPR(reg2)) => {
809 dynasm!(self ; ldp X(reg1), X(reg2), [X(addr)], offset as _);
810 }
811 _ => codegen_error!("singlepass can't emit LDPIA"),
812 }
813 Ok(())
814 }
815
816 fn emit_ldrb(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
817 match (reg, dst) {
818 (Location::GPR(reg), Location::Memory(addr, offset)) => {
819 let offset = offset as u32;
820 assert!(offset < 0x1000);
821 dynasm!(self ; ldrb W(reg), [X(addr), offset]);
822 }
823 (Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
824 assert!(offs == 0);
825 let mult = mult as u32;
826 match mult {
827 0 => dynasm!(self ; ldrb W(reg), [X(addr)]),
828 1 => dynasm!(self ; ldrb W(reg), [X(addr), X(r2)]),
829 _ => dynasm!(self ; ldrb W(reg), [X(addr), X(r2), LSL mult]),
830 };
831 }
832 _ => codegen_error!("singlepass can't emit LDRB {:?}, {:?}", reg, dst),
833 }
834 Ok(())
835 }
836 fn emit_ldrh(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
837 match (reg, dst) {
838 (Location::GPR(reg), Location::Memory(addr, offset)) => {
839 let offset = offset as u32;
840 assert!((offset & 1 == 0) && (offset < 0x2000));
841 dynasm!(self ; ldrh W(reg), [X(addr), offset]);
842 }
843 (Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
844 assert!(offs == 0);
845 let mult = mult as u32;
846 match mult {
847 0 => dynasm!(self ; ldrh W(reg), [X(addr)]),
848 1 => dynasm!(self ; ldrh W(reg), [X(addr), X(r2)]),
849 _ => dynasm!(self ; ldrh W(reg), [X(addr), X(r2), LSL mult]),
850 };
851 }
852 _ => codegen_error!("singlepass can't emit LDRH {:?}, {:?}", reg, dst),
853 }
854 Ok(())
855 }
856 fn emit_ldrsb(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
857 match (sz, reg, dst) {
858 (Size::S64, Location::GPR(reg), Location::Memory(addr, offset)) => {
859 let offset = offset as u32;
860 assert!(offset < 0x1000);
861 dynasm!(self ; ldrsb X(reg), [X(addr), offset]);
862 }
863 (Size::S32, Location::GPR(reg), Location::Memory(addr, offset)) => {
864 let offset = offset as u32;
865 assert!(offset < 0x1000);
866 dynasm!(self ; ldrsb W(reg), [X(addr), offset]);
867 }
868 (Size::S64, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
869 assert!(offs == 0);
870 let mult = mult as u32;
871 match mult {
872 0 => dynasm!(self ; ldrsb X(reg), [X(addr)]),
873 1 => dynasm!(self ; ldrsb X(reg), [X(addr), X(r2)]),
874 _ => dynasm!(self ; ldrsb X(reg), [X(addr), X(r2), LSL mult]),
875 };
876 }
877 (Size::S32, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
878 assert!(offs == 0);
879 let mult = mult as u32;
880 match mult {
881 0 => dynasm!(self ; ldrsb W(reg), [X(addr)]),
882 1 => dynasm!(self ; ldrsb W(reg), [X(addr), X(r2)]),
883 _ => dynasm!(self ; ldrsb W(reg), [X(addr), X(r2), LSL mult]),
884 };
885 }
886 _ => codegen_error!("singlepass can't emit LDRSB {:?}, {:?}, {:?}", sz, reg, dst),
887 }
888 Ok(())
889 }
890 fn emit_ldrsh(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
891 match (sz, reg, dst) {
892 (Size::S64, Location::GPR(reg), Location::Memory(addr, offset)) => {
893 let offset = offset as u32;
894 assert!((offset & 1 == 0) && (offset < 0x2000));
895 dynasm!(self ; ldrsh X(reg), [X(addr), offset]);
896 }
897 (Size::S32, Location::GPR(reg), Location::Memory(addr, offset)) => {
898 let offset = offset as u32;
899 assert!((offset & 1 == 0) && (offset < 0x2000));
900 dynasm!(self ; ldrsh W(reg), [X(addr), offset]);
901 }
902 (Size::S64, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
903 assert!(offs == 0);
904 let mult = mult as u32;
905 match mult {
906 0 => dynasm!(self ; ldrsh X(reg), [X(addr)]),
907 1 => dynasm!(self ; ldrsh X(reg), [X(addr), X(r2)]),
908 _ => dynasm!(self ; ldrsh X(reg), [X(addr), X(r2), LSL mult]),
909 };
910 }
911 (Size::S32, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
912 assert!(offs == 0);
913 let mult = mult as u32;
914 match mult {
915 0 => dynasm!(self ; ldrsh W(reg), [X(addr)]),
916 1 => dynasm!(self ; ldrsh W(reg), [X(addr), X(r2)]),
917 _ => dynasm!(self ; ldrsh W(reg), [X(addr), X(r2), LSL mult]),
918 };
919 }
920 _ => codegen_error!("singlepass can't emit LDRSH {:?}, {:?}, {:?}", sz, reg, dst),
921 }
922 Ok(())
923 }
924 fn emit_ldrsw(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
925 match (sz, reg, dst) {
926 (Size::S64, Location::GPR(reg), Location::Memory(addr, offset)) => {
927 let offset = offset as u32;
928 assert!((offset & 3 == 0) && (offset < 0x4000));
929 dynasm!(self ; ldrsw X(reg), [X(addr), offset]);
930 }
931 (Size::S64, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
932 assert!(offs == 0);
933 let mult = mult as u32;
934 match mult {
935 0 => dynasm!(self ; ldrsw X(reg), [X(addr)]),
936 1 => dynasm!(self ; ldrsw X(reg), [X(addr), X(r2)]),
937 _ => dynasm!(self ; ldrsw X(reg), [X(addr), X(r2), LSL mult]),
938 };
939 }
940 _ => codegen_error!("singlepass can't emit LDRSW {:?}, {:?}, {:?}", sz, reg, dst),
941 }
942 Ok(())
943 }
944 fn emit_strb(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
945 match (reg, dst) {
946 (Location::GPR(reg), Location::Memory(addr, offset)) => {
947 let offset = offset as u32;
948 assert!(offset < 0x1000);
949 dynasm!(self ; strb W(reg), [X(addr), offset]);
950 }
951 (Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
952 assert!(offs == 0);
953 let mult = mult as u32;
954 match mult {
955 0 => dynasm!(self ; strb W(reg), [X(addr)]),
956 1 => dynasm!(self ; strb W(reg), [X(addr), X(r2)]),
957 _ => dynasm!(self ; strb W(reg), [X(addr), X(r2), LSL mult]),
958 };
959 }
960 _ => codegen_error!("singlepass can't emit STRB {:?}, {:?}", reg, dst),
961 }
962 Ok(())
963 }
964 fn emit_strh(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
965 match (reg, dst) {
966 (Location::GPR(reg), Location::Memory(addr, offset)) => {
967 let offset = offset as u32;
968 assert!((offset & 1 == 0) && (offset < 0x2000));
969 dynasm!(self ; strh W(reg), [X(addr), offset]);
970 }
971 (Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
972 assert!(offs == 0);
973 let mult = mult as u32;
974 match mult {
975 0 => dynasm!(self ; strh W(reg), [X(addr)]),
976 1 => dynasm!(self ; strh W(reg), [X(addr), X(r2)]),
977 _ => dynasm!(self ; strh W(reg), [X(addr), X(r2), LSL mult]),
978 };
979 }
980 _ => codegen_error!("singlepass can't emit STRH {:?}, {:?}", reg, dst),
981 }
982 Ok(())
983 }
984
985 fn emit_ldaxr(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
986 match (sz, reg, dst) {
987 (Size::S32, Location::GPR(reg), Location::GPR(dst)) => {
988 dynasm!(self ; ldaxr W(reg), [X(dst)]);
989 }
990 (Size::S64, Location::GPR(reg), Location::GPR(dst)) => {
991 dynasm!(self ; ldaxr X(reg), [X(dst)]);
992 }
993 _ => codegen_error!("singlepass can't emit LDAXR {:?}, {:?}", reg, dst),
994 }
995 Ok(())
996 }
997 fn emit_ldaxrb(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
998 match (reg, dst) {
999 (Location::GPR(reg), Location::GPR(dst)) => {
1000 dynasm!(self ; ldaxrb W(reg), [X(dst)]);
1001 }
1002 _ => codegen_error!("singlepass can't emit LDAXRB {:?}, {:?}", reg, dst),
1003 }
1004 Ok(())
1005 }
1006 fn emit_ldaxrh(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
1007 match (reg, dst) {
1008 (Location::GPR(reg), Location::GPR(dst)) => {
1009 dynasm!(self ; ldaxrh W(reg), [X(dst)]);
1010 }
1011 _ => codegen_error!("singlepass can't emit LDAXRH {:?}, {:?}", reg, dst),
1012 }
1013 Ok(())
1014 }
1015 fn emit_stlxr(
1016 &mut self,
1017 sz: Size,
1018 status: Location,
1019 reg: Location,
1020 dst: Location,
1021 ) -> Result<(), CompileError> {
1022 match (sz, status, reg, dst) {
1023 (Size::S32, Location::GPR(status), Location::GPR(reg), Location::GPR(dst)) => {
1024 dynasm!(self ; stlxr W(status), W(reg), [X(dst)]);
1025 }
1026 (Size::S64, Location::GPR(status), Location::GPR(reg), Location::GPR(dst)) => {
1027 dynasm!(self ; stlxr W(status), X(reg), [X(dst)]);
1028 }
1029 _ => codegen_error!("singlepass can't emit STLXR {:?}, {:?}", reg, dst),
1030 }
1031 Ok(())
1032 }
1033 fn emit_stlxrb(
1034 &mut self,
1035 _sz: Size,
1036 status: Location,
1037 reg: Location,
1038 dst: Location,
1039 ) -> Result<(), CompileError> {
1040 match (status, reg, dst) {
1041 (Location::GPR(status), Location::GPR(reg), Location::GPR(dst)) => {
1042 dynasm!(self ; stlxrb W(status), W(reg), [X(dst)]);
1043 }
1044 _ => codegen_error!("singlepass can't emit STLXRB {:?}, {:?}", reg, dst),
1045 }
1046 Ok(())
1047 }
1048 fn emit_stlxrh(
1049 &mut self,
1050 _sz: Size,
1051 status: Location,
1052 reg: Location,
1053 dst: Location,
1054 ) -> Result<(), CompileError> {
1055 match (status, reg, dst) {
1056 (Location::GPR(status), Location::GPR(reg), Location::GPR(dst)) => {
1057 dynasm!(self ; stlxrh W(status), W(reg), [X(dst)]);
1058 }
1059 _ => codegen_error!("singlepass can't emit STLXRH {:?}, {:?}", reg, dst),
1060 }
1061 Ok(())
1062 }
1063
1064 fn emit_mov(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1065 match (sz, src, dst) {
1066 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1067 dynasm!(self ; mov X(dst), X(src));
1068 }
1069 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1070 dynasm!(self ; mov W(dst), W(src));
1071 }
1072 (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
1073 dynasm!(self ; mov V(dst).D[0], V(src).D[0]);
1074 }
1075 (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
1076 dynasm!(self ; mov V(dst).S[0], V(src).S[0]);
1077 }
1078 (Size::S64, Location::GPR(src), Location::SIMD(dst)) => {
1079 dynasm!(self ; mov V(dst).D[0], X(src));
1080 }
1081 (Size::S32, Location::GPR(src), Location::SIMD(dst)) => {
1082 dynasm!(self ; mov V(dst).S[0], W(src));
1083 }
1084 (Size::S64, Location::SIMD(src), Location::GPR(dst)) => {
1085 dynasm!(self ; mov X(dst), V(src).D[0]);
1086 }
1087 (Size::S32, Location::SIMD(src), Location::GPR(dst)) => {
1088 dynasm!(self ; mov W(dst), V(src).S[0]);
1089 }
1090 (Size::S32, Location::Imm32(val), Location::GPR(dst)) => {
1091 if val < 0x1000 || encode_logical_immediate_32bit(val as _).is_some() {
1092 dynasm!(self ; mov W(dst), val);
1093 } else {
1094 dynasm!(self
1095 ; movz W(dst), val
1096 ; movk W(dst), val, lsl #16);
1097 }
1098 }
1099 (Size::S64, Location::Imm32(val), Location::GPR(dst)) => {
1100 if val < 0x1000 {
1101 dynasm!(self ; mov W(dst), val);
1102 } else if encode_logical_immediate_64bit(val as _).is_some() {
1103 dynasm!(self ; mov X(dst), val as _);
1104 } else {
1105 dynasm!(self
1106 ; movz X(dst), val
1107 ; movk X(dst), val, lsl #16);
1108 }
1109 }
1110 (Size::S64, Location::Imm64(val), Location::GPR(dst)) => {
1111 if val < 0x1000 || encode_logical_immediate_64bit(val as _).is_some() {
1112 dynasm!(self ; mov X(dst), val as _);
1113 } else {
1114 dynasm!(self
1115 ; movz X(dst), val as _
1116 ; movk X(dst), val as _, lsl #16
1117 ; movk X(dst), val as _, lsl #32
1118 ; movk X(dst), val as _, lsl #48);
1119 }
1120 }
1121 _ => codegen_error!("singlepass can't emit MOV {:?}, {:?}, {:?}", sz, src, dst),
1122 }
1123 Ok(())
1124 }
1125
1126 fn emit_movn(&mut self, sz: Size, reg: Location, val: u32) -> Result<(), CompileError> {
1127 match (sz, reg) {
1128 (Size::S32, Location::GPR(reg)) => {
1129 dynasm!(self ; movn W(reg), val);
1130 }
1131 (Size::S64, Location::GPR(reg)) => {
1132 dynasm!(self ; movn X(reg), val);
1133 }
1134 _ => codegen_error!("singlepass can't emit MOVN"),
1135 }
1136 Ok(())
1137 }
1138 fn emit_movz(&mut self, reg: Location, val: u32) -> Result<(), CompileError> {
1139 match reg {
1140 Location::GPR(reg) => {
1141 dynasm!(self ; movz W(reg), val);
1142 }
1143 _ => codegen_error!("singlepass can't emit MOVZ"),
1144 }
1145 Ok(())
1146 }
1147 fn emit_movk(&mut self, reg: Location, val: u32, shift: u32) -> Result<(), CompileError> {
1148 match reg {
1149 Location::GPR(reg) => {
1150 dynasm!(self ; movk X(reg), val, LSL shift);
1151 }
1152 _ => codegen_error!("singlepass can't emit MOVK"),
1153 }
1154 Ok(())
1155 }
1156
1157 fn emit_mov_imm(&mut self, dst: Location, val: u64) -> Result<(), CompileError> {
1158 match dst {
1159 Location::GPR(dst) => {
1160 let offset = val.trailing_zeros() & 48;
1161 let masked = 0xffff & (val >> offset);
1162 if (masked << offset) == val {
1163 dynasm!(self ; movz X(dst), masked as u32, LSL offset);
1164 } else if val >> 16 == 0xffff_ffff_ffff {
1165 let val: u16 = !((val & 0xffff) as u16);
1166 dynasm!(self ; movn X(dst), val as u32);
1167 } else if val >> 16 == 0xffff {
1168 let val: u16 = !((val & 0xffff) as u16);
1169 dynasm!(self ; movn W(dst), val as u32);
1170 } else {
1171 dynasm!(self ; movz W(dst), (val&0xffff) as u32);
1172 let val = val >> 16;
1173 if val != 0 {
1174 dynasm!(self ; movk X(dst), (val&0xffff) as u32, LSL 16);
1175 let val = val >> 16;
1176 if val != 0 {
1177 dynasm!(self ; movk X(dst), (val&0xffff) as u32, LSL 32);
1178 let val = val >> 16;
1179 if val != 0 {
1180 dynasm!(self ; movk X(dst), (val&0xffff) as u32, LSL 48);
1181 }
1182 }
1183 }
1184 }
1185 }
1186 _ => codegen_error!("singlepass can't emit MOVW {:?}", dst),
1187 }
1188 Ok(())
1189 }
1190
1191 fn emit_add(
1192 &mut self,
1193 sz: Size,
1194 src1: Location,
1195 src2: Location,
1196 dst: Location,
1197 ) -> Result<(), CompileError> {
1198 match (sz, src1, src2, dst) {
1199 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1200 dynasm!(self ; add XSP(dst), XSP(src1), X(src2), UXTX);
1201 }
1202 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1203 dynasm!(self ; add WSP(dst), WSP(src1), W(src2), UXTX);
1204 }
1205 (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1206 | (Size::S64, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1207 dynasm!(self ; add XSP(dst), XSP(src1), imm as _);
1208 }
1209 (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
1210 | (Size::S64, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
1211 if imm >= 0x1000 {
1212 unreachable!();
1213 }
1214 dynasm!(self ; add XSP(dst), XSP(src1), imm);
1215 }
1216 (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst))
1217 | (Size::S64, Location::Imm64(imm), Location::GPR(src1), Location::GPR(dst)) => {
1218 if imm >= 0x1000 {
1219 unreachable!();
1220 }
1221 let imm = imm as u32;
1222 dynasm!(self ; add XSP(dst), XSP(src1), imm);
1223 }
1224 (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1225 | (Size::S32, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1226 dynasm!(self ; add WSP(dst), WSP(src1), imm as u32);
1227 }
1228 (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
1229 | (Size::S32, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
1230 if imm >= 0x1000 {
1231 unreachable!();
1232 }
1233 dynasm!(self ; add WSP(dst), WSP(src1), imm);
1234 }
1235 _ => codegen_error!(
1236 "singlepass can't emit ADD {:?} {:?} {:?} {:?}",
1237 sz,
1238 src1,
1239 src2,
1240 dst
1241 ),
1242 }
1243 Ok(())
1244 }
1245 fn emit_sub(
1246 &mut self,
1247 sz: Size,
1248 src1: Location,
1249 src2: Location,
1250 dst: Location,
1251 ) -> Result<(), CompileError> {
1252 match (sz, src1, src2, dst) {
1253 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1254 dynasm!(self; sub XSP(dst), XSP(src1), X(src2), UXTX 0);
1255 }
1256 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1257 dynasm!(self ; sub WSP(dst), WSP(src1), W(src2), UXTX 0);
1258 }
1259 (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1260 dynasm!(self ; sub XSP(dst), XSP(src1), imm as u32);
1261 }
1262 (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1263 dynasm!(self ; sub WSP(dst), WSP(src1), imm as u32);
1264 }
1265 (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1266 if imm >= 0x1000 {
1267 unreachable!();
1268 }
1269 dynasm!(self ; sub WSP(dst), WSP(src1), imm);
1270 }
1271 (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1272 if imm >= 0x1000 {
1273 unreachable!();
1274 }
1275 dynasm!(self ; sub XSP(dst), XSP(src1), imm);
1276 }
1277 (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
1278 if imm >= 0x1000 {
1279 unreachable!();
1280 }
1281 dynasm!(self ; sub XSP(dst), XSP(src1), imm as u32);
1282 }
1283 _ => codegen_error!(
1284 "singlepass can't emit SUB {:?} {:?} {:?} {:?}",
1285 sz,
1286 src1,
1287 src2,
1288 dst
1289 ),
1290 }
1291 Ok(())
1292 }
1293 fn emit_mul(
1294 &mut self,
1295 sz: Size,
1296 src1: Location,
1297 src2: Location,
1298 dst: Location,
1299 ) -> Result<(), CompileError> {
1300 match (sz, src1, src2, dst) {
1301 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1302 dynasm!(self ; mul X(dst), X(src1), X(src2));
1303 }
1304 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1305 dynasm!(self ; mul W(dst), W(src1), W(src2));
1306 }
1307 _ => codegen_error!(
1308 "singlepass can't emit MUL {:?} {:?} {:?} {:?}",
1309 sz,
1310 src1,
1311 src2,
1312 dst
1313 ),
1314 }
1315 Ok(())
1316 }
1317 fn emit_adds(
1318 &mut self,
1319 sz: Size,
1320 src1: Location,
1321 src2: Location,
1322 dst: Location,
1323 ) -> Result<(), CompileError> {
1324 match (sz, src1, src2, dst) {
1325 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1326 dynasm!(self ; adds X(dst), X(src1), X(src2));
1327 }
1328 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1329 dynasm!(self ; adds W(dst), W(src1), W(src2));
1330 }
1331 (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1332 | (Size::S64, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1333 dynasm!(self ; adds X(dst), XSP(src1), imm as u32);
1334 }
1335 (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
1336 | (Size::S64, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
1337 if imm >= 0x1000 {
1338 codegen_error!("singlepass ADD.S with imm too large {}", imm);
1339 }
1340 dynasm!(self ; adds X(dst), XSP(src1), imm);
1341 }
1342 (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1343 | (Size::S32, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1344 dynasm!(self ; adds W(dst), WSP(src1), imm as u32);
1345 }
1346 (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
1347 | (Size::S32, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
1348 if imm >= 0x1000 {
1349 codegen_error!("singlepass ADD.S with imm too large {}", imm);
1350 }
1351 dynasm!(self ; adds W(dst), WSP(src1), imm);
1352 }
1353 _ => codegen_error!(
1354 "singlepass can't emit ADD.S {:?} {:?} {:?} {:?}",
1355 sz,
1356 src1,
1357 src2,
1358 dst
1359 ),
1360 }
1361 Ok(())
1362 }
1363 fn emit_subs(
1364 &mut self,
1365 sz: Size,
1366 src1: Location,
1367 src2: Location,
1368 dst: Location,
1369 ) -> Result<(), CompileError> {
1370 match (sz, src1, src2, dst) {
1371 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1372 dynasm!(self ; subs X(dst), X(src1), X(src2));
1373 }
1374 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1375 dynasm!(self ; subs W(dst), W(src1), W(src2));
1376 }
1377 (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1378 dynasm!(self ; subs X(dst), XSP(src1), imm as u32);
1379 }
1380 (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1381 dynasm!(self ; subs W(dst), WSP(src1), imm as u32);
1382 }
1383 _ => codegen_error!(
1384 "singlepass can't emit SUB.S {:?} {:?} {:?} {:?}",
1385 sz,
1386 src1,
1387 src2,
1388 dst
1389 ),
1390 }
1391 Ok(())
1392 }
1393 fn emit_add_lsl(
1394 &mut self,
1395 sz: Size,
1396 src1: Location,
1397 src2: Location,
1398 lsl: u32,
1399 dst: Location,
1400 ) -> Result<(), CompileError> {
1401 match (sz, src1, src2, dst) {
1402 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1403 dynasm!(self ; add X(dst), X(src1), X(src2), LSL lsl);
1404 }
1405 _ => codegen_error!(
1406 "singlepass can't emit LSL {:?} {:?} {:?} {:?} LSL {:?}",
1407 sz,
1408 src1,
1409 src2,
1410 dst,
1411 lsl
1412 ),
1413 }
1414 Ok(())
1415 }
1416
1417 fn emit_cmp(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1418 match (sz, src, dst) {
1419 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1420 dynasm!(self ; cmp X(dst), X(src));
1421 }
1422 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1423 dynasm!(self ; cmp W(dst), W(src));
1424 }
1425 (Size::S64, Location::Imm8(imm), Location::GPR(dst)) => {
1426 dynasm!(self ; cmp XSP(dst), imm as u32);
1427 }
1428 (Size::S64, Location::Imm32(imm), Location::GPR(dst)) => {
1429 if imm >= 0x1000 {
1430 codegen_error!("singlepass CMP with imm too large {}", imm);
1431 }
1432 dynasm!(self ; cmp XSP(dst), imm as u32);
1433 }
1434 (Size::S64, Location::Imm64(imm), Location::GPR(dst)) => {
1435 if imm >= 0x1000 {
1436 codegen_error!("singlepass CMP with imm too large {}", imm);
1437 }
1438 dynasm!(self ; cmp XSP(dst), imm as u32);
1439 }
1440 (Size::S32, Location::Imm8(imm), Location::GPR(dst)) => {
1441 dynasm!(self ; cmp WSP(dst), imm as u32);
1442 }
1443 (Size::S32, Location::Imm32(imm), Location::GPR(dst)) => {
1444 if imm >= 0x1000 {
1445 codegen_error!("singlepass CMP with imm too large {}", imm);
1446 }
1447 dynasm!(self ; cmp WSP(dst), imm as u32);
1448 }
1449 _ => codegen_error!("singlepass can't emit CMP {:?} {:?} {:?}", sz, src, dst),
1450 }
1451 Ok(())
1452 }
1453
1454 fn emit_tst(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1455 match (sz, src, dst) {
1456 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1457 dynasm!(self ; tst X(dst), X(src));
1458 }
1459 (Size::S64, Location::Imm32(imm), Location::GPR(dst)) => {
1460 if encode_logical_immediate_64bit(imm as u64).is_none() {
1461 codegen_error!("singlepass TST with incompatible imm {}", imm);
1462 }
1463 dynasm!(self ; tst X(dst), imm as u64);
1464 }
1465 (Size::S64, Location::Imm64(imm), Location::GPR(dst)) => {
1466 if encode_logical_immediate_64bit(imm as u64).is_none() {
1467 codegen_error!("singlepass TST with incompatible imm {}", imm);
1468 }
1469 dynasm!(self ; tst X(dst), imm as u64);
1470 }
1471 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1472 dynasm!(self ; tst W(dst), W(src));
1473 }
1474 (Size::S32, Location::Imm32(imm), Location::GPR(dst)) => {
1475 if encode_logical_immediate_64bit(imm as u64).is_none() {
1476 codegen_error!("singlepass TST with incompatible imm {}", imm);
1477 }
1478 dynasm!(self ; tst W(dst), imm);
1479 }
1480 _ => codegen_error!("singlepass can't emit TST"),
1481 }
1482 Ok(())
1483 }
1484
1485 fn emit_lsl(
1486 &mut self,
1487 sz: Size,
1488 src1: Location,
1489 src2: Location,
1490 dst: Location,
1491 ) -> Result<(), CompileError> {
1492 match (sz, src1, src2, dst) {
1493 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1494 dynasm!(self ; lsl X(dst), X(src1), X(src2));
1495 }
1496 (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1497 if imm > 63 {
1498 codegen_error!("singlepass LSL with incompatible imm {}", imm);
1499 }
1500 let imm = imm as u32;
1501
1502 dynasm!(self ; lsl X(dst), X(src1), imm);
1503 }
1504 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1505 dynasm!(self ; lsl W(dst), W(src1), W(src2));
1506 }
1507 (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1508 | (Size::S64, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1509 if imm > 63 {
1510 codegen_error!("singlepass LSL with incompatible imm {}", imm);
1511 }
1512 dynasm!(self ; lsl X(dst), X(src1), imm as u32);
1513 }
1514 (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst))
1515 | (Size::S64, Location::Imm64(imm), Location::GPR(src1), Location::GPR(dst)) => {
1516 if imm > 63 {
1517 codegen_error!("singlepass LSL with incompatible imm {}", imm);
1518 }
1519 dynasm!(self ; lsl X(dst), X(src1), imm as u32);
1520 }
1521 (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1522 | (Size::S32, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1523 if imm > 31 {
1524 codegen_error!("singlepass LSL with incompatible imm {}", imm);
1525 }
1526 dynasm!(self ; lsl W(dst), W(src1), imm as u32);
1527 }
1528 (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
1529 | (Size::S32, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
1530 if imm > 31 {
1531 codegen_error!("singlepass LSL with incompatible imm {}", imm);
1532 }
1533 dynasm!(self ; lsl W(dst), W(src1), imm as u32);
1534 }
1535 _ => codegen_error!(
1536 "singlepass can't emit LSL {:?} {:?} {:?} {:?}",
1537 sz,
1538 src1,
1539 src2,
1540 dst
1541 ),
1542 }
1543 Ok(())
1544 }
1545 fn emit_asr(
1546 &mut self,
1547 sz: Size,
1548 src1: Location,
1549 src2: Location,
1550 dst: Location,
1551 ) -> Result<(), CompileError> {
1552 match (sz, src1, src2, dst) {
1553 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1554 dynasm!(self ; asr X(dst), X(src1), X(src2));
1555 }
1556 (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1557 let imm = imm as u32;
1558
1559 if imm == 0 || imm > 63 {
1560 codegen_error!("singlepass ASR with incompatible imm {}", imm);
1561 }
1562 dynasm!(self ; asr X(dst), X(src1), imm);
1563 }
1564 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1565 dynasm!(self ; asr W(dst), W(src1), W(src2));
1566 }
1567 (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1568 | (Size::S64, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1569 if imm == 0 || imm > 63 {
1570 codegen_error!("singlepass ASR with incompatible imm {}", imm);
1571 }
1572 dynasm!(self ; asr X(dst), X(src1), imm as u32);
1573 }
1574 (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst))
1575 | (Size::S64, Location::Imm64(imm), Location::GPR(src1), Location::GPR(dst)) => {
1576 if imm == 0 || imm > 63 {
1577 codegen_error!("singlepass ASR with incompatible imm {}", imm);
1578 }
1579 dynasm!(self ; asr X(dst), X(src1), imm as u32);
1580 }
1581 (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1582 | (Size::S32, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1583 if imm == 0 || imm > 31 {
1584 codegen_error!("singlepass ASR with incompatible imm {}", imm);
1585 }
1586 dynasm!(self ; asr W(dst), W(src1), imm as u32);
1587 }
1588 (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
1589 | (Size::S32, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
1590 if imm == 0 || imm > 31 {
1591 codegen_error!("singlepass ASR with incompatible imm {}", imm);
1592 }
1593 dynasm!(self ; asr W(dst), W(src1), imm as u32);
1594 }
1595 _ => codegen_error!(
1596 "singlepass can't emit ASR {:?} {:?} {:?} {:?}",
1597 sz,
1598 src1,
1599 src2,
1600 dst
1601 ),
1602 }
1603 Ok(())
1604 }
1605 fn emit_lsr(
1606 &mut self,
1607 sz: Size,
1608 src1: Location,
1609 src2: Location,
1610 dst: Location,
1611 ) -> Result<(), CompileError> {
1612 match (sz, src1, src2, dst) {
1613 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1614 dynasm!(self ; lsr X(dst), X(src1), X(src2));
1615 }
1616 (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1617 let imm = imm as u32;
1618
1619 if imm == 0 || imm > 63 {
1620 codegen_error!("singlepass LSR with incompatible imm {}", imm);
1621 }
1622 dynasm!(self ; lsr X(dst), X(src1), imm);
1623 }
1624 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1625 dynasm!(self ; lsr W(dst), W(src1), W(src2));
1626 }
1627 (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1628 | (Size::S64, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1629 if imm == 0 || imm > 63 {
1630 codegen_error!("singlepass LSR with incompatible imm {}", imm);
1631 }
1632 dynasm!(self ; lsr X(dst), X(src1), imm as u32);
1633 }
1634 (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst))
1635 | (Size::S64, Location::Imm64(imm), Location::GPR(src1), Location::GPR(dst)) => {
1636 if imm == 0 || imm > 63 {
1637 codegen_error!("singlepass LSR with incompatible imm {}", imm);
1638 }
1639 dynasm!(self ; lsr X(dst), X(src1), imm as u32);
1640 }
1641 (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1642 | (Size::S32, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1643 if imm == 0 || imm > 31 {
1644 codegen_error!("singlepass LSR with incompatible imm {}", imm);
1645 }
1646 dynasm!(self ; lsr W(dst), W(src1), imm as u32);
1647 }
1648 (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
1649 | (Size::S32, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
1650 if imm == 0 || imm > 31 {
1651 codegen_error!("singlepass LSR with incompatible imm {}", imm);
1652 }
1653 dynasm!(self ; lsr W(dst), W(src1), imm as u32);
1654 }
1655 _ => codegen_error!(
1656 "singlepass can't emit LSR {:?} {:?} {:?} {:?}",
1657 sz,
1658 src1,
1659 src2,
1660 dst
1661 ),
1662 }
1663 Ok(())
1664 }
1665 fn emit_ror(
1666 &mut self,
1667 sz: Size,
1668 src1: Location,
1669 src2: Location,
1670 dst: Location,
1671 ) -> Result<(), CompileError> {
1672 match (sz, src1, src2, dst) {
1673 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1674 dynasm!(self ; ror X(dst), X(src1), X(src2));
1675 }
1676 (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
1677 | (Size::S64, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
1678 let imm = imm as u32;
1679
1680 if imm == 0 || imm > 63 {
1681 codegen_error!("singlepass ROR with incompatible imm {}", imm);
1682 }
1683 dynasm!(self ; ror X(dst), X(src1), imm);
1684 }
1685 (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst))
1686 | (Size::S64, Location::Imm64(imm), Location::GPR(src1), Location::GPR(dst)) => {
1687 let imm = imm as u32;
1688
1689 if imm == 0 || imm > 63 {
1690 codegen_error!("singlepass ROR with incompatible imm {}", imm);
1691 }
1692 dynasm!(self ; ror X(dst), X(src1), imm);
1693 }
1694 (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
1695 | (Size::S32, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
1696 if imm == 0 || imm > 31 {
1697 codegen_error!("singlepass ROR with incompatible imm {}", imm);
1698 }
1699 dynasm!(self ; ror W(dst), W(src1), imm as u32);
1700 }
1701 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1702 dynasm!(self ; ror W(dst), W(src1), W(src2));
1703 }
1704 (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1705 | (Size::S64, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1706 if imm == 0 || imm > 63 {
1707 codegen_error!("singlepass ROR with incompatible imm {}", imm);
1708 }
1709 dynasm!(self ; ror X(dst), X(src1), imm as u32);
1710 }
1711 (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1712 | (Size::S32, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1713 if imm == 0 || imm > 31 {
1714 codegen_error!("singlepass ROR with incompatible imm {}", imm);
1715 }
1716 dynasm!(self ; ror W(dst), W(src1), imm as u32);
1717 }
1718 _ => codegen_error!(
1719 "singlepass can't emit ROR {:?} {:?} {:?} {:?}",
1720 sz,
1721 src1,
1722 src2,
1723 dst
1724 ),
1725 }
1726 Ok(())
1727 }
1728
1729 fn emit_or(
1730 &mut self,
1731 sz: Size,
1732 src1: Location,
1733 src2: Location,
1734 dst: Location,
1735 ) -> Result<(), CompileError> {
1736 match (sz, src1, src2, dst) {
1737 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1738 dynasm!(self ; orr X(dst), X(src1), X(src2));
1739 }
1740 (Size::S64, Location::GPR(src1), Location::Imm64(src2), Location::GPR(dst)) => {
1741 let src2 = src2 as u64;
1742
1743 if encode_logical_immediate_64bit(src2 as u64).is_none() {
1744 codegen_error!("singlepass OR with incompatible imm {}", src2);
1745 }
1746 dynasm!(self ; orr XSP(dst), X(src1), src2);
1747 }
1748 (Size::S32, Location::GPR(src1), Location::Imm32(src2), Location::GPR(dst)) => {
1749 let src2 = src2 as u32;
1750
1751 if encode_logical_immediate_32bit(src2).is_none() {
1752 codegen_error!("singlepass OR with incompatible imm {}", src2);
1753 }
1754 dynasm!(self ; orr WSP(dst), W(src1), src2);
1755 }
1756 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1757 dynasm!(self ; orr W(dst), W(src1), W(src2));
1758 }
1759 _ => codegen_error!(
1760 "singlepass can't emit OR {:?} {:?} {:?} {:?}",
1761 sz,
1762 src1,
1763 src2,
1764 dst
1765 ),
1766 }
1767 Ok(())
1768 }
1769 fn emit_and(
1770 &mut self,
1771 sz: Size,
1772 src1: Location,
1773 src2: Location,
1774 dst: Location,
1775 ) -> Result<(), CompileError> {
1776 match (sz, src1, src2, dst) {
1777 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1778 dynasm!(self ; and X(dst), X(src1), X(src2));
1779 }
1780 (Size::S64, Location::GPR(src1), Location::Imm64(src2), Location::GPR(dst)) => {
1781 let src2 = src2 as u64;
1782
1783 if encode_logical_immediate_64bit(src2 as u64).is_none() {
1784 codegen_error!("singlepass AND with incompatible imm {}", src2);
1785 }
1786 dynasm!(self ; and XSP(dst), X(src1), src2);
1787 }
1788 (Size::S32, Location::GPR(src1), Location::Imm32(src2), Location::GPR(dst)) => {
1789 let src2 = src2 as u32;
1790
1791 if encode_logical_immediate_32bit(src2).is_none() {
1792 codegen_error!("singlepass AND with incompatible imm {}", src2);
1793 }
1794 dynasm!(self ; and WSP(dst), W(src1), src2);
1795 }
1796 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1797 dynasm!(self ; and W(dst), W(src1), W(src2));
1798 }
1799 _ => codegen_error!(
1800 "singlepass can't emit AND {:?} {:?} {:?} {:?}",
1801 sz,
1802 src1,
1803 src2,
1804 dst
1805 ),
1806 }
1807 Ok(())
1808 }
1809 fn emit_eor(
1810 &mut self,
1811 sz: Size,
1812 src1: Location,
1813 src2: Location,
1814 dst: Location,
1815 ) -> Result<(), CompileError> {
1816 match (sz, src1, src2, dst) {
1817 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1818 dynasm!(self ; eor X(dst), X(src1), X(src2));
1819 }
1820 (Size::S64, Location::GPR(src1), Location::Imm64(src2), Location::GPR(dst)) => {
1821 let src2 = src2 as u64;
1822
1823 if encode_logical_immediate_64bit(src2 as u64).is_none() {
1824 codegen_error!("singlepass EOR with incompatible imm {}", src2);
1825 }
1826 dynasm!(self ; eor XSP(dst), X(src1), src2);
1827 }
1828 (Size::S32, Location::GPR(src1), Location::Imm32(src2), Location::GPR(dst)) => {
1829 let src2 = src2 as u32;
1830
1831 if encode_logical_immediate_32bit(src2).is_none() {
1832 codegen_error!("singlepass EOR with incompatible imm {}", src2);
1833 }
1834 dynasm!(self ; eor WSP(dst), W(src1), src2);
1835 }
1836 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1837 dynasm!(self ; eor W(dst), W(src1), W(src2));
1838 }
1839 _ => codegen_error!(
1840 "singlepass can't emit EOR {:?} {:?} {:?} {:?}",
1841 sz,
1842 src1,
1843 src2,
1844 dst
1845 ),
1846 }
1847 Ok(())
1848 }
1849
1850 fn emit_bfc(
1851 &mut self,
1852 sz: Size,
1853 lsb: u32,
1854 width: u32,
1855 dst: Location,
1856 ) -> Result<(), CompileError> {
1857 match (sz, dst) {
1858 (Size::S32, Location::GPR(dst)) => {
1859 dynasm!(self ; bfc W(dst), lsb, width);
1860 }
1861 (Size::S64, Location::GPR(dst)) => {
1862 dynasm!(self ; bfc X(dst), lsb, width);
1863 }
1864 _ => codegen_error!("singlepass can't emit BFC"),
1865 }
1866 Ok(())
1867 }
1868 fn emit_bfi(
1869 &mut self,
1870 sz: Size,
1871 src: Location,
1872 lsb: u32,
1873 width: u32,
1874 dst: Location,
1875 ) -> Result<(), CompileError> {
1876 match (sz, src, dst) {
1877 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1878 dynasm!(self ; bfi W(dst), W(src), lsb, width);
1879 }
1880 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1881 dynasm!(self ; bfi X(dst), X(src), lsb, width);
1882 }
1883 _ => codegen_error!("singlepass can't emit BFI"),
1884 }
1885 Ok(())
1886 }
1887
1888 fn emit_udiv(
1889 &mut self,
1890 sz: Size,
1891 src1: Location,
1892 src2: Location,
1893 dst: Location,
1894 ) -> Result<(), CompileError> {
1895 match (sz, src1, src2, dst) {
1896 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1897 dynasm!(self ; udiv W(dst), W(src1), W(src2));
1898 }
1899 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1900 dynasm!(self ; udiv X(dst), X(src1), X(src2));
1901 }
1902 _ => codegen_error!(
1903 "singlepass can't emit UDIV {:?} {:?} {:?} {:?}",
1904 sz,
1905 src1,
1906 src2,
1907 dst
1908 ),
1909 }
1910 Ok(())
1911 }
1912 fn emit_sdiv(
1913 &mut self,
1914 sz: Size,
1915 src1: Location,
1916 src2: Location,
1917 dst: Location,
1918 ) -> Result<(), CompileError> {
1919 match (sz, src1, src2, dst) {
1920 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1921 dynasm!(self ; sdiv W(dst), W(src1), W(src2));
1922 }
1923 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1924 dynasm!(self ; sdiv X(dst), X(src1), X(src2));
1925 }
1926 _ => codegen_error!(
1927 "singlepass can't emit UDIV {:?} {:?} {:?} {:?}",
1928 sz,
1929 src1,
1930 src2,
1931 dst
1932 ),
1933 }
1934 Ok(())
1935 }
1936
1937 fn emit_msub(
1939 &mut self,
1940 sz: Size,
1941 a: Location,
1942 b: Location,
1943 c: Location,
1944 dst: Location,
1945 ) -> Result<(), CompileError> {
1946 match (sz, a, b, c, dst) {
1947 (
1948 Size::S32,
1949 Location::GPR(a),
1950 Location::GPR(b),
1951 Location::GPR(c),
1952 Location::GPR(dst),
1953 ) => {
1954 dynasm!(self ; msub W(dst), W(a), W(b), W(c));
1955 }
1956 (
1957 Size::S64,
1958 Location::GPR(a),
1959 Location::GPR(b),
1960 Location::GPR(c),
1961 Location::GPR(dst),
1962 ) => {
1963 dynasm!(self ; msub X(dst), X(a), X(b), X(c));
1964 }
1965 _ => codegen_error!(
1966 "singlepass can't emit msub {:?} {:?} {:?} {:?} {:?}",
1967 sz,
1968 a,
1969 b,
1970 c,
1971 dst
1972 ),
1973 }
1974 Ok(())
1975 }
1976
1977 fn emit_sxtb(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1978 match (sz, src, dst) {
1979 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1980 dynasm!(self ; sxtb W(dst), W(src));
1981 }
1982 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1983 dynasm!(self ; sxtb X(dst), W(src));
1984 }
1985 _ => codegen_error!("singlepass can't emit SXTB {:?} {:?} {:?}", sz, src, dst),
1986 }
1987 Ok(())
1988 }
1989 fn emit_sxth(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1990 match (sz, src, dst) {
1991 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1992 dynasm!(self ; sxth W(dst), W(src));
1993 }
1994 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1995 dynasm!(self ; sxth X(dst), W(src));
1996 }
1997 _ => codegen_error!("singlepass can't emit SXTH {:?} {:?} {:?}", sz, src, dst),
1998 }
1999 Ok(())
2000 }
2001 fn emit_sxtw(&mut self, _sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2002 match (src, dst) {
2003 (Location::GPR(src), Location::GPR(dst)) => {
2004 dynasm!(self ; sxtw X(dst), W(src));
2005 }
2006 _ => codegen_error!("singlepass can't emit SXTW {:?} {:?}", src, dst),
2007 }
2008 Ok(())
2009 }
2010 fn emit_uxtb(&mut self, _sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2011 match (src, dst) {
2012 (Location::GPR(src), Location::GPR(dst)) => {
2013 dynasm!(self ; uxtb W(dst), W(src));
2014 }
2015 _ => codegen_error!("singlepass can't emit UXTB {:?} {:?}", src, dst),
2016 }
2017 Ok(())
2018 }
2019 fn emit_uxth(&mut self, _sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2020 match (src, dst) {
2021 (Location::GPR(src), Location::GPR(dst)) => {
2022 dynasm!(self ; uxth W(dst), W(src));
2023 }
2024 _ => codegen_error!("singlepass can't emit UXTH {:?} {:?}", src, dst),
2025 }
2026 Ok(())
2027 }
2028
2029 fn emit_cset(&mut self, sz: Size, dst: Location, cond: Condition) -> Result<(), CompileError> {
2030 match (sz, dst) {
2031 (Size::S32, Location::GPR(reg)) => match cond {
2032 Condition::Eq => dynasm!(self ; cset W(reg), eq),
2033 Condition::Ne => dynasm!(self ; cset W(reg), ne),
2034 Condition::Cs => dynasm!(self ; cset W(reg), cs),
2035 Condition::Cc => dynasm!(self ; cset W(reg), cc),
2036 Condition::Mi => dynasm!(self ; cset W(reg), mi),
2037 Condition::Pl => dynasm!(self ; cset W(reg), pl),
2038 Condition::Vs => dynasm!(self ; cset W(reg), vs),
2039 Condition::Vc => dynasm!(self ; cset W(reg), vc),
2040 Condition::Hi => dynasm!(self ; cset W(reg), hi),
2041 Condition::Ls => dynasm!(self ; cset W(reg), ls),
2042 Condition::Ge => dynasm!(self ; cset W(reg), ge),
2043 Condition::Lt => dynasm!(self ; cset W(reg), lt),
2044 Condition::Gt => dynasm!(self ; cset W(reg), gt),
2045 Condition::Le => dynasm!(self ; cset W(reg), le),
2046 Condition::Al => dynasm!(self ; cset W(reg), al),
2047 },
2048 (Size::S64, Location::GPR(reg)) => match cond {
2049 Condition::Eq => dynasm!(self ; cset X(reg), eq),
2050 Condition::Ne => dynasm!(self ; cset X(reg), ne),
2051 Condition::Cs => dynasm!(self ; cset X(reg), cs),
2052 Condition::Cc => dynasm!(self ; cset X(reg), cc),
2053 Condition::Mi => dynasm!(self ; cset X(reg), mi),
2054 Condition::Pl => dynasm!(self ; cset X(reg), pl),
2055 Condition::Vs => dynasm!(self ; cset X(reg), vs),
2056 Condition::Vc => dynasm!(self ; cset X(reg), vc),
2057 Condition::Hi => dynasm!(self ; cset X(reg), hi),
2058 Condition::Ls => dynasm!(self ; cset X(reg), ls),
2059 Condition::Ge => dynasm!(self ; cset X(reg), ge),
2060 Condition::Lt => dynasm!(self ; cset X(reg), lt),
2061 Condition::Gt => dynasm!(self ; cset X(reg), gt),
2062 Condition::Le => dynasm!(self ; cset X(reg), le),
2063 Condition::Al => dynasm!(self ; cset X(reg), al),
2064 },
2065 _ => codegen_error!("singlepass can't emit CSET {:?} {:?} {:?}", sz, dst, cond),
2066 }
2067 Ok(())
2068 }
2069 fn emit_csetm(&mut self, sz: Size, dst: Location, cond: Condition) -> Result<(), CompileError> {
2070 match (sz, dst) {
2071 (Size::S32, Location::GPR(reg)) => match cond {
2072 Condition::Eq => dynasm!(self ; csetm W(reg), eq),
2073 Condition::Ne => dynasm!(self ; csetm W(reg), ne),
2074 Condition::Cs => dynasm!(self ; csetm W(reg), cs),
2075 Condition::Cc => dynasm!(self ; csetm W(reg), cc),
2076 Condition::Mi => dynasm!(self ; csetm W(reg), mi),
2077 Condition::Pl => dynasm!(self ; csetm W(reg), pl),
2078 Condition::Vs => dynasm!(self ; csetm W(reg), vs),
2079 Condition::Vc => dynasm!(self ; csetm W(reg), vc),
2080 Condition::Hi => dynasm!(self ; csetm W(reg), hi),
2081 Condition::Ls => dynasm!(self ; csetm W(reg), ls),
2082 Condition::Ge => dynasm!(self ; csetm W(reg), ge),
2083 Condition::Lt => dynasm!(self ; csetm W(reg), lt),
2084 Condition::Gt => dynasm!(self ; csetm W(reg), gt),
2085 Condition::Le => dynasm!(self ; csetm W(reg), le),
2086 Condition::Al => dynasm!(self ; csetm W(reg), al),
2087 },
2088 (Size::S64, Location::GPR(reg)) => match cond {
2089 Condition::Eq => dynasm!(self ; csetm X(reg), eq),
2090 Condition::Ne => dynasm!(self ; csetm X(reg), ne),
2091 Condition::Cs => dynasm!(self ; csetm X(reg), cs),
2092 Condition::Cc => dynasm!(self ; csetm X(reg), cc),
2093 Condition::Mi => dynasm!(self ; csetm X(reg), mi),
2094 Condition::Pl => dynasm!(self ; csetm X(reg), pl),
2095 Condition::Vs => dynasm!(self ; csetm X(reg), vs),
2096 Condition::Vc => dynasm!(self ; csetm X(reg), vc),
2097 Condition::Hi => dynasm!(self ; csetm X(reg), hi),
2098 Condition::Ls => dynasm!(self ; csetm X(reg), ls),
2099 Condition::Ge => dynasm!(self ; csetm X(reg), ge),
2100 Condition::Lt => dynasm!(self ; csetm X(reg), lt),
2101 Condition::Gt => dynasm!(self ; csetm X(reg), gt),
2102 Condition::Le => dynasm!(self ; csetm X(reg), le),
2103 Condition::Al => dynasm!(self ; csetm X(reg), al),
2104 },
2105 _ => codegen_error!("singlepass can't emit CSETM {:?} {:?} {:?}", sz, dst, cond),
2106 }
2107 Ok(())
2108 }
2109 fn emit_cinc(
2110 &mut self,
2111 sz: Size,
2112 src: Location,
2113 dst: Location,
2114 cond: Condition,
2115 ) -> Result<(), CompileError> {
2116 match (sz, src, dst) {
2117 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
2118 match cond {
2119 Condition::Eq => dynasm!(self ; cinc W(dst), W(src), eq),
2120 Condition::Ne => dynasm!(self ; cinc W(dst), W(src), ne),
2121 Condition::Cs => dynasm!(self ; cinc W(dst), W(src), cs),
2122 Condition::Cc => dynasm!(self ; cinc W(dst), W(src), cc),
2123 Condition::Mi => dynasm!(self ; cinc W(dst), W(src), mi),
2124 Condition::Pl => dynasm!(self ; cinc W(dst), W(src), pl),
2125 Condition::Vs => dynasm!(self ; cinc W(dst), W(src), vs),
2126 Condition::Vc => dynasm!(self ; cinc W(dst), W(src), vc),
2127 Condition::Hi => dynasm!(self ; cinc W(dst), W(src), hi),
2128 Condition::Ls => dynasm!(self ; cinc W(dst), W(src), ls),
2129 Condition::Ge => dynasm!(self ; cinc W(dst), W(src), ge),
2130 Condition::Lt => dynasm!(self ; cinc W(dst), W(src), lt),
2131 Condition::Gt => dynasm!(self ; cinc W(dst), W(src), gt),
2132 Condition::Le => dynasm!(self ; cinc W(dst), W(src), le),
2133 Condition::Al => dynasm!(self ; cinc W(dst), W(src), al),
2134 };
2135 }
2136 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
2137 match cond {
2138 Condition::Eq => dynasm!(self ; cinc X(src), X(dst), eq),
2139 Condition::Ne => dynasm!(self ; cinc X(src), X(dst), ne),
2140 Condition::Cs => dynasm!(self ; cinc X(src), X(dst), cs),
2141 Condition::Cc => dynasm!(self ; cinc X(src), X(dst), cc),
2142 Condition::Mi => dynasm!(self ; cinc X(src), X(dst), mi),
2143 Condition::Pl => dynasm!(self ; cinc X(src), X(dst), pl),
2144 Condition::Vs => dynasm!(self ; cinc X(src), X(dst), vs),
2145 Condition::Vc => dynasm!(self ; cinc X(src), X(dst), vc),
2146 Condition::Hi => dynasm!(self ; cinc X(src), X(dst), hi),
2147 Condition::Ls => dynasm!(self ; cinc X(src), X(dst), ls),
2148 Condition::Ge => dynasm!(self ; cinc X(src), X(dst), ge),
2149 Condition::Lt => dynasm!(self ; cinc X(src), X(dst), lt),
2150 Condition::Gt => dynasm!(self ; cinc X(src), X(dst), gt),
2151 Condition::Le => dynasm!(self ; cinc X(src), X(dst), le),
2152 Condition::Al => dynasm!(self ; cinc X(src), X(dst), al),
2153 };
2154 }
2155 _ => codegen_error!("singlepass can't emit CINC"),
2156 }
2157 Ok(())
2158 }
2159
2160 fn emit_clz(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2161 match (sz, src, dst) {
2162 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
2163 dynasm!(self ; clz X(dst), X(src));
2164 }
2165 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
2166 dynasm!(self ; clz W(dst), W(src));
2167 }
2168 _ => codegen_error!("singlepass can't emit CLS {:?} {:?} {:?}", sz, src, dst),
2169 }
2170 Ok(())
2171 }
2172 fn emit_rbit(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2173 match (sz, src, dst) {
2174 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
2175 dynasm!(self ; rbit X(dst), X(src));
2176 }
2177 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
2178 dynasm!(self ; rbit W(dst), W(src));
2179 }
2180 _ => codegen_error!("singlepass can't emit CLS {:?} {:?} {:?}", sz, src, dst),
2181 }
2182 Ok(())
2183 }
2184
2185 fn emit_label(&mut self, label: Label) -> Result<(), CompileError> {
2186 dynasm!(self ; => label);
2187 Ok(())
2188 }
2189 fn emit_load_label(&mut self, reg: GPR, label: Label) -> Result<(), CompileError> {
2190 dynasm!(self ; adr X(reg), =>label);
2191 Ok(())
2192 }
2193
2194 fn emit_b_label(&mut self, label: Label) -> Result<(), CompileError> {
2195 dynasm!(self ; b =>label);
2196 Ok(())
2197 }
2198 fn emit_cbz_label(
2199 &mut self,
2200 sz: Size,
2201 reg: Location,
2202 label: Label,
2203 ) -> Result<(), CompileError> {
2204 match (sz, reg) {
2205 (Size::S32, Location::GPR(reg)) => {
2206 dynasm!(self ; cbz W(reg), =>label);
2207 }
2208 (Size::S64, Location::GPR(reg)) => {
2209 dynasm!(self ; cbz X(reg), =>label);
2210 }
2211 _ => codegen_error!("singlepass can't emit CBZ {:?} {:?} {:?}", sz, reg, label),
2212 }
2213 Ok(())
2214 }
2215 fn emit_cbnz_label(
2216 &mut self,
2217 sz: Size,
2218 reg: Location,
2219 label: Label,
2220 ) -> Result<(), CompileError> {
2221 match (sz, reg) {
2222 (Size::S32, Location::GPR(reg)) => {
2223 dynasm!(self ; cbnz W(reg), =>label);
2224 }
2225 (Size::S64, Location::GPR(reg)) => {
2226 dynasm!(self ; cbnz X(reg), =>label);
2227 }
2228 _ => codegen_error!("singlepass can't emit CBNZ {:?} {:?} {:?}", sz, reg, label),
2229 }
2230 Ok(())
2231 }
2232 fn emit_cbz_label_far(
2233 &mut self,
2234 sz: Size,
2235 reg: Location,
2236 label: Label,
2237 ) -> Result<(), CompileError> {
2238 let near_label: Label = self.get_label();
2239 let continue_label: Label = self.get_label();
2240
2241 match (sz, reg) {
2242 (Size::S32, Location::GPR(reg)) => {
2243 dynasm!(self ; cbz W(reg), => near_label);
2244 dynasm!(self ; b => continue_label);
2245 }
2246 (Size::S64, Location::GPR(reg)) => {
2247 dynasm!(self ; cbz X(reg), => near_label);
2248 dynasm!(self ; b => continue_label);
2249 }
2250 _ => codegen_error!("singlepass can't emit CBZ {:?} {:?} {:?}", sz, reg, label),
2251 }
2252 self.emit_label(near_label)?;
2253 dynasm!(self ; b => label );
2254
2255 self.emit_label(continue_label)?;
2256
2257 Ok(())
2258 }
2259 fn emit_tbz_label(
2260 &mut self,
2261 sz: Size,
2262 reg: Location,
2263 n: u32,
2264 label: Label,
2265 ) -> Result<(), CompileError> {
2266 match (sz, reg) {
2267 (Size::S32, Location::GPR(reg)) => {
2268 dynasm!(self ; tbz W(reg), n, =>label);
2269 }
2270 (Size::S64, Location::GPR(reg)) => {
2271 dynasm!(self ; tbz X(reg), n, =>label);
2272 }
2273 _ => codegen_error!(
2274 "singlepass can't emit TBZ {:?} {:?} {:?} {:?}",
2275 sz,
2276 reg,
2277 n,
2278 label
2279 ),
2280 }
2281 Ok(())
2282 }
2283 fn emit_tbnz_label(
2284 &mut self,
2285 sz: Size,
2286 reg: Location,
2287 n: u32,
2288 label: Label,
2289 ) -> Result<(), CompileError> {
2290 match (sz, reg) {
2291 (Size::S32, Location::GPR(reg)) => {
2292 dynasm!(self ; tbnz W(reg), n, =>label);
2293 }
2294 (Size::S64, Location::GPR(reg)) => {
2295 dynasm!(self ; tbnz X(reg), n, =>label);
2296 }
2297 _ => codegen_error!(
2298 "singlepass can't emit TBNZ {:?} {:?} {:?} {:?}",
2299 sz,
2300 reg,
2301 n,
2302 label
2303 ),
2304 }
2305 Ok(())
2306 }
2307 fn emit_bcond_label(&mut self, condition: Condition, label: Label) -> Result<(), CompileError> {
2308 match condition {
2309 Condition::Eq => dynasm!(self ; b.eq => label),
2310 Condition::Ne => dynasm!(self ; b.ne => label),
2311 Condition::Cs => dynasm!(self ; b.cs => label),
2312 Condition::Cc => dynasm!(self ; b.cc => label),
2313 Condition::Mi => dynasm!(self ; b.mi => label),
2314 Condition::Pl => dynasm!(self ; b.pl => label),
2315 Condition::Vs => dynasm!(self ; b.vs => label),
2316 Condition::Vc => dynasm!(self ; b.vc => label),
2317 Condition::Hi => dynasm!(self ; b.hi => label),
2318 Condition::Ls => dynasm!(self ; b.ls => label),
2319 Condition::Ge => dynasm!(self ; b.ge => label),
2320 Condition::Lt => dynasm!(self ; b.lt => label),
2321 Condition::Gt => dynasm!(self ; b.gt => label),
2322 Condition::Le => dynasm!(self ; b.le => label),
2323 Condition::Al => dynasm!(self ; b => label),
2324 }
2325 Ok(())
2326 }
2327 fn emit_bcond_label_far(
2328 &mut self,
2329 condition: Condition,
2330 label: Label,
2331 ) -> Result<(), CompileError> {
2332 let cont: Label = self.get_label();
2333 match condition {
2334 Condition::Eq => dynasm!(self ; b.ne => cont),
2336 Condition::Ne => dynasm!(self ; b.eq => cont),
2337 Condition::Cs => dynasm!(self ; b.cc => cont),
2338 Condition::Cc => dynasm!(self ; b.cs => cont),
2339 Condition::Mi => dynasm!(self ; b.pl => cont),
2340 Condition::Pl => dynasm!(self ; b.mi => cont),
2341 Condition::Vs => dynasm!(self ; b.vc => cont),
2342 Condition::Vc => dynasm!(self ; b.vs => cont),
2343 Condition::Hi => dynasm!(self ; b.ls => cont),
2344 Condition::Ls => dynasm!(self ; b.hi => cont),
2345 Condition::Ge => dynasm!(self ; b.lt => cont),
2346 Condition::Lt => dynasm!(self ; b.ge => cont),
2347 Condition::Gt => dynasm!(self ; b.le => cont),
2348 Condition::Le => dynasm!(self ; b.gt => cont),
2349 Condition::Al => { }
2350 }
2351 dynasm!(self ; b => label);
2352 self.emit_label(cont)?;
2353 Ok(())
2354 }
2355 fn emit_b_register(&mut self, reg: GPR) -> Result<(), CompileError> {
2356 dynasm!(self ; br X(reg));
2357 Ok(())
2358 }
2359 fn emit_call_label(&mut self, label: Label) -> Result<(), CompileError> {
2360 dynasm!(self ; bl =>label);
2361 Ok(())
2362 }
2363 fn emit_call_register(&mut self, reg: GPR) -> Result<(), CompileError> {
2364 dynasm!(self ; blr X(reg));
2365 Ok(())
2366 }
2367 fn emit_ret(&mut self) -> Result<(), CompileError> {
2368 dynasm!(self ; ret);
2369 Ok(())
2370 }
2371
2372 fn emit_udf(&mut self, payload: u16) -> Result<(), CompileError> {
2373 dynasm!(self ; udf payload as u32);
2374 Ok(())
2375 }
2376 fn emit_dmb(&mut self) -> Result<(), CompileError> {
2377 dynasm!(self ; dmb ish);
2378 Ok(())
2379 }
2380 fn emit_brk(&mut self) -> Result<(), CompileError> {
2381 dynasm!(self ; brk 0);
2382 Ok(())
2383 }
2384
2385 fn emit_fcmp(&mut self, sz: Size, src1: Location, src2: Location) -> Result<(), CompileError> {
2386 match (sz, src1, src2) {
2387 (Size::S32, Location::SIMD(src1), Location::SIMD(src2)) => {
2388 dynasm!(self ; fcmp S(src1), S(src2));
2389 }
2390 (Size::S64, Location::SIMD(src1), Location::SIMD(src2)) => {
2391 dynasm!(self ; fcmp D(src1), D(src2));
2392 }
2393 _ => codegen_error!("singlepass can't emit FCMP {:?} {:?} {:?}", sz, src1, src2),
2394 }
2395 Ok(())
2396 }
2397
2398 fn emit_fneg(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2399 match (sz, src, dst) {
2400 (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2401 dynasm!(self ; fneg S(dst), S(src));
2402 }
2403 (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2404 dynasm!(self ; fneg D(dst), D(src));
2405 }
2406 _ => codegen_error!("singlepass can't emit FNEG {:?} {:?} {:?}", sz, src, dst),
2407 }
2408 Ok(())
2409 }
2410 fn emit_fsqrt(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2411 match (sz, src, dst) {
2412 (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2413 dynasm!(self ; fsqrt S(dst), S(src));
2414 }
2415 (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2416 dynasm!(self ; fsqrt D(dst), D(src));
2417 }
2418 _ => codegen_error!("singlepass can't emit FSQRT {:?} {:?} {:?}", sz, src, dst),
2419 }
2420 Ok(())
2421 }
2422
2423 fn emit_fadd(
2424 &mut self,
2425 sz: Size,
2426 src1: Location,
2427 src2: Location,
2428 dst: Location,
2429 ) -> Result<(), CompileError> {
2430 match (sz, src1, src2, dst) {
2431 (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2432 dynasm!(self ; fadd S(dst), S(src1), S(src2));
2433 }
2434 (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2435 dynasm!(self ; fadd D(dst), D(src1), D(src2));
2436 }
2437 _ => codegen_error!(
2438 "singlepass can't emit FADD {:?} {:?} {:?} {:?}",
2439 sz,
2440 src1,
2441 src2,
2442 dst
2443 ),
2444 }
2445 Ok(())
2446 }
2447 fn emit_fsub(
2448 &mut self,
2449 sz: Size,
2450 src1: Location,
2451 src2: Location,
2452 dst: Location,
2453 ) -> Result<(), CompileError> {
2454 match (sz, src1, src2, dst) {
2455 (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2456 dynasm!(self ; fsub S(dst), S(src1), S(src2));
2457 }
2458 (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2459 dynasm!(self ; fsub D(dst), D(src1), D(src2));
2460 }
2461 _ => codegen_error!(
2462 "singlepass can't emit FSUB {:?} {:?} {:?} {:?}",
2463 sz,
2464 src1,
2465 src2,
2466 dst
2467 ),
2468 }
2469 Ok(())
2470 }
2471 fn emit_fmul(
2472 &mut self,
2473 sz: Size,
2474 src1: Location,
2475 src2: Location,
2476 dst: Location,
2477 ) -> Result<(), CompileError> {
2478 match (sz, src1, src2, dst) {
2479 (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2480 dynasm!(self ; fmul S(dst), S(src1), S(src2));
2481 }
2482 (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2483 dynasm!(self ; fmul D(dst), D(src1), D(src2));
2484 }
2485 _ => codegen_error!(
2486 "singlepass can't emit FMUL {:?} {:?} {:?} {:?}",
2487 sz,
2488 src1,
2489 src2,
2490 dst
2491 ),
2492 }
2493 Ok(())
2494 }
2495 fn emit_fdiv(
2496 &mut self,
2497 sz: Size,
2498 src1: Location,
2499 src2: Location,
2500 dst: Location,
2501 ) -> Result<(), CompileError> {
2502 match (sz, src1, src2, dst) {
2503 (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2504 dynasm!(self ; fdiv S(dst), S(src1), S(src2));
2505 }
2506 (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2507 dynasm!(self ; fdiv D(dst), D(src1), D(src2));
2508 }
2509 _ => codegen_error!(
2510 "singlepass can't emit FDIV {:?} {:?} {:?} {:?}",
2511 sz,
2512 src1,
2513 src2,
2514 dst
2515 ),
2516 }
2517 Ok(())
2518 }
2519
2520 fn emit_fmin(
2521 &mut self,
2522 sz: Size,
2523 src1: Location,
2524 src2: Location,
2525 dst: Location,
2526 ) -> Result<(), CompileError> {
2527 match (sz, src1, src2, dst) {
2528 (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2529 dynasm!(self ; fmin S(dst), S(src1), S(src2));
2530 }
2531 (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2532 dynasm!(self ; fmin D(dst), D(src1), D(src2));
2533 }
2534 _ => codegen_error!(
2535 "singlepass can't emit FMIN {:?} {:?} {:?} {:?}",
2536 sz,
2537 src1,
2538 src2,
2539 dst
2540 ),
2541 }
2542 Ok(())
2543 }
2544 fn emit_fmax(
2545 &mut self,
2546 sz: Size,
2547 src1: Location,
2548 src2: Location,
2549 dst: Location,
2550 ) -> Result<(), CompileError> {
2551 match (sz, src1, src2, dst) {
2552 (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2553 dynasm!(self ; fmax S(dst), S(src1), S(src2));
2554 }
2555 (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2556 dynasm!(self ; fmax D(dst), D(src1), D(src2));
2557 }
2558 _ => codegen_error!(
2559 "singlepass can't emit FMAX {:?} {:?} {:?} {:?}",
2560 sz,
2561 src1,
2562 src2,
2563 dst
2564 ),
2565 }
2566 Ok(())
2567 }
2568
2569 fn emit_frintz(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2570 match (sz, src, dst) {
2571 (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2572 dynasm!(self ; frintz S(dst), S(src));
2573 }
2574 (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2575 dynasm!(self ; frintz D(dst), D(src));
2576 }
2577 _ => codegen_error!("singlepass can't emit FRINTZ {:?} {:?} {:?}", sz, src, dst),
2578 }
2579 Ok(())
2580 }
2581 fn emit_frintn(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2582 match (sz, src, dst) {
2583 (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2584 dynasm!(self ; frintn S(dst), S(src));
2585 }
2586 (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2587 dynasm!(self ; frintn D(dst), D(src));
2588 }
2589 _ => codegen_error!("singlepass can't emit FRINTN {:?} {:?} {:?}", sz, src, dst),
2590 }
2591 Ok(())
2592 }
2593 fn emit_frintm(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2594 match (sz, src, dst) {
2595 (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2596 dynasm!(self ; frintm S(dst), S(src));
2597 }
2598 (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2599 dynasm!(self ; frintm D(dst), D(src));
2600 }
2601 _ => codegen_error!("singlepass can't emit FRINTM {:?} {:?} {:?}", sz, src, dst),
2602 }
2603 Ok(())
2604 }
2605 fn emit_frintp(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2606 match (sz, src, dst) {
2607 (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2608 dynasm!(self ; frintp S(dst), S(src));
2609 }
2610 (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2611 dynasm!(self ; frintp D(dst), D(src));
2612 }
2613 _ => codegen_error!("singlepass can't emit FRINTP {:?} {:?} {:?}", sz, src, dst),
2614 }
2615 Ok(())
2616 }
2617
2618 fn emit_scvtf(
2619 &mut self,
2620 sz_in: Size,
2621 src: Location,
2622 sz_out: Size,
2623 dst: Location,
2624 ) -> Result<(), CompileError> {
2625 match (sz_in, src, sz_out, dst) {
2626 (Size::S32, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
2627 dynasm!(self ; scvtf S(dst), W(src));
2628 }
2629 (Size::S64, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
2630 dynasm!(self ; scvtf S(dst), X(src));
2631 }
2632 (Size::S32, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
2633 dynasm!(self ; scvtf D(dst), W(src));
2634 }
2635 (Size::S64, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
2636 dynasm!(self ; scvtf D(dst), X(src));
2637 }
2638 _ => codegen_error!(
2639 "singlepass can't emit SCVTF {:?} {:?} {:?} {:?}",
2640 sz_in,
2641 src,
2642 sz_out,
2643 dst
2644 ),
2645 }
2646 Ok(())
2647 }
2648 fn emit_ucvtf(
2649 &mut self,
2650 sz_in: Size,
2651 src: Location,
2652 sz_out: Size,
2653 dst: Location,
2654 ) -> Result<(), CompileError> {
2655 match (sz_in, src, sz_out, dst) {
2656 (Size::S32, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
2657 dynasm!(self ; ucvtf S(dst), W(src));
2658 }
2659 (Size::S64, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
2660 dynasm!(self ; ucvtf S(dst), X(src));
2661 }
2662 (Size::S32, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
2663 dynasm!(self ; ucvtf D(dst), W(src));
2664 }
2665 (Size::S64, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
2666 dynasm!(self ; ucvtf D(dst), X(src));
2667 }
2668 _ => codegen_error!(
2669 "singlepass can't emit UCVTF {:?} {:?} {:?} {:?}",
2670 sz_in,
2671 src,
2672 sz_out,
2673 dst
2674 ),
2675 }
2676 Ok(())
2677 }
2678 fn emit_fcvt(&mut self, sz_in: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2679 match (sz_in, src, dst) {
2680 (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2681 dynasm!(self ; fcvt D(dst), S(src));
2682 }
2683 (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2684 dynasm!(self ; fcvt S(dst), D(src));
2685 }
2686 _ => codegen_error!(
2687 "singlepass can't emit UCVTF {:?} {:?} {:?}",
2688 sz_in,
2689 src,
2690 dst
2691 ),
2692 }
2693 Ok(())
2694 }
2695 fn emit_fcvtzs(
2696 &mut self,
2697 sz_in: Size,
2698 src: Location,
2699 sz_out: Size,
2700 dst: Location,
2701 ) -> Result<(), CompileError> {
2702 match (sz_in, src, sz_out, dst) {
2703 (Size::S32, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
2704 dynasm!(self ; fcvtzs W(dst), S(src));
2705 }
2706 (Size::S64, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
2707 dynasm!(self ; fcvtzs W(dst), D(src));
2708 }
2709 (Size::S32, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
2710 dynasm!(self ; fcvtzs X(dst), S(src));
2711 }
2712 (Size::S64, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
2713 dynasm!(self ; fcvtzs X(dst), D(src));
2714 }
2715 _ => codegen_error!(
2716 "singlepass can't emit FCVTZS {:?} {:?} {:?} {:?}",
2717 sz_in,
2718 src,
2719 sz_out,
2720 dst
2721 ),
2722 }
2723 Ok(())
2724 }
2725 fn emit_fcvtzu(
2726 &mut self,
2727 sz_in: Size,
2728 src: Location,
2729 sz_out: Size,
2730 dst: Location,
2731 ) -> Result<(), CompileError> {
2732 match (sz_in, src, sz_out, dst) {
2733 (Size::S32, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
2734 dynasm!(self ; fcvtzu W(dst), S(src));
2735 }
2736 (Size::S64, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
2737 dynasm!(self ; fcvtzu W(dst), D(src));
2738 }
2739 (Size::S32, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
2740 dynasm!(self ; fcvtzu X(dst), S(src));
2741 }
2742 (Size::S64, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
2743 dynasm!(self ; fcvtzu X(dst), D(src));
2744 }
2745 _ => codegen_error!(
2746 "singlepass can't emit FCVTZU {:?} {:?} {:?} {:?}",
2747 sz_in,
2748 src,
2749 sz_out,
2750 dst
2751 ),
2752 }
2753 Ok(())
2754 }
2755
2756 fn emit_read_fpcr(&mut self, reg: GPR) -> Result<(), CompileError> {
2758 dynasm!(self ; mrs X(reg), 0b1_011_0100_0100_000);
2759 Ok(())
2760 }
2761 fn emit_write_fpcr(&mut self, reg: GPR) -> Result<(), CompileError> {
2762 dynasm!(self ; msr 0b1_011_0100_0100_000, X(reg));
2763 Ok(())
2764 }
2765 fn emit_read_fpsr(&mut self, reg: GPR) -> Result<(), CompileError> {
2767 dynasm!(self ; mrs X(reg), 0b1_011_0100_0100_001);
2768 Ok(())
2769 }
2770 fn emit_write_fpsr(&mut self, reg: GPR) -> Result<(), CompileError> {
2771 dynasm!(self ; msr 0b1_011_0100_0100_001, X(reg));
2772 Ok(())
2773 }
2774
2775 fn emit_cnt(&mut self, src: NEON, dst: NEON) -> Result<(), CompileError> {
2776 dynasm!(self ; cnt V(dst).B8, V(src).B8);
2777 Ok(())
2778 }
2779
2780 fn emit_addv(&mut self, src: NEON, dst: NEON) -> Result<(), CompileError> {
2781 dynasm!(self ; addv B(dst), V(src).B8);
2782 Ok(())
2783 }
2784
2785 #[allow(clippy::unit_arg)]
2786 fn emit_fmov(
2787 &mut self,
2788 src_size: Size,
2789 src: Location,
2790 dst_size: Size,
2791 dst: Location,
2792 ) -> Result<(), CompileError> {
2793 let mut err = false;
2794 match (src, dst) {
2795 (AbstractLocation::GPR(src), AbstractLocation::SIMD(dst)) => {
2796 match (src_size, dst_size) {
2797 (Size::S32, Size::S32) => dynasm!(self ; fmov S(dst), W(src)),
2798 (Size::S64, Size::S64) => dynasm!(self ; fmov D(dst), X(src)),
2799 _ => {
2800 err = true;
2801 }
2802 }
2803 }
2804 (AbstractLocation::SIMD(src), AbstractLocation::GPR(dst)) => {
2805 match (src_size, dst_size) {
2806 (Size::S32, Size::S32) => dynasm!(self ; fmov W(dst), S(src)),
2807 (Size::S64, Size::S64) => dynasm!(self ; fmov X(dst), D(src)),
2808 _ => {
2809 err = true;
2810 }
2811 }
2812 }
2813 _ => {
2817 err = true;
2818 }
2819 };
2820
2821 if err {
2822 codegen_error!(
2823 "singlepass can't generate fmov instruction for src_size: {:?}, src: {:?}, dst_size: {:?}, dst: {:?}",
2824 src_size,
2825 src,
2826 dst_size,
2827 dst,
2828 )
2829 }
2830 Ok(())
2831 }
2832}
2833
2834pub fn gen_std_trampoline_arm64(
2835 sig: &FunctionType,
2836 calling_convention: CallingConvention,
2837) -> Result<FunctionBody, CompileError> {
2838 let mut a = Assembler::new(0);
2839
2840 let fptr = GPR::X27;
2841 let args = GPR::X28;
2842
2843 dynasm!(a
2844 ; sub sp, sp, 32
2845 ; stp x29, x30, [sp]
2846 ; stp X(fptr), X(args), [sp, 16]
2847 ; mov x29, sp
2848 ; mov X(fptr), x1
2849 ; mov X(args), x2
2850 );
2851
2852 let stack_args = sig.params().len().saturating_sub(7); let mut stack_offset = stack_args as u32 * 8;
2854 if stack_args > 0 {
2855 if stack_offset % 16 != 0 {
2856 stack_offset += 8;
2857 assert!(stack_offset % 16 == 0);
2858 }
2859 dynasm!(a ; sub sp, sp, stack_offset);
2860 }
2861
2862 let mut caller_stack_offset: i32 = 0;
2865 for (i, param) in sig.params().iter().enumerate() {
2866 let sz = match *param {
2867 Type::I32 | Type::F32 => Size::S32,
2868 Type::I64 | Type::F64 => Size::S64,
2869 Type::ExternRef => Size::S64,
2870 Type::FuncRef => Size::S64,
2871 _ => codegen_error!(
2872 "singlepass unsupported param type for trampoline {:?}",
2873 *param
2874 ),
2875 };
2876 match i {
2877 0..=6 => {
2878 a.emit_ldr(
2879 sz,
2880 Location::GPR(GPR::from_index(i + 1).unwrap()),
2881 Location::Memory(args, (i * 16) as i32),
2882 )?;
2883 }
2884 _ => {
2885 #[allow(clippy::single_match)]
2886 match calling_convention {
2887 CallingConvention::AppleAarch64 => {
2888 let sz = 1
2889 << match sz {
2890 Size::S8 => 0,
2891 Size::S16 => 1,
2892 Size::S32 => 2,
2893 Size::S64 => 3,
2894 };
2895 if sz > 1 && caller_stack_offset & (sz - 1) != 0 {
2897 caller_stack_offset = (caller_stack_offset + (sz - 1)) & !(sz - 1);
2898 }
2899 }
2900 _ => (),
2901 };
2902 a.emit_ldr(
2904 sz,
2905 Location::GPR(GPR::X16),
2906 Location::Memory(args, (i * 16) as i32),
2907 )?;
2908 a.emit_str(
2909 sz,
2910 Location::GPR(GPR::X16),
2911 Location::Memory(GPR::XzrSp, caller_stack_offset),
2912 )?;
2913 match calling_convention {
2914 CallingConvention::AppleAarch64 => {
2915 caller_stack_offset += 1
2916 << match sz {
2917 Size::S8 => 0,
2918 Size::S16 => 1,
2919 Size::S32 => 2,
2920 Size::S64 => 3,
2921 };
2922 }
2923 _ => {
2924 caller_stack_offset += 8;
2925 }
2926 }
2927 }
2928 }
2929 }
2930
2931 dynasm!(a ; blr X(fptr));
2932
2933 if !sig.results().is_empty() {
2935 a.emit_str(Size::S64, Location::GPR(GPR::X0), Location::Memory(args, 0))?;
2936 }
2937
2938 dynasm!(a
2940 ; ldp X(fptr), X(args), [x29, 16]
2941 ; ldp x29, x30, [x29]
2942 ; add sp, sp, 32 + stack_offset as u32
2943 ; ret
2944 );
2945
2946 let mut body = a.finalize().unwrap();
2947 body.shrink_to_fit();
2948 Ok(FunctionBody {
2949 body,
2950 unwind_info: None,
2951 })
2952}
2953pub fn gen_std_dynamic_import_trampoline_arm64(
2955 vmoffsets: &VMOffsets,
2956 sig: &FunctionType,
2957 calling_convention: CallingConvention,
2958) -> Result<FunctionBody, CompileError> {
2959 let mut a = Assembler::new(0);
2960 let stack_offset: usize = 16 * std::cmp::max(sig.params().len(), sig.results().len());
2962 a.emit_stpdb(
2964 Size::S64,
2965 Location::GPR(GPR::X30),
2966 Location::GPR(GPR::X26),
2967 GPR::XzrSp,
2968 16,
2969 )?;
2970
2971 if stack_offset != 0 {
2972 if stack_offset < 0x1000 {
2973 a.emit_sub(
2974 Size::S64,
2975 Location::GPR(GPR::XzrSp),
2976 Location::Imm32(stack_offset as _),
2977 Location::GPR(GPR::XzrSp),
2978 )?;
2979 } else {
2980 a.emit_mov_imm(Location::GPR(GPR::X26), stack_offset as u64)?;
2981 a.emit_sub(
2982 Size::S64,
2983 Location::GPR(GPR::XzrSp),
2984 Location::GPR(GPR::X26),
2985 Location::GPR(GPR::XzrSp),
2986 )?;
2987 }
2988 }
2989
2990 if !sig.params().is_empty() {
2992 let mut argalloc = ArgumentRegisterAllocator::default();
2993 argalloc.next(Type::I64, calling_convention).unwrap(); let mut stack_param_count: usize = 0;
2996
2997 for (i, ty) in sig.params().iter().enumerate() {
2998 let source_loc = match argalloc.next(*ty, calling_convention)? {
2999 Some(ARM64Register::GPR(gpr)) => Location::GPR(gpr),
3000 Some(ARM64Register::NEON(neon)) => Location::SIMD(neon),
3001 None => {
3002 let sz = match calling_convention {
3003 CallingConvention::AppleAarch64 => match *ty {
3004 Type::I32 | Type::F32 => Size::S32,
3005 _ => {
3006 if stack_param_count & 7 != 0 {
3007 stack_param_count = (stack_param_count + 7) & !7;
3008 };
3009 Size::S64
3010 }
3011 },
3012 _ => Size::S64,
3013 };
3014 a.emit_ldr(
3015 sz,
3016 Location::GPR(GPR::X26),
3017 Location::Memory(GPR::XzrSp, (stack_offset + 16 + stack_param_count) as _),
3018 )?;
3019 stack_param_count += match sz {
3020 Size::S32 => 4,
3021 Size::S64 => 8,
3022 _ => codegen_error!(
3023 "singlepass unreachable in gen_std_dynamic_import_trampoline_arm64"
3024 ),
3025 };
3026 Location::GPR(GPR::X26)
3027 }
3028 };
3029 a.emit_str(
3030 Size::S64,
3031 source_loc,
3032 Location::Memory(GPR::XzrSp, (i * 16) as _),
3033 )?;
3034
3035 a.emit_str(
3037 Size::S64,
3038 Location::GPR(GPR::XzrSp), Location::Memory(GPR::XzrSp, (i * 16 + 8) as _), )?;
3041 }
3042 }
3043
3044 #[allow(clippy::match_single_binding)]
3045 match calling_convention {
3046 _ => {
3047 let offset = vmoffsets.vmdynamicfunction_import_context_address();
3049 a.emit_ldur(Size::S64, Location::GPR(GPR::X26), GPR::X0, offset as i32)?;
3050 a.emit_add(
3052 Size::S64,
3053 Location::GPR(GPR::XzrSp),
3054 Location::Imm8(0),
3055 Location::GPR(GPR::X1),
3056 )?;
3057 }
3058 };
3059
3060 a.emit_call_register(GPR::X26)?;
3062
3063 if !sig.results().is_empty() {
3065 assert_eq!(sig.results().len(), 1);
3066 a.emit_ldr(
3067 Size::S64,
3068 Location::GPR(GPR::X0),
3069 Location::Memory(GPR::XzrSp, 0),
3070 )?;
3071 }
3072
3073 if stack_offset != 0 {
3075 if stack_offset < 0x1000 {
3076 a.emit_add(
3077 Size::S64,
3078 Location::GPR(GPR::XzrSp),
3079 Location::Imm32(stack_offset as _),
3080 Location::GPR(GPR::XzrSp),
3081 )?;
3082 } else {
3083 a.emit_mov_imm(Location::GPR(GPR::X26), stack_offset as u64)?;
3084 a.emit_add(
3085 Size::S64,
3086 Location::GPR(GPR::XzrSp),
3087 Location::GPR(GPR::X26),
3088 Location::GPR(GPR::XzrSp),
3089 )?;
3090 }
3091 }
3092 a.emit_ldpia(
3093 Size::S64,
3094 Location::GPR(GPR::X30),
3095 Location::GPR(GPR::X26),
3096 GPR::XzrSp,
3097 16,
3098 )?;
3099
3100 a.emit_ret()?;
3102
3103 let mut body = a.finalize().unwrap();
3104 body.shrink_to_fit();
3105 Ok(FunctionBody {
3106 body,
3107 unwind_info: None,
3108 })
3109}
3110pub fn gen_import_call_trampoline_arm64(
3112 vmoffsets: &VMOffsets,
3113 index: FunctionIndex,
3114 sig: &FunctionType,
3115 calling_convention: CallingConvention,
3116) -> Result<CustomSection, CompileError> {
3117 let mut a = Assembler::new(0);
3118
3119 if sig
3124 .params()
3125 .iter()
3126 .any(|&x| x == Type::F32 || x == Type::F64)
3127 {
3128 #[allow(clippy::match_single_binding)]
3129 match calling_convention {
3130 _ => {
3131 let stack_offset: i32 = if sig.params().len() > 7 {
3133 7 * 8
3134 } else {
3135 (sig.params().len() as i32) * 8
3136 };
3137 let stack_offset = if stack_offset & 15 != 0 {
3138 stack_offset + 8
3139 } else {
3140 stack_offset
3141 };
3142 if stack_offset > 0 {
3143 if stack_offset < 0x1000 {
3144 a.emit_sub(
3145 Size::S64,
3146 Location::GPR(GPR::XzrSp),
3147 Location::Imm32(stack_offset as u32),
3148 Location::GPR(GPR::XzrSp),
3149 )?;
3150 } else {
3151 a.emit_mov_imm(Location::GPR(GPR::X16), stack_offset as u64)?;
3152 a.emit_sub(
3153 Size::S64,
3154 Location::GPR(GPR::XzrSp),
3155 Location::GPR(GPR::X16),
3156 Location::GPR(GPR::XzrSp),
3157 )?;
3158 }
3159 }
3160
3161 static PARAM_REGS: &[GPR] = &[
3163 GPR::X1,
3164 GPR::X2,
3165 GPR::X3,
3166 GPR::X4,
3167 GPR::X5,
3168 GPR::X6,
3169 GPR::X7,
3170 ];
3171 let mut param_locations = vec![];
3172 #[allow(clippy::needless_range_loop)]
3174 for i in 0..sig.params().len() {
3175 let loc = match i {
3176 0..=6 => {
3177 let loc = Location::Memory(GPR::XzrSp, (i * 8) as i32);
3178 a.emit_str(Size::S64, Location::GPR(PARAM_REGS[i]), loc)?;
3179 loc
3180 }
3181 _ => Location::Memory(GPR::XzrSp, stack_offset + ((i - 7) * 8) as i32),
3182 };
3183 param_locations.push(loc);
3184 }
3185
3186 let mut caller_stack_offset: i32 = 0;
3188 let mut argalloc = ArgumentRegisterAllocator::default();
3189 argalloc.next(Type::I64, calling_convention).unwrap(); for (i, ty) in sig.params().iter().enumerate() {
3191 let prev_loc = param_locations[i];
3192 let targ = match argalloc.next(*ty, calling_convention)? {
3193 Some(ARM64Register::GPR(gpr)) => Location::GPR(gpr),
3194 Some(ARM64Register::NEON(neon)) => Location::SIMD(neon),
3195 None => {
3196 a.emit_ldr(Size::S64, Location::GPR(GPR::X16), prev_loc)?;
3198 a.emit_str(
3199 Size::S64,
3200 Location::GPR(GPR::X16),
3201 Location::Memory(GPR::XzrSp, stack_offset + caller_stack_offset),
3202 )?;
3203 caller_stack_offset += 8;
3204 continue;
3205 }
3206 };
3207 a.emit_ldr(Size::S64, targ, prev_loc)?;
3208 }
3209
3210 if stack_offset > 0 {
3212 if stack_offset < 0x1000 {
3213 a.emit_add(
3214 Size::S64,
3215 Location::GPR(GPR::XzrSp),
3216 Location::Imm32(stack_offset as u32),
3217 Location::GPR(GPR::XzrSp),
3218 )?;
3219 } else {
3220 a.emit_mov_imm(Location::GPR(GPR::X16), stack_offset as u64)?;
3221 a.emit_add(
3222 Size::S64,
3223 Location::GPR(GPR::XzrSp),
3224 Location::GPR(GPR::X16),
3225 Location::GPR(GPR::XzrSp),
3226 )?;
3227 }
3228 }
3229 }
3230 }
3231 }
3232
3233 let offset = vmoffsets.vmctx_vmfunction_import(index);
3237 let offset =
3240 if (offset > 0 && offset < 0xF8) || (offset > 0 && offset < 0x7FF8 && (offset & 7) == 0) {
3241 offset
3242 } else {
3243 a.emit_mov_imm(Location::GPR(GPR::X16), (offset as i64) as u64)?;
3244 a.emit_add(
3245 Size::S64,
3246 Location::GPR(GPR::X0),
3247 Location::GPR(GPR::X16),
3248 Location::GPR(GPR::X0),
3249 )?;
3250 0
3251 };
3252 #[allow(clippy::match_single_binding)]
3253 match calling_convention {
3254 _ => {
3255 if (offset & 7) == 0 {
3256 a.emit_ldr(
3257 Size::S64,
3258 Location::GPR(GPR::X16),
3259 Location::Memory(GPR::X0, offset as i32), )?;
3261 a.emit_ldr(
3262 Size::S64,
3263 Location::GPR(GPR::X0),
3264 Location::Memory(GPR::X0, offset as i32 + 8), )?;
3266 } else {
3267 a.emit_ldur(
3268 Size::S64,
3269 Location::GPR(GPR::X16),
3270 GPR::X0,
3271 offset as i32, )?;
3273 a.emit_ldur(
3274 Size::S64,
3275 Location::GPR(GPR::X0),
3276 GPR::X0,
3277 offset as i32 + 8, )?;
3279 }
3280 }
3281 }
3282 a.emit_b_register(GPR::X16)?;
3283
3284 let mut contents = a.finalize().unwrap();
3285 contents.shrink_to_fit();
3286 let section_body = SectionBody::new_with_vec(contents);
3287
3288 Ok(CustomSection {
3289 protection: CustomSectionProtection::ReadExecute,
3290 alignment: None,
3291 bytes: section_body,
3292 relocations: vec![],
3293 })
3294}