1pub use crate::{
2 arm64_decl::{ARM64Register, ArgumentRegisterAllocator, GPR, NEON},
3 location::{Multiplier, Reg},
4 machine::{Label, Offset},
5};
6use crate::{
7 codegen_error, common_decl::Size, location::Location as AbstractLocation,
8 machine_arm64::ARM64_RETURN_VALUE_REGISTERS,
9};
10pub use dynasmrt::aarch64::{encode_logical_immediate_32bit, encode_logical_immediate_64bit};
11use dynasmrt::{
12 AssemblyOffset, DynamicLabel, DynasmApi, DynasmLabelApi, VecAssembler,
13 aarch64::Aarch64Relocation,
14};
15use wasmer_compiler::types::{
16 function::FunctionBody,
17 section::{CustomSection, CustomSectionProtection, SectionBody},
18};
19use wasmer_types::{
20 CompileError, FunctionIndex, FunctionType, Type, VMOffsets, target::CallingConvention,
21};
22
23type Assembler = VecAssembler<Aarch64Relocation>;
24
25macro_rules! dynasm {
30 ($a:expr ; $($tt:tt)*) => {
31 dynasm::dynasm!(
32 $a
33 ; .arch aarch64
34 ; $($tt)*
35 )
36 };
37}
38
39pub type Location = AbstractLocation<GPR, NEON>;
40
41#[derive(Copy, Clone, Debug, Eq, PartialEq)]
42#[allow(dead_code)]
43#[repr(u8)]
44pub enum Condition {
45 Eq = 0,
48 Ne = 1,
50 Cs = 2,
52 Cc = 3,
54 Mi = 4,
56 Pl = 5,
58 Vs = 6,
60 Vc = 7,
62 Hi = 8,
64 Ls = 9,
66 Ge = 10,
68 Lt = 11,
70 Gt = 12,
72 Le = 13,
74 Al = 14,
76}
77
78#[derive(Copy, Clone, Debug, Eq, PartialEq)]
79#[allow(dead_code, clippy::upper_case_acronyms)]
80pub enum NeonOrMemory {
81 NEON(NEON),
82 Memory(GPR, i32),
83}
84
85#[derive(Copy, Clone, Debug)]
86#[allow(dead_code, clippy::upper_case_acronyms)]
87pub enum GPROrMemory {
88 GPR(GPR),
89 Memory(GPR, i32),
90}
91
92#[allow(unused)]
93pub trait EmitterARM64 {
94 fn get_label(&mut self) -> Label;
95 fn get_offset(&self) -> Offset;
96 fn get_jmp_instr_size(&self) -> u8;
97
98 fn finalize_function(&mut self);
99
100 fn emit_str(&mut self, sz: Size, reg: Location, addr: Location) -> Result<(), CompileError>;
101 fn emit_ldr(&mut self, sz: Size, reg: Location, addr: Location) -> Result<(), CompileError>;
102 fn emit_stur(
103 &mut self,
104 sz: Size,
105 reg: Location,
106 addr: GPR,
107 offset: i32,
108 ) -> Result<(), CompileError>;
109 fn emit_ldur(
110 &mut self,
111 sz: Size,
112 reg: Location,
113 addr: GPR,
114 offset: i32,
115 ) -> Result<(), CompileError>;
116 fn emit_strdb(
117 &mut self,
118 sz: Size,
119 reg: Location,
120 addr: GPR,
121 offset: u32,
122 ) -> Result<(), CompileError>;
123 fn emit_stria(
124 &mut self,
125 sz: Size,
126 reg: Location,
127 addr: GPR,
128 offset: u32,
129 ) -> Result<(), CompileError>;
130 fn emit_ldria(
131 &mut self,
132 sz: Size,
133 reg: Location,
134 addr: GPR,
135 offset: u32,
136 ) -> Result<(), CompileError>;
137 fn emit_stpdb(
138 &mut self,
139 sz: Size,
140 reg1: Location,
141 reg2: Location,
142 addr: GPR,
143 offset: u32,
144 ) -> Result<(), CompileError>;
145 fn emit_ldpia(
146 &mut self,
147 sz: Size,
148 reg1: Location,
149 reg2: Location,
150 addr: GPR,
151 offset: u32,
152 ) -> Result<(), CompileError>;
153
154 fn emit_ldrb(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
155 fn emit_ldrh(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
156 fn emit_ldrsb(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
157 fn emit_ldrsh(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
158 fn emit_ldrsw(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
159 fn emit_strb(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
160 fn emit_strh(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
161
162 fn emit_ldaxr(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
163 fn emit_ldaxrb(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
164 fn emit_ldaxrh(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError>;
165 fn emit_stlxr(
166 &mut self,
167 sz: Size,
168 status: Location,
169 reg: Location,
170 dst: Location,
171 ) -> Result<(), CompileError>;
172 fn emit_stlxrb(
173 &mut self,
174 sz: Size,
175 status: Location,
176 reg: Location,
177 dst: Location,
178 ) -> Result<(), CompileError>;
179 fn emit_stlxrh(
180 &mut self,
181 sz: Size,
182 status: Location,
183 reg: Location,
184 dst: Location,
185 ) -> Result<(), CompileError>;
186
187 fn emit_mov(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
188
189 fn emit_movn(&mut self, sz: Size, reg: Location, val: u32) -> Result<(), CompileError>;
190 fn emit_movz(&mut self, reg: Location, val: u32) -> Result<(), CompileError>;
191 fn emit_movk(&mut self, reg: Location, val: u32, shift: u32) -> Result<(), CompileError>;
192
193 fn emit_mov_imm(&mut self, dst: Location, val: u64) -> Result<(), CompileError>;
194
195 fn emit_add(
196 &mut self,
197 sz: Size,
198 src1: Location,
199 src2: Location,
200 dst: Location,
201 ) -> Result<(), CompileError>;
202 fn emit_sub(
203 &mut self,
204 sz: Size,
205 src1: Location,
206 src2: Location,
207 dst: Location,
208 ) -> Result<(), CompileError>;
209 fn emit_mul(
210 &mut self,
211 sz: Size,
212 src1: Location,
213 src2: Location,
214 dst: Location,
215 ) -> Result<(), CompileError>;
216 fn emit_adds(
217 &mut self,
218 sz: Size,
219 src1: Location,
220 src2: Location,
221 dst: Location,
222 ) -> Result<(), CompileError>;
223 fn emit_subs(
224 &mut self,
225 sz: Size,
226 src1: Location,
227 src2: Location,
228 dst: Location,
229 ) -> Result<(), CompileError>;
230
231 fn emit_add_lsl(
232 &mut self,
233 sz: Size,
234 src1: Location,
235 src2: Location,
236 lsl: u32,
237 dst: Location,
238 ) -> Result<(), CompileError>;
239
240 fn emit_cmp(&mut self, sz: Size, left: Location, right: Location) -> Result<(), CompileError>;
241 fn emit_tst(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
242
243 fn emit_lsl(
244 &mut self,
245 sz: Size,
246 src1: Location,
247 src2: Location,
248 dst: Location,
249 ) -> Result<(), CompileError>;
250 fn emit_lsr(
251 &mut self,
252 sz: Size,
253 src1: Location,
254 src2: Location,
255 dst: Location,
256 ) -> Result<(), CompileError>;
257 fn emit_asr(
258 &mut self,
259 sz: Size,
260 src1: Location,
261 src2: Location,
262 dst: Location,
263 ) -> Result<(), CompileError>;
264 fn emit_ror(
265 &mut self,
266 sz: Size,
267 src1: Location,
268 src2: Location,
269 dst: Location,
270 ) -> Result<(), CompileError>;
271
272 fn emit_or(
273 &mut self,
274 sz: Size,
275 src1: Location,
276 src2: Location,
277 dst: Location,
278 ) -> Result<(), CompileError>;
279 fn emit_and(
280 &mut self,
281 sz: Size,
282 src1: Location,
283 src2: Location,
284 dst: Location,
285 ) -> Result<(), CompileError>;
286 fn emit_eor(
287 &mut self,
288 sz: Size,
289 src1: Location,
290 src2: Location,
291 dst: Location,
292 ) -> Result<(), CompileError>;
293
294 fn emit_bfc(
295 &mut self,
296 se: Size,
297 lsb: u32,
298 width: u32,
299 dst: Location,
300 ) -> Result<(), CompileError>;
301 fn emit_bfi(
302 &mut self,
303 se: Size,
304 src: Location,
305 lsb: u32,
306 width: u32,
307 dst: Location,
308 ) -> Result<(), CompileError>;
309
310 fn emit_udiv(
311 &mut self,
312 sz: Size,
313 src1: Location,
314 src2: Location,
315 dst: Location,
316 ) -> Result<(), CompileError>;
317 fn emit_sdiv(
318 &mut self,
319 sz: Size,
320 src1: Location,
321 src2: Location,
322 dst: Location,
323 ) -> Result<(), CompileError>;
324 fn emit_msub(
326 &mut self,
327 sz: Size,
328 a: Location,
329 b: Location,
330 c: Location,
331 dst: Location,
332 ) -> Result<(), CompileError>;
333
334 fn emit_sxtb(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
335 fn emit_sxth(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
336 fn emit_sxtw(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
337 fn emit_uxtb(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
338 fn emit_uxth(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
339
340 fn emit_cset(&mut self, sz: Size, dst: Location, cond: Condition) -> Result<(), CompileError>;
341 fn emit_csetm(&mut self, sz: Size, dst: Location, cond: Condition) -> Result<(), CompileError>;
342 fn emit_cinc(
343 &mut self,
344 sz: Size,
345 src: Location,
346 dst: Location,
347 cond: Condition,
348 ) -> Result<(), CompileError>;
349 fn emit_clz(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
350 fn emit_rbit(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
351
352 fn emit_label(&mut self, label: Label) -> Result<(), CompileError>;
353 fn emit_load_label(&mut self, reg: GPR, label: Label) -> Result<(), CompileError>;
354 fn emit_b_label(&mut self, label: Label) -> Result<(), CompileError>;
355 fn emit_cbz_label(&mut self, sz: Size, reg: Location, label: Label)
356 -> Result<(), CompileError>;
357 fn emit_cbnz_label(
358 &mut self,
359 sz: Size,
360 reg: Location,
361 label: Label,
362 ) -> Result<(), CompileError>;
363 fn emit_cbz_label_far(
364 &mut self,
365 sz: Size,
366 reg: Location,
367 label: Label,
368 ) -> Result<(), CompileError>;
369 fn emit_tbz_label(
370 &mut self,
371 sz: Size,
372 reg: Location,
373 n: u32,
374 label: Label,
375 ) -> Result<(), CompileError>;
376 fn emit_tbnz_label(
377 &mut self,
378 sz: Size,
379 reg: Location,
380 n: u32,
381 label: Label,
382 ) -> Result<(), CompileError>;
383 fn emit_bcond_label(&mut self, condition: Condition, label: Label) -> Result<(), CompileError>;
384 fn emit_bcond_label_far(
385 &mut self,
386 condition: Condition,
387 label: Label,
388 ) -> Result<(), CompileError>;
389 fn emit_b_register(&mut self, reg: GPR) -> Result<(), CompileError>;
390 fn emit_call_label(&mut self, label: Label) -> Result<(), CompileError>;
391 fn emit_call_register(&mut self, reg: GPR) -> Result<(), CompileError>;
392 fn emit_ret(&mut self) -> Result<(), CompileError>;
393
394 fn emit_udf(&mut self, payload: u16) -> Result<(), CompileError>;
395 fn emit_dmb(&mut self) -> Result<(), CompileError>;
396 fn emit_brk(&mut self) -> Result<(), CompileError>;
397
398 fn emit_fcmp(&mut self, sz: Size, src1: Location, src2: Location) -> Result<(), CompileError>;
399 fn emit_fneg(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
400 fn emit_fsqrt(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
401
402 fn emit_fadd(
403 &mut self,
404 sz: Size,
405 src1: Location,
406 src2: Location,
407 dst: Location,
408 ) -> Result<(), CompileError>;
409 fn emit_fsub(
410 &mut self,
411 sz: Size,
412 src1: Location,
413 src2: Location,
414 dst: Location,
415 ) -> Result<(), CompileError>;
416 fn emit_fmul(
417 &mut self,
418 sz: Size,
419 src1: Location,
420 src2: Location,
421 dst: Location,
422 ) -> Result<(), CompileError>;
423 fn emit_fdiv(
424 &mut self,
425 sz: Size,
426 src1: Location,
427 src2: Location,
428 dst: Location,
429 ) -> Result<(), CompileError>;
430
431 fn emit_fmin(
432 &mut self,
433 sz: Size,
434 src1: Location,
435 src2: Location,
436 dst: Location,
437 ) -> Result<(), CompileError>;
438 fn emit_fmax(
439 &mut self,
440 sz: Size,
441 src1: Location,
442 src2: Location,
443 dst: Location,
444 ) -> Result<(), CompileError>;
445
446 fn emit_frintz(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
447 fn emit_frintn(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
448 fn emit_frintm(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
449 fn emit_frintp(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError>;
450
451 fn emit_scvtf(
452 &mut self,
453 sz_in: Size,
454 src: Location,
455 sz_out: Size,
456 dst: Location,
457 ) -> Result<(), CompileError>;
458 fn emit_ucvtf(
459 &mut self,
460 sz_in: Size,
461 src: Location,
462 sz_out: Size,
463 dst: Location,
464 ) -> Result<(), CompileError>;
465 fn emit_fcvt(&mut self, sz_in: Size, src: Location, dst: Location) -> Result<(), CompileError>;
466 fn emit_fcvtzs(
467 &mut self,
468 sz_in: Size,
469 src: Location,
470 sz_out: Size,
471 dst: Location,
472 ) -> Result<(), CompileError>;
473 fn emit_fcvtzu(
474 &mut self,
475 sz_in: Size,
476 src: Location,
477 sz_out: Size,
478 dst: Location,
479 ) -> Result<(), CompileError>;
480
481 fn emit_fmov(
482 &mut self,
483 src_size: Size,
484 src: Location,
485 dst_size: Size,
486 dst: Location,
487 ) -> Result<(), CompileError>;
488 fn emit_cnt(&mut self, src: NEON, dst: NEON) -> Result<(), CompileError>;
489 fn emit_addv(&mut self, src: NEON, dst: NEON) -> Result<(), CompileError>;
490 fn emit_read_fpcr(&mut self, reg: GPR) -> Result<(), CompileError>;
491 fn emit_write_fpcr(&mut self, reg: GPR) -> Result<(), CompileError>;
492 fn emit_read_fpsr(&mut self, reg: GPR) -> Result<(), CompileError>;
493 fn emit_write_fpsr(&mut self, reg: GPR) -> Result<(), CompileError>;
494
495 fn arch_emit_indirect_call_with_trampoline(
496 &mut self,
497 _loc: Location,
498 ) -> Result<(), CompileError> {
499 codegen_error!("singlepass arch_emit_indirect_call_with_trampoline unimplemented")
500 }
501}
502
503impl EmitterARM64 for Assembler {
504 fn get_label(&mut self) -> DynamicLabel {
505 self.new_dynamic_label()
506 }
507
508 fn get_offset(&self) -> AssemblyOffset {
509 self.offset()
510 }
511
512 fn get_jmp_instr_size(&self) -> u8 {
513 4 }
515
516 fn finalize_function(&mut self) {
517 dynasm!(
518 self
519 ; const_neg_one_32:
520 ; .i32 -1
521 ; const_zero_32:
522 ; .i32 0
523 ; const_pos_one_32:
524 ; .i32 1
525 );
526 }
527
528 fn emit_str(&mut self, sz: Size, reg: Location, addr: Location) -> Result<(), CompileError> {
529 match (sz, reg, addr) {
530 (Size::S64, Location::GPR(reg), Location::Memory(addr, disp)) => {
531 let disp = disp as u32;
532 assert!(disp.is_multiple_of(8) && (disp < 0x8000));
533 dynasm!(self ; str X(reg), [X(addr), disp]);
534 }
535 (Size::S32, Location::GPR(reg), Location::Memory(addr, disp)) => {
536 let disp = disp as u32;
537 assert!(disp.is_multiple_of(4) && (disp < 0x4000));
538 dynasm!(self ; str W(reg), [X(addr), disp]);
539 }
540 (Size::S16, Location::GPR(reg), Location::Memory(addr, disp)) => {
541 let disp = disp as u32;
542 assert!(disp.is_multiple_of(2) && (disp < 0x2000));
543 dynasm!(self ; strh W(reg), [X(addr), disp]);
544 }
545 (Size::S8, Location::GPR(reg), Location::Memory(addr, disp)) => {
546 let disp = disp as u32;
547 assert!(disp < 0x1000);
548 dynasm!(self ; strb W(reg), [X(addr), disp]);
549 }
550 (Size::S64, Location::SIMD(reg), Location::Memory(addr, disp)) => {
551 let disp = disp as u32;
552 assert!(disp.is_multiple_of(8) && (disp < 0x8000));
553 dynasm!(self ; str D(reg), [X(addr), disp]);
554 }
555 (Size::S32, Location::SIMD(reg), Location::Memory(addr, disp)) => {
556 let disp = disp as u32;
557 assert!(disp.is_multiple_of(4) && (disp < 0x4000));
558 dynasm!(self ; str S(reg), [X(addr), disp]);
559 }
560 (Size::S64, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
561 assert!(offs == 0);
562 let mult = mult as u32;
563 match mult {
564 0 => dynasm!(self ; str X(reg), [X(addr)]),
565 1 => dynasm!(self ; str X(reg), [X(addr), X(r2)]),
566 _ => dynasm!(self ; str X(reg), [X(addr), X(r2), LSL mult]),
567 };
568 }
569 (Size::S32, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
570 assert!(offs == 0);
571 let mult = mult as u32;
572 match mult {
573 0 => dynasm!(self ; str W(reg), [X(addr)]),
574 1 => dynasm!(self ; str W(reg), [X(addr), X(r2)]),
575 _ => dynasm!(self ; str W(reg), [X(addr), X(r2), LSL mult]),
576 };
577 }
578 _ => codegen_error!("singlepass can't emit STR {:?}, {:?}, {:?}", sz, reg, addr),
579 }
580 Ok(())
581 }
582 fn emit_ldr(&mut self, sz: Size, reg: Location, addr: Location) -> Result<(), CompileError> {
583 match (sz, reg, addr) {
584 (Size::S64, Location::GPR(reg), Location::Memory(addr, disp)) => {
585 assert!((disp & 0x7) == 0 && (disp < 0x8000));
586 let disp = disp as u32;
587 dynasm!(self ; ldr X(reg), [X(addr), disp]);
588 }
589 (Size::S32, Location::GPR(reg), Location::Memory(addr, disp)) => {
590 assert!((disp & 0x3) == 0 && (disp < 0x4000));
591 let disp = disp as u32;
592 dynasm!(self ; ldr W(reg), [X(addr), disp]);
593 }
594 (Size::S16, Location::GPR(reg), Location::Memory(addr, disp)) => {
595 assert!((disp & 0x1 == 0) && (disp < 0x2000));
596 let disp = disp as u32;
597 dynasm!(self ; ldrh W(reg), [X(addr), disp]);
598 }
599 (Size::S8, Location::GPR(reg), Location::Memory(addr, disp)) => {
600 assert!(disp < 0x1000);
601 let disp = disp as u32;
602 dynasm!(self ; ldrb W(reg), [X(addr), disp]);
603 }
604 (Size::S64, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
605 assert!(offs == 0);
606 let mult = mult as u32;
607 match mult {
608 0 => dynasm!(self ; ldr X(reg), [X(addr)]),
609 1 => dynasm!(self ; ldr X(reg), [X(addr), X(r2)]),
610 _ => dynasm!(self ; ldr X(reg), [X(addr), X(r2), LSL mult]),
611 };
612 }
613 (Size::S32, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
614 assert!(offs == 0);
615 let mult = mult as u32;
616 match mult {
617 0 => dynasm!(self ; ldr W(reg), [X(addr)]),
618 1 => dynasm!(self ; ldr W(reg), [X(addr), X(r2)]),
619 _ => dynasm!(self ; ldr W(reg), [X(addr), X(r2), LSL mult]),
620 };
621 }
622 (Size::S64, Location::SIMD(reg), Location::Memory(addr, disp)) => {
623 let disp = disp as u32;
624 assert!((disp & 0x7) == 0 && (disp < 0x8000));
625 dynasm!(self ; ldr D(reg), [X(addr), disp]);
626 }
627 (Size::S32, Location::SIMD(reg), Location::Memory(addr, disp)) => {
628 let disp = disp as u32;
629 assert!((disp & 0x3) == 0 && (disp < 0x4000));
630 dynasm!(self ; ldr S(reg), [X(addr), disp]);
631 }
632 (Size::S64, Location::SIMD(reg), Location::Memory2(addr, r2, mult, offs)) => {
633 assert!(offs == 0);
634 let mult = mult as u32;
635 match mult {
636 0 => dynasm!(self ; ldr D(reg), [X(addr)]),
637 1 => dynasm!(self ; ldr D(reg), [X(addr), X(r2)]),
638 _ => dynasm!(self ; ldr D(reg), [X(addr), X(r2), LSL mult]),
639 };
640 }
641 (Size::S32, Location::SIMD(reg), Location::Memory2(addr, r2, mult, offs)) => {
642 assert!(offs == 0);
643 let mult = mult as u32;
644 match mult {
645 0 => dynasm!(self ; ldr S(reg), [X(addr)]),
646 1 => dynasm!(self ; ldr S(reg), [X(addr), X(r2)]),
647 _ => dynasm!(self ; ldr S(reg), [X(addr), X(r2), LSL mult]),
648 };
649 }
650 _ => codegen_error!("singlepass can't emit LDR {:?}, {:?}, {:?}", sz, reg, addr),
651 }
652 Ok(())
653 }
654 fn emit_stur(
655 &mut self,
656 sz: Size,
657 reg: Location,
658 addr: GPR,
659 offset: i32,
660 ) -> Result<(), CompileError> {
661 assert!((-255..=255).contains(&offset));
662 match (sz, reg) {
663 (Size::S64, Location::GPR(reg)) => {
664 dynasm!(self ; stur X(reg), [X(addr), offset]);
665 }
666 (Size::S32, Location::GPR(reg)) => {
667 dynasm!(self ; stur W(reg), [X(addr), offset]);
668 }
669 (Size::S64, Location::SIMD(reg)) => {
670 dynasm!(self ; stur D(reg), [X(addr), offset]);
671 }
672 (Size::S32, Location::SIMD(reg)) => {
673 dynasm!(self ; stur S(reg), [X(addr), offset]);
674 }
675 _ => codegen_error!(
676 "singlepass can't emit STUR {:?}, {:?}, {:?}, {:?}",
677 sz,
678 reg,
679 addr,
680 offset
681 ),
682 }
683 Ok(())
684 }
685 fn emit_ldur(
686 &mut self,
687 sz: Size,
688 reg: Location,
689 addr: GPR,
690 offset: i32,
691 ) -> Result<(), CompileError> {
692 assert!((-255..=255).contains(&offset));
693 match (sz, reg) {
694 (Size::S64, Location::GPR(reg)) => {
695 dynasm!(self ; ldur X(reg), [X(addr), offset]);
696 }
697 (Size::S32, Location::GPR(reg)) => {
698 dynasm!(self ; ldur W(reg), [X(addr), offset]);
699 }
700 (Size::S64, Location::SIMD(reg)) => {
701 dynasm!(self ; ldur D(reg), [X(addr), offset]);
702 }
703 (Size::S32, Location::SIMD(reg)) => {
704 dynasm!(self ; ldur S(reg), [X(addr), offset]);
705 }
706 _ => codegen_error!(
707 "singlepass can't emit LDUR {:?}, {:?}, {:?}, {:?}",
708 sz,
709 reg,
710 addr,
711 offset
712 ),
713 }
714 Ok(())
715 }
716
717 fn emit_strdb(
718 &mut self,
719 sz: Size,
720 reg: Location,
721 addr: GPR,
722 offset: u32,
723 ) -> Result<(), CompileError> {
724 assert!(offset <= 255);
725 match (sz, reg) {
726 (Size::S64, Location::GPR(reg)) => {
727 dynasm!(self ; str X(reg), [X(addr), -(offset as i32)]!);
728 }
729 (Size::S64, Location::SIMD(reg)) => {
730 dynasm!(self ; str D(reg), [X(addr), -(offset as i32)]!);
731 }
732 _ => codegen_error!("singlepass can't emit STRDB"),
733 }
734 Ok(())
735 }
736 fn emit_stria(
737 &mut self,
738 sz: Size,
739 reg: Location,
740 addr: GPR,
741 offset: u32,
742 ) -> Result<(), CompileError> {
743 assert!(offset <= 255);
744 match (sz, reg) {
745 (Size::S64, Location::GPR(reg)) => {
746 dynasm!(self ; str X(reg), [X(addr)], offset as i32);
747 }
748 (Size::S64, Location::SIMD(reg)) => {
749 dynasm!(self ; str D(reg), [X(addr)], offset as i32);
750 }
751 _ => codegen_error!("singlepass can't emit STRIA"),
752 }
753 Ok(())
754 }
755 fn emit_ldria(
756 &mut self,
757 sz: Size,
758 reg: Location,
759 addr: GPR,
760 offset: u32,
761 ) -> Result<(), CompileError> {
762 assert!(offset <= 255);
763 match (sz, reg) {
764 (Size::S64, Location::GPR(reg)) => {
765 dynasm!(self ; ldr X(reg), [X(addr)], offset as _);
766 }
767 (Size::S64, Location::SIMD(reg)) => {
768 dynasm!(self ; ldr D(reg), [X(addr)], offset as _);
769 }
770 _ => codegen_error!("singlepass can't emit LDRIA"),
771 }
772 Ok(())
773 }
774
775 fn emit_stpdb(
776 &mut self,
777 sz: Size,
778 reg1: Location,
779 reg2: Location,
780 addr: GPR,
781 offset: u32,
782 ) -> Result<(), CompileError> {
783 assert!(offset <= 255);
784 match (sz, reg1, reg2) {
785 (Size::S64, Location::GPR(reg1), Location::GPR(reg2)) => {
786 dynasm!(self ; stp X(reg1), X(reg2), [X(addr), -(offset as i32)]!);
787 }
788 _ => codegen_error!("singlepass can't emit STPDB"),
789 }
790 Ok(())
791 }
792 fn emit_ldpia(
793 &mut self,
794 sz: Size,
795 reg1: Location,
796 reg2: Location,
797 addr: GPR,
798 offset: u32,
799 ) -> Result<(), CompileError> {
800 assert!(offset <= 255);
801 match (sz, reg1, reg2) {
802 (Size::S64, Location::GPR(reg1), Location::GPR(reg2)) => {
803 dynasm!(self ; ldp X(reg1), X(reg2), [X(addr)], offset as _);
804 }
805 _ => codegen_error!("singlepass can't emit LDPIA"),
806 }
807 Ok(())
808 }
809
810 fn emit_ldrb(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
811 match (reg, dst) {
812 (Location::GPR(reg), Location::Memory(addr, offset)) => {
813 let offset = offset as u32;
814 assert!(offset < 0x1000);
815 dynasm!(self ; ldrb W(reg), [X(addr), offset]);
816 }
817 (Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
818 assert!(offs == 0);
819 let mult = mult as u32;
820 match mult {
821 0 => dynasm!(self ; ldrb W(reg), [X(addr)]),
822 1 => dynasm!(self ; ldrb W(reg), [X(addr), X(r2)]),
823 _ => dynasm!(self ; ldrb W(reg), [X(addr), X(r2), LSL mult]),
824 };
825 }
826 _ => codegen_error!("singlepass can't emit LDRB {:?}, {:?}", reg, dst),
827 }
828 Ok(())
829 }
830 fn emit_ldrh(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
831 match (reg, dst) {
832 (Location::GPR(reg), Location::Memory(addr, offset)) => {
833 let offset = offset as u32;
834 assert!((offset & 1 == 0) && (offset < 0x2000));
835 dynasm!(self ; ldrh W(reg), [X(addr), offset]);
836 }
837 (Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
838 assert!(offs == 0);
839 let mult = mult as u32;
840 match mult {
841 0 => dynasm!(self ; ldrh W(reg), [X(addr)]),
842 1 => dynasm!(self ; ldrh W(reg), [X(addr), X(r2)]),
843 _ => dynasm!(self ; ldrh W(reg), [X(addr), X(r2), LSL mult]),
844 };
845 }
846 _ => codegen_error!("singlepass can't emit LDRH {:?}, {:?}", reg, dst),
847 }
848 Ok(())
849 }
850 fn emit_ldrsb(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
851 match (sz, reg, dst) {
852 (Size::S64, Location::GPR(reg), Location::Memory(addr, offset)) => {
853 let offset = offset as u32;
854 assert!(offset < 0x1000);
855 dynasm!(self ; ldrsb X(reg), [X(addr), offset]);
856 }
857 (Size::S32, Location::GPR(reg), Location::Memory(addr, offset)) => {
858 let offset = offset as u32;
859 assert!(offset < 0x1000);
860 dynasm!(self ; ldrsb W(reg), [X(addr), offset]);
861 }
862 (Size::S64, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
863 assert!(offs == 0);
864 let mult = mult as u32;
865 match mult {
866 0 => dynasm!(self ; ldrsb X(reg), [X(addr)]),
867 1 => dynasm!(self ; ldrsb X(reg), [X(addr), X(r2)]),
868 _ => dynasm!(self ; ldrsb X(reg), [X(addr), X(r2), LSL mult]),
869 };
870 }
871 (Size::S32, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
872 assert!(offs == 0);
873 let mult = mult as u32;
874 match mult {
875 0 => dynasm!(self ; ldrsb W(reg), [X(addr)]),
876 1 => dynasm!(self ; ldrsb W(reg), [X(addr), X(r2)]),
877 _ => dynasm!(self ; ldrsb W(reg), [X(addr), X(r2), LSL mult]),
878 };
879 }
880 _ => codegen_error!("singlepass can't emit LDRSB {:?}, {:?}, {:?}", sz, reg, dst),
881 }
882 Ok(())
883 }
884 fn emit_ldrsh(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
885 match (sz, reg, dst) {
886 (Size::S64, Location::GPR(reg), Location::Memory(addr, offset)) => {
887 let offset = offset as u32;
888 assert!((offset & 1 == 0) && (offset < 0x2000));
889 dynasm!(self ; ldrsh X(reg), [X(addr), offset]);
890 }
891 (Size::S32, Location::GPR(reg), Location::Memory(addr, offset)) => {
892 let offset = offset as u32;
893 assert!((offset & 1 == 0) && (offset < 0x2000));
894 dynasm!(self ; ldrsh W(reg), [X(addr), offset]);
895 }
896 (Size::S64, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
897 assert!(offs == 0);
898 let mult = mult as u32;
899 match mult {
900 0 => dynasm!(self ; ldrsh X(reg), [X(addr)]),
901 1 => dynasm!(self ; ldrsh X(reg), [X(addr), X(r2)]),
902 _ => dynasm!(self ; ldrsh X(reg), [X(addr), X(r2), LSL mult]),
903 };
904 }
905 (Size::S32, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
906 assert!(offs == 0);
907 let mult = mult as u32;
908 match mult {
909 0 => dynasm!(self ; ldrsh W(reg), [X(addr)]),
910 1 => dynasm!(self ; ldrsh W(reg), [X(addr), X(r2)]),
911 _ => dynasm!(self ; ldrsh W(reg), [X(addr), X(r2), LSL mult]),
912 };
913 }
914 _ => codegen_error!("singlepass can't emit LDRSH {:?}, {:?}, {:?}", sz, reg, dst),
915 }
916 Ok(())
917 }
918 fn emit_ldrsw(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
919 match (sz, reg, dst) {
920 (Size::S64, Location::GPR(reg), Location::Memory(addr, offset)) => {
921 let offset = offset as u32;
922 assert!((offset & 3 == 0) && (offset < 0x4000));
923 dynasm!(self ; ldrsw X(reg), [X(addr), offset]);
924 }
925 (Size::S64, Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
926 assert!(offs == 0);
927 let mult = mult as u32;
928 match mult {
929 0 => dynasm!(self ; ldrsw X(reg), [X(addr)]),
930 1 => dynasm!(self ; ldrsw X(reg), [X(addr), X(r2)]),
931 _ => dynasm!(self ; ldrsw X(reg), [X(addr), X(r2), LSL mult]),
932 };
933 }
934 _ => codegen_error!("singlepass can't emit LDRSW {:?}, {:?}, {:?}", sz, reg, dst),
935 }
936 Ok(())
937 }
938 fn emit_strb(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
939 match (reg, dst) {
940 (Location::GPR(reg), Location::Memory(addr, offset)) => {
941 let offset = offset as u32;
942 assert!(offset < 0x1000);
943 dynasm!(self ; strb W(reg), [X(addr), offset]);
944 }
945 (Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
946 assert!(offs == 0);
947 let mult = mult as u32;
948 match mult {
949 0 => dynasm!(self ; strb W(reg), [X(addr)]),
950 1 => dynasm!(self ; strb W(reg), [X(addr), X(r2)]),
951 _ => dynasm!(self ; strb W(reg), [X(addr), X(r2), LSL mult]),
952 };
953 }
954 _ => codegen_error!("singlepass can't emit STRB {:?}, {:?}", reg, dst),
955 }
956 Ok(())
957 }
958 fn emit_strh(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
959 match (reg, dst) {
960 (Location::GPR(reg), Location::Memory(addr, offset)) => {
961 let offset = offset as u32;
962 assert!((offset & 1 == 0) && (offset < 0x2000));
963 dynasm!(self ; strh W(reg), [X(addr), offset]);
964 }
965 (Location::GPR(reg), Location::Memory2(addr, r2, mult, offs)) => {
966 assert!(offs == 0);
967 let mult = mult as u32;
968 match mult {
969 0 => dynasm!(self ; strh W(reg), [X(addr)]),
970 1 => dynasm!(self ; strh W(reg), [X(addr), X(r2)]),
971 _ => dynasm!(self ; strh W(reg), [X(addr), X(r2), LSL mult]),
972 };
973 }
974 _ => codegen_error!("singlepass can't emit STRH {:?}, {:?}", reg, dst),
975 }
976 Ok(())
977 }
978
979 fn emit_ldaxr(&mut self, sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
980 match (sz, reg, dst) {
981 (Size::S32, Location::GPR(reg), Location::GPR(dst)) => {
982 dynasm!(self ; ldaxr W(reg), [X(dst)]);
983 }
984 (Size::S64, Location::GPR(reg), Location::GPR(dst)) => {
985 dynasm!(self ; ldaxr X(reg), [X(dst)]);
986 }
987 _ => codegen_error!("singlepass can't emit LDAXR {:?}, {:?}", reg, dst),
988 }
989 Ok(())
990 }
991 fn emit_ldaxrb(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
992 match (reg, dst) {
993 (Location::GPR(reg), Location::GPR(dst)) => {
994 dynasm!(self ; ldaxrb W(reg), [X(dst)]);
995 }
996 _ => codegen_error!("singlepass can't emit LDAXRB {:?}, {:?}", reg, dst),
997 }
998 Ok(())
999 }
1000 fn emit_ldaxrh(&mut self, _sz: Size, reg: Location, dst: Location) -> Result<(), CompileError> {
1001 match (reg, dst) {
1002 (Location::GPR(reg), Location::GPR(dst)) => {
1003 dynasm!(self ; ldaxrh W(reg), [X(dst)]);
1004 }
1005 _ => codegen_error!("singlepass can't emit LDAXRH {:?}, {:?}", reg, dst),
1006 }
1007 Ok(())
1008 }
1009 fn emit_stlxr(
1010 &mut self,
1011 sz: Size,
1012 status: Location,
1013 reg: Location,
1014 dst: Location,
1015 ) -> Result<(), CompileError> {
1016 match (sz, status, reg, dst) {
1017 (Size::S32, Location::GPR(status), Location::GPR(reg), Location::GPR(dst)) => {
1018 dynasm!(self ; stlxr W(status), W(reg), [X(dst)]);
1019 }
1020 (Size::S64, Location::GPR(status), Location::GPR(reg), Location::GPR(dst)) => {
1021 dynasm!(self ; stlxr W(status), X(reg), [X(dst)]);
1022 }
1023 _ => codegen_error!("singlepass can't emit STLXR {:?}, {:?}", reg, dst),
1024 }
1025 Ok(())
1026 }
1027 fn emit_stlxrb(
1028 &mut self,
1029 _sz: Size,
1030 status: Location,
1031 reg: Location,
1032 dst: Location,
1033 ) -> Result<(), CompileError> {
1034 match (status, reg, dst) {
1035 (Location::GPR(status), Location::GPR(reg), Location::GPR(dst)) => {
1036 dynasm!(self ; stlxrb W(status), W(reg), [X(dst)]);
1037 }
1038 _ => codegen_error!("singlepass can't emit STLXRB {:?}, {:?}", reg, dst),
1039 }
1040 Ok(())
1041 }
1042 fn emit_stlxrh(
1043 &mut self,
1044 _sz: Size,
1045 status: Location,
1046 reg: Location,
1047 dst: Location,
1048 ) -> Result<(), CompileError> {
1049 match (status, reg, dst) {
1050 (Location::GPR(status), Location::GPR(reg), Location::GPR(dst)) => {
1051 dynasm!(self ; stlxrh W(status), W(reg), [X(dst)]);
1052 }
1053 _ => codegen_error!("singlepass can't emit STLXRH {:?}, {:?}", reg, dst),
1054 }
1055 Ok(())
1056 }
1057
1058 fn emit_mov(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1059 match (sz, src, dst) {
1060 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1061 dynasm!(self ; mov X(dst), X(src));
1062 }
1063 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1064 dynasm!(self ; mov W(dst), W(src));
1065 }
1066 (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
1067 dynasm!(self ; mov V(dst).D[0], V(src).D[0]);
1068 }
1069 (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
1070 dynasm!(self ; mov V(dst).S[0], V(src).S[0]);
1071 }
1072 (Size::S64, Location::GPR(src), Location::SIMD(dst)) => {
1073 dynasm!(self ; mov V(dst).D[0], X(src));
1074 }
1075 (Size::S32, Location::GPR(src), Location::SIMD(dst)) => {
1076 dynasm!(self ; mov V(dst).S[0], W(src));
1077 }
1078 (Size::S64, Location::SIMD(src), Location::GPR(dst)) => {
1079 dynasm!(self ; mov X(dst), V(src).D[0]);
1080 }
1081 (Size::S32, Location::SIMD(src), Location::GPR(dst)) => {
1082 dynasm!(self ; mov W(dst), V(src).S[0]);
1083 }
1084 (Size::S32, Location::Imm32(val), Location::GPR(dst)) => {
1085 if val < 0x1000 || encode_logical_immediate_32bit(val as _).is_some() {
1086 dynasm!(self ; mov W(dst), val);
1087 } else {
1088 dynasm!(self
1089 ; movz W(dst), val
1090 ; movk W(dst), val, lsl #16);
1091 }
1092 }
1093 (Size::S64, Location::Imm32(val), Location::GPR(dst)) => {
1094 if val < 0x1000 {
1095 dynasm!(self ; mov W(dst), val);
1096 } else if encode_logical_immediate_64bit(val as _).is_some() {
1097 dynasm!(self ; mov X(dst), val as _);
1098 } else {
1099 dynasm!(self
1100 ; movz X(dst), val
1101 ; movk X(dst), val, lsl #16);
1102 }
1103 }
1104 (Size::S64, Location::Imm64(val), Location::GPR(dst)) => {
1105 if val < 0x1000 || encode_logical_immediate_64bit(val as _).is_some() {
1106 dynasm!(self ; mov X(dst), val as _);
1107 } else {
1108 dynasm!(self
1109 ; movz X(dst), val as _
1110 ; movk X(dst), val as _, lsl #16
1111 ; movk X(dst), val as _, lsl #32
1112 ; movk X(dst), val as _, lsl #48);
1113 }
1114 }
1115 _ => codegen_error!("singlepass can't emit MOV {:?}, {:?}, {:?}", sz, src, dst),
1116 }
1117 Ok(())
1118 }
1119
1120 fn emit_movn(&mut self, sz: Size, reg: Location, val: u32) -> Result<(), CompileError> {
1121 match (sz, reg) {
1122 (Size::S32, Location::GPR(reg)) => {
1123 dynasm!(self ; movn W(reg), val);
1124 }
1125 (Size::S64, Location::GPR(reg)) => {
1126 dynasm!(self ; movn X(reg), val);
1127 }
1128 _ => codegen_error!("singlepass can't emit MOVN"),
1129 }
1130 Ok(())
1131 }
1132 fn emit_movz(&mut self, reg: Location, val: u32) -> Result<(), CompileError> {
1133 match reg {
1134 Location::GPR(reg) => {
1135 dynasm!(self ; movz W(reg), val);
1136 }
1137 _ => codegen_error!("singlepass can't emit MOVZ"),
1138 }
1139 Ok(())
1140 }
1141 fn emit_movk(&mut self, reg: Location, val: u32, shift: u32) -> Result<(), CompileError> {
1142 match reg {
1143 Location::GPR(reg) => {
1144 dynasm!(self ; movk X(reg), val, LSL shift);
1145 }
1146 _ => codegen_error!("singlepass can't emit MOVK"),
1147 }
1148 Ok(())
1149 }
1150
1151 fn emit_mov_imm(&mut self, dst: Location, val: u64) -> Result<(), CompileError> {
1152 match dst {
1153 Location::GPR(dst) => {
1154 let offset = val.trailing_zeros() & 48;
1155 let masked = 0xffff & (val >> offset);
1156 if (masked << offset) == val {
1157 dynasm!(self ; movz X(dst), masked as u32, LSL offset);
1158 } else if val >> 16 == 0xffff_ffff_ffff {
1159 let val: u16 = !((val & 0xffff) as u16);
1160 dynasm!(self ; movn X(dst), val as u32);
1161 } else if val >> 16 == 0xffff {
1162 let val: u16 = !((val & 0xffff) as u16);
1163 dynasm!(self ; movn W(dst), val as u32);
1164 } else {
1165 dynasm!(self ; movz W(dst), (val&0xffff) as u32);
1166 let val = val >> 16;
1167 if val != 0 {
1168 dynasm!(self ; movk X(dst), (val&0xffff) as u32, LSL 16);
1169 let val = val >> 16;
1170 if val != 0 {
1171 dynasm!(self ; movk X(dst), (val&0xffff) as u32, LSL 32);
1172 let val = val >> 16;
1173 if val != 0 {
1174 dynasm!(self ; movk X(dst), (val&0xffff) as u32, LSL 48);
1175 }
1176 }
1177 }
1178 }
1179 }
1180 _ => codegen_error!("singlepass can't emit MOVW {:?}", dst),
1181 }
1182 Ok(())
1183 }
1184
1185 fn emit_add(
1186 &mut self,
1187 sz: Size,
1188 src1: Location,
1189 src2: Location,
1190 dst: Location,
1191 ) -> Result<(), CompileError> {
1192 match (sz, src1, src2, dst) {
1193 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1194 dynasm!(self ; add XSP(dst), XSP(src1), X(src2), UXTX);
1195 }
1196 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1197 dynasm!(self ; add WSP(dst), WSP(src1), W(src2), UXTX);
1198 }
1199 (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1200 | (Size::S64, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1201 dynasm!(self ; add XSP(dst), XSP(src1), imm as _);
1202 }
1203 (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
1204 | (Size::S64, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
1205 if imm >= 0x1000 {
1206 unreachable!();
1207 }
1208 dynasm!(self ; add XSP(dst), XSP(src1), imm);
1209 }
1210 (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst))
1211 | (Size::S64, Location::Imm64(imm), Location::GPR(src1), Location::GPR(dst)) => {
1212 if imm >= 0x1000 {
1213 unreachable!();
1214 }
1215 let imm = imm as u32;
1216 dynasm!(self ; add XSP(dst), XSP(src1), imm);
1217 }
1218 (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1219 | (Size::S32, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1220 dynasm!(self ; add WSP(dst), WSP(src1), imm as u32);
1221 }
1222 (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
1223 | (Size::S32, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
1224 if imm >= 0x1000 {
1225 unreachable!();
1226 }
1227 dynasm!(self ; add WSP(dst), WSP(src1), imm);
1228 }
1229 _ => codegen_error!(
1230 "singlepass can't emit ADD {:?} {:?} {:?} {:?}",
1231 sz,
1232 src1,
1233 src2,
1234 dst
1235 ),
1236 }
1237 Ok(())
1238 }
1239 fn emit_sub(
1240 &mut self,
1241 sz: Size,
1242 src1: Location,
1243 src2: Location,
1244 dst: Location,
1245 ) -> Result<(), CompileError> {
1246 match (sz, src1, src2, dst) {
1247 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1248 dynasm!(self; sub XSP(dst), XSP(src1), X(src2), UXTX 0);
1249 }
1250 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1251 dynasm!(self ; sub WSP(dst), WSP(src1), W(src2), UXTX 0);
1252 }
1253 (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1254 dynasm!(self ; sub XSP(dst), XSP(src1), imm as u32);
1255 }
1256 (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1257 dynasm!(self ; sub WSP(dst), WSP(src1), imm as u32);
1258 }
1259 (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1260 if imm >= 0x1000 {
1261 unreachable!();
1262 }
1263 dynasm!(self ; sub WSP(dst), WSP(src1), imm);
1264 }
1265 (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1266 if imm >= 0x1000 {
1267 unreachable!();
1268 }
1269 dynasm!(self ; sub XSP(dst), XSP(src1), imm);
1270 }
1271 (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
1272 if imm >= 0x1000 {
1273 unreachable!();
1274 }
1275 dynasm!(self ; sub XSP(dst), XSP(src1), imm as u32);
1276 }
1277 _ => codegen_error!(
1278 "singlepass can't emit SUB {:?} {:?} {:?} {:?}",
1279 sz,
1280 src1,
1281 src2,
1282 dst
1283 ),
1284 }
1285 Ok(())
1286 }
1287 fn emit_mul(
1288 &mut self,
1289 sz: Size,
1290 src1: Location,
1291 src2: Location,
1292 dst: Location,
1293 ) -> Result<(), CompileError> {
1294 match (sz, src1, src2, dst) {
1295 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1296 dynasm!(self ; mul X(dst), X(src1), X(src2));
1297 }
1298 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1299 dynasm!(self ; mul W(dst), W(src1), W(src2));
1300 }
1301 _ => codegen_error!(
1302 "singlepass can't emit MUL {:?} {:?} {:?} {:?}",
1303 sz,
1304 src1,
1305 src2,
1306 dst
1307 ),
1308 }
1309 Ok(())
1310 }
1311 fn emit_adds(
1312 &mut self,
1313 sz: Size,
1314 src1: Location,
1315 src2: Location,
1316 dst: Location,
1317 ) -> Result<(), CompileError> {
1318 match (sz, src1, src2, dst) {
1319 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1320 dynasm!(self ; adds X(dst), X(src1), X(src2));
1321 }
1322 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1323 dynasm!(self ; adds W(dst), W(src1), W(src2));
1324 }
1325 (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1326 | (Size::S64, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1327 dynasm!(self ; adds X(dst), XSP(src1), imm as u32);
1328 }
1329 (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
1330 | (Size::S64, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
1331 if imm >= 0x1000 {
1332 codegen_error!("singlepass ADD.S with imm too large {}", imm);
1333 }
1334 dynasm!(self ; adds X(dst), XSP(src1), imm);
1335 }
1336 (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst))
1337 | (Size::S32, Location::Imm8(imm), Location::GPR(src1), Location::GPR(dst)) => {
1338 dynasm!(self ; adds W(dst), WSP(src1), imm as u32);
1339 }
1340 (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst))
1341 | (Size::S32, Location::Imm32(imm), Location::GPR(src1), Location::GPR(dst)) => {
1342 if imm >= 0x1000 {
1343 codegen_error!("singlepass ADD.S with imm too large {}", imm);
1344 }
1345 dynasm!(self ; adds W(dst), WSP(src1), imm);
1346 }
1347 _ => codegen_error!(
1348 "singlepass can't emit ADD.S {:?} {:?} {:?} {:?}",
1349 sz,
1350 src1,
1351 src2,
1352 dst
1353 ),
1354 }
1355 Ok(())
1356 }
1357 fn emit_subs(
1358 &mut self,
1359 sz: Size,
1360 src1: Location,
1361 src2: Location,
1362 dst: Location,
1363 ) -> Result<(), CompileError> {
1364 match (sz, src1, src2, dst) {
1365 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1366 dynasm!(self ; subs X(dst), X(src1), X(src2));
1367 }
1368 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1369 dynasm!(self ; subs W(dst), W(src1), W(src2));
1370 }
1371 (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1372 dynasm!(self ; subs X(dst), XSP(src1), imm as u32);
1373 }
1374 (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1375 dynasm!(self ; subs W(dst), WSP(src1), imm as u32);
1376 }
1377 _ => codegen_error!(
1378 "singlepass can't emit SUB.S {:?} {:?} {:?} {:?}",
1379 sz,
1380 src1,
1381 src2,
1382 dst
1383 ),
1384 }
1385 Ok(())
1386 }
1387 fn emit_add_lsl(
1388 &mut self,
1389 sz: Size,
1390 src1: Location,
1391 src2: Location,
1392 lsl: u32,
1393 dst: Location,
1394 ) -> Result<(), CompileError> {
1395 match (sz, src1, src2, dst) {
1396 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1397 dynasm!(self ; add X(dst), X(src1), X(src2), LSL lsl);
1398 }
1399 _ => codegen_error!(
1400 "singlepass can't emit LSL {:?} {:?} {:?} {:?} LSL {:?}",
1401 sz,
1402 src1,
1403 src2,
1404 dst,
1405 lsl
1406 ),
1407 }
1408 Ok(())
1409 }
1410
1411 fn emit_cmp(&mut self, sz: Size, left: Location, right: Location) -> Result<(), CompileError> {
1417 match (sz, left, right) {
1418 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1419 dynasm!(self ; cmp X(dst), X(src));
1420 }
1421 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1422 dynasm!(self ; cmp W(dst), W(src));
1423 }
1424 (Size::S64, Location::Imm8(imm), Location::GPR(dst)) => {
1425 dynasm!(self ; cmp XSP(dst), imm as u32);
1426 }
1427 (Size::S64, Location::Imm32(imm), Location::GPR(dst)) => {
1428 if imm >= 0x1000 {
1429 codegen_error!("singlepass CMP with imm too large {}", imm);
1430 }
1431 dynasm!(self ; cmp XSP(dst), imm as u32);
1432 }
1433 (Size::S64, Location::Imm64(imm), Location::GPR(dst)) => {
1434 if imm >= 0x1000 {
1435 codegen_error!("singlepass CMP with imm too large {}", imm);
1436 }
1437 dynasm!(self ; cmp XSP(dst), imm as u32);
1438 }
1439 (Size::S32, Location::Imm8(imm), Location::GPR(dst)) => {
1440 dynasm!(self ; cmp WSP(dst), imm as u32);
1441 }
1442 (Size::S32, Location::Imm32(imm), Location::GPR(dst)) => {
1443 if imm >= 0x1000 {
1444 codegen_error!("singlepass CMP with imm too large {}", imm);
1445 }
1446 dynasm!(self ; cmp WSP(dst), imm as u32);
1447 }
1448 _ => codegen_error!("singlepass can't emit CMP {:?} {:?} {:?}", sz, left, right),
1449 }
1450 Ok(())
1451 }
1452
1453 fn emit_tst(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1454 match (sz, src, dst) {
1455 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1456 dynasm!(self ; tst X(dst), X(src));
1457 }
1458 (Size::S64, Location::Imm32(imm), Location::GPR(dst)) => {
1459 if encode_logical_immediate_64bit(imm as u64).is_none() {
1460 codegen_error!("singlepass TST with incompatible imm {}", imm);
1461 }
1462 dynasm!(self ; tst X(dst), imm as u64);
1463 }
1464 (Size::S64, Location::Imm64(imm), Location::GPR(dst)) => {
1465 if encode_logical_immediate_64bit(imm as u64).is_none() {
1466 codegen_error!("singlepass TST with incompatible imm {}", imm);
1467 }
1468 dynasm!(self ; tst X(dst), imm as u64);
1469 }
1470 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1471 dynasm!(self ; tst W(dst), W(src));
1472 }
1473 (Size::S32, Location::Imm32(imm), Location::GPR(dst)) => {
1474 if encode_logical_immediate_64bit(imm as u64).is_none() {
1475 codegen_error!("singlepass TST with incompatible imm {}", imm);
1476 }
1477 dynasm!(self ; tst W(dst), imm);
1478 }
1479 _ => codegen_error!("singlepass can't emit TST"),
1480 }
1481 Ok(())
1482 }
1483
1484 fn emit_lsl(
1485 &mut self,
1486 sz: Size,
1487 src1: Location,
1488 src2: Location,
1489 dst: Location,
1490 ) -> Result<(), CompileError> {
1491 match (sz, src1, src2, dst) {
1492 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1493 dynasm!(self ; lsl X(dst), X(src1), X(src2));
1494 }
1495 (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1496 if imm > 63 {
1497 codegen_error!("singlepass LSL with incompatible imm {}", imm);
1498 }
1499 let imm = imm as u32;
1500
1501 dynasm!(self ; lsl X(dst), X(src1), imm);
1502 }
1503 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1504 dynasm!(self ; lsl W(dst), W(src1), W(src2));
1505 }
1506 (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1507 if imm > 63 {
1508 codegen_error!("singlepass LSL with incompatible imm {}", imm);
1509 }
1510 dynasm!(self ; lsl X(dst), X(src1), imm as u32);
1511 }
1512 (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
1513 if imm > 63 {
1514 codegen_error!("singlepass LSL with incompatible imm {}", imm);
1515 }
1516 dynasm!(self ; lsl X(dst), X(src1), imm as u32);
1517 }
1518 (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1519 if imm > 31 {
1520 codegen_error!("singlepass LSL with incompatible imm {}", imm);
1521 }
1522 dynasm!(self ; lsl W(dst), W(src1), imm as u32);
1523 }
1524 (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1525 if imm > 31 {
1526 codegen_error!("singlepass LSL with incompatible imm {}", imm);
1527 }
1528 dynasm!(self ; lsl W(dst), W(src1), imm as u32);
1529 }
1530 _ => codegen_error!(
1531 "singlepass can't emit LSL {:?} {:?} {:?} {:?}",
1532 sz,
1533 src1,
1534 src2,
1535 dst
1536 ),
1537 }
1538 Ok(())
1539 }
1540 fn emit_asr(
1541 &mut self,
1542 sz: Size,
1543 src1: Location,
1544 src2: Location,
1545 dst: Location,
1546 ) -> Result<(), CompileError> {
1547 match (sz, src1, src2, dst) {
1548 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1549 dynasm!(self ; asr X(dst), X(src1), X(src2));
1550 }
1551 (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1552 let imm = imm as u32;
1553
1554 if imm == 0 || imm > 63 {
1555 codegen_error!("singlepass ASR with incompatible imm {}", imm);
1556 }
1557 dynasm!(self ; asr X(dst), X(src1), imm);
1558 }
1559 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1560 dynasm!(self ; asr W(dst), W(src1), W(src2));
1561 }
1562 (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1563 if imm == 0 || imm > 63 {
1564 codegen_error!("singlepass ASR with incompatible imm {}", imm);
1565 }
1566 dynasm!(self ; asr X(dst), X(src1), imm as u32);
1567 }
1568 (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
1569 if imm == 0 || imm > 63 {
1570 codegen_error!("singlepass ASR with incompatible imm {}", imm);
1571 }
1572 dynasm!(self ; asr X(dst), X(src1), imm as u32);
1573 }
1574 (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1575 if imm == 0 || imm > 31 {
1576 codegen_error!("singlepass ASR with incompatible imm {}", imm);
1577 }
1578 dynasm!(self ; asr W(dst), W(src1), imm as u32);
1579 }
1580 (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1581 if imm == 0 || imm > 31 {
1582 codegen_error!("singlepass ASR with incompatible imm {}", imm);
1583 }
1584 dynasm!(self ; asr W(dst), W(src1), imm as u32);
1585 }
1586 _ => codegen_error!(
1587 "singlepass can't emit ASR {:?} {:?} {:?} {:?}",
1588 sz,
1589 src1,
1590 src2,
1591 dst
1592 ),
1593 }
1594 Ok(())
1595 }
1596 fn emit_lsr(
1597 &mut self,
1598 sz: Size,
1599 src1: Location,
1600 src2: Location,
1601 dst: Location,
1602 ) -> Result<(), CompileError> {
1603 match (sz, src1, src2, dst) {
1604 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1605 dynasm!(self ; lsr X(dst), X(src1), X(src2));
1606 }
1607 (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1608 let imm = imm as u32;
1609
1610 if imm == 0 || imm > 63 {
1611 codegen_error!("singlepass LSR with incompatible imm {}", imm);
1612 }
1613 dynasm!(self ; lsr X(dst), X(src1), imm);
1614 }
1615 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1616 dynasm!(self ; lsr W(dst), W(src1), W(src2));
1617 }
1618 (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1619 if imm == 0 || imm > 63 {
1620 codegen_error!("singlepass LSR with incompatible imm {}", imm);
1621 }
1622 dynasm!(self ; lsr X(dst), X(src1), imm as u32);
1623 }
1624 (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
1625 if imm == 0 || imm > 63 {
1626 codegen_error!("singlepass LSR with incompatible imm {}", imm);
1627 }
1628 dynasm!(self ; lsr X(dst), X(src1), imm as u32);
1629 }
1630 (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1631 if imm == 0 || imm > 31 {
1632 codegen_error!("singlepass LSR with incompatible imm {}", imm);
1633 }
1634 dynasm!(self ; lsr W(dst), W(src1), imm as u32);
1635 }
1636 (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1637 if imm == 0 || imm > 31 {
1638 codegen_error!("singlepass LSR with incompatible imm {}", imm);
1639 }
1640 dynasm!(self ; lsr W(dst), W(src1), imm as u32);
1641 }
1642 _ => codegen_error!(
1643 "singlepass can't emit LSR {:?} {:?} {:?} {:?}",
1644 sz,
1645 src1,
1646 src2,
1647 dst
1648 ),
1649 }
1650 Ok(())
1651 }
1652 fn emit_ror(
1653 &mut self,
1654 sz: Size,
1655 src1: Location,
1656 src2: Location,
1657 dst: Location,
1658 ) -> Result<(), CompileError> {
1659 match (sz, src1, src2, dst) {
1660 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1661 dynasm!(self ; ror X(dst), X(src1), X(src2));
1662 }
1663 (Size::S64, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1664 let imm = imm as u32;
1665
1666 if imm == 0 || imm > 63 {
1667 codegen_error!("singlepass ROR with incompatible imm {}", imm);
1668 }
1669 dynasm!(self ; ror X(dst), X(src1), imm);
1670 }
1671 (Size::S64, Location::GPR(src1), Location::Imm64(imm), Location::GPR(dst)) => {
1672 let imm = imm as u32;
1673
1674 if imm == 0 || imm > 63 {
1675 codegen_error!("singlepass ROR with incompatible imm {}", imm);
1676 }
1677 dynasm!(self ; ror X(dst), X(src1), imm);
1678 }
1679 (Size::S32, Location::GPR(src1), Location::Imm32(imm), Location::GPR(dst)) => {
1680 if imm == 0 || imm > 31 {
1681 codegen_error!("singlepass ROR with incompatible imm {}", imm);
1682 }
1683 dynasm!(self ; ror W(dst), W(src1), imm as u32);
1684 }
1685 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1686 dynasm!(self ; ror W(dst), W(src1), W(src2));
1687 }
1688 (Size::S64, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1689 if imm == 0 || imm > 63 {
1690 codegen_error!("singlepass ROR with incompatible imm {}", imm);
1691 }
1692 dynasm!(self ; ror X(dst), X(src1), imm as u32);
1693 }
1694 (Size::S32, Location::GPR(src1), Location::Imm8(imm), Location::GPR(dst)) => {
1695 if imm == 0 || imm > 31 {
1696 codegen_error!("singlepass ROR with incompatible imm {}", imm);
1697 }
1698 dynasm!(self ; ror W(dst), W(src1), imm as u32);
1699 }
1700 _ => codegen_error!(
1701 "singlepass can't emit ROR {:?} {:?} {:?} {:?}",
1702 sz,
1703 src1,
1704 src2,
1705 dst
1706 ),
1707 }
1708 Ok(())
1709 }
1710
1711 fn emit_or(
1712 &mut self,
1713 sz: Size,
1714 src1: Location,
1715 src2: Location,
1716 dst: Location,
1717 ) -> Result<(), CompileError> {
1718 match (sz, src1, src2, dst) {
1719 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1720 dynasm!(self ; orr X(dst), X(src1), X(src2));
1721 }
1722 (Size::S64, Location::GPR(src1), Location::Imm64(src2), Location::GPR(dst)) => {
1723 let src2 = src2 as u64;
1724
1725 if encode_logical_immediate_64bit(src2 as u64).is_none() {
1726 codegen_error!("singlepass OR with incompatible imm {}", src2);
1727 }
1728 dynasm!(self ; orr XSP(dst), X(src1), src2);
1729 }
1730 (Size::S32, Location::GPR(src1), Location::Imm32(src2), Location::GPR(dst)) => {
1731 let src2 = src2 as u32;
1732
1733 if encode_logical_immediate_32bit(src2).is_none() {
1734 codegen_error!("singlepass OR with incompatible imm {}", src2);
1735 }
1736 dynasm!(self ; orr WSP(dst), W(src1), src2);
1737 }
1738 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1739 dynasm!(self ; orr W(dst), W(src1), W(src2));
1740 }
1741 _ => codegen_error!(
1742 "singlepass can't emit OR {:?} {:?} {:?} {:?}",
1743 sz,
1744 src1,
1745 src2,
1746 dst
1747 ),
1748 }
1749 Ok(())
1750 }
1751 fn emit_and(
1752 &mut self,
1753 sz: Size,
1754 src1: Location,
1755 src2: Location,
1756 dst: Location,
1757 ) -> Result<(), CompileError> {
1758 match (sz, src1, src2, dst) {
1759 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1760 dynasm!(self ; and X(dst), X(src1), X(src2));
1761 }
1762 (Size::S64, Location::GPR(src1), Location::Imm64(src2), Location::GPR(dst)) => {
1763 let src2 = src2 as u64;
1764
1765 if encode_logical_immediate_64bit(src2 as u64).is_none() {
1766 codegen_error!("singlepass AND with incompatible imm {}", src2);
1767 }
1768 dynasm!(self ; and XSP(dst), X(src1), src2);
1769 }
1770 (Size::S32, Location::GPR(src1), Location::Imm32(src2), Location::GPR(dst)) => {
1771 let src2 = src2 as u32;
1772
1773 if encode_logical_immediate_32bit(src2).is_none() {
1774 codegen_error!("singlepass AND with incompatible imm {}", src2);
1775 }
1776 dynasm!(self ; and WSP(dst), W(src1), src2);
1777 }
1778 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1779 dynasm!(self ; and W(dst), W(src1), W(src2));
1780 }
1781 _ => codegen_error!(
1782 "singlepass can't emit AND {:?} {:?} {:?} {:?}",
1783 sz,
1784 src1,
1785 src2,
1786 dst
1787 ),
1788 }
1789 Ok(())
1790 }
1791 fn emit_eor(
1792 &mut self,
1793 sz: Size,
1794 src1: Location,
1795 src2: Location,
1796 dst: Location,
1797 ) -> Result<(), CompileError> {
1798 match (sz, src1, src2, dst) {
1799 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1800 dynasm!(self ; eor X(dst), X(src1), X(src2));
1801 }
1802 (Size::S64, Location::GPR(src1), Location::Imm64(src2), Location::GPR(dst)) => {
1803 let src2 = src2 as u64;
1804
1805 if encode_logical_immediate_64bit(src2 as u64).is_none() {
1806 codegen_error!("singlepass EOR with incompatible imm {}", src2);
1807 }
1808 dynasm!(self ; eor XSP(dst), X(src1), src2);
1809 }
1810 (Size::S32, Location::GPR(src1), Location::Imm32(src2), Location::GPR(dst)) => {
1811 let src2 = src2 as u32;
1812
1813 if encode_logical_immediate_32bit(src2).is_none() {
1814 codegen_error!("singlepass EOR with incompatible imm {}", src2);
1815 }
1816 dynasm!(self ; eor WSP(dst), W(src1), src2);
1817 }
1818 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1819 dynasm!(self ; eor W(dst), W(src1), W(src2));
1820 }
1821 _ => codegen_error!(
1822 "singlepass can't emit EOR {:?} {:?} {:?} {:?}",
1823 sz,
1824 src1,
1825 src2,
1826 dst
1827 ),
1828 }
1829 Ok(())
1830 }
1831
1832 fn emit_bfc(
1833 &mut self,
1834 sz: Size,
1835 lsb: u32,
1836 width: u32,
1837 dst: Location,
1838 ) -> Result<(), CompileError> {
1839 match (sz, dst) {
1840 (Size::S32, Location::GPR(dst)) => {
1841 dynasm!(self ; bfc W(dst), lsb, width);
1842 }
1843 (Size::S64, Location::GPR(dst)) => {
1844 dynasm!(self ; bfc X(dst), lsb, width);
1845 }
1846 _ => codegen_error!("singlepass can't emit BFC"),
1847 }
1848 Ok(())
1849 }
1850 fn emit_bfi(
1851 &mut self,
1852 sz: Size,
1853 src: Location,
1854 lsb: u32,
1855 width: u32,
1856 dst: Location,
1857 ) -> Result<(), CompileError> {
1858 match (sz, src, dst) {
1859 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1860 dynasm!(self ; bfi W(dst), W(src), lsb, width);
1861 }
1862 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1863 dynasm!(self ; bfi X(dst), X(src), lsb, width);
1864 }
1865 _ => codegen_error!("singlepass can't emit BFI"),
1866 }
1867 Ok(())
1868 }
1869
1870 fn emit_udiv(
1871 &mut self,
1872 sz: Size,
1873 src1: Location,
1874 src2: Location,
1875 dst: Location,
1876 ) -> Result<(), CompileError> {
1877 match (sz, src1, src2, dst) {
1878 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1879 dynasm!(self ; udiv W(dst), W(src1), W(src2));
1880 }
1881 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1882 dynasm!(self ; udiv X(dst), X(src1), X(src2));
1883 }
1884 _ => codegen_error!(
1885 "singlepass can't emit UDIV {:?} {:?} {:?} {:?}",
1886 sz,
1887 src1,
1888 src2,
1889 dst
1890 ),
1891 }
1892 Ok(())
1893 }
1894 fn emit_sdiv(
1895 &mut self,
1896 sz: Size,
1897 src1: Location,
1898 src2: Location,
1899 dst: Location,
1900 ) -> Result<(), CompileError> {
1901 match (sz, src1, src2, dst) {
1902 (Size::S32, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1903 dynasm!(self ; sdiv W(dst), W(src1), W(src2));
1904 }
1905 (Size::S64, Location::GPR(src1), Location::GPR(src2), Location::GPR(dst)) => {
1906 dynasm!(self ; sdiv X(dst), X(src1), X(src2));
1907 }
1908 _ => codegen_error!(
1909 "singlepass can't emit SDIV {:?} {:?} {:?} {:?}",
1910 sz,
1911 src1,
1912 src2,
1913 dst
1914 ),
1915 }
1916 Ok(())
1917 }
1918
1919 fn emit_msub(
1921 &mut self,
1922 sz: Size,
1923 a: Location,
1924 b: Location,
1925 c: Location,
1926 dst: Location,
1927 ) -> Result<(), CompileError> {
1928 match (sz, a, b, c, dst) {
1929 (
1930 Size::S32,
1931 Location::GPR(a),
1932 Location::GPR(b),
1933 Location::GPR(c),
1934 Location::GPR(dst),
1935 ) => {
1936 dynasm!(self ; msub W(dst), W(a), W(b), W(c));
1937 }
1938 (
1939 Size::S64,
1940 Location::GPR(a),
1941 Location::GPR(b),
1942 Location::GPR(c),
1943 Location::GPR(dst),
1944 ) => {
1945 dynasm!(self ; msub X(dst), X(a), X(b), X(c));
1946 }
1947 _ => codegen_error!(
1948 "singlepass can't emit msub {:?} {:?} {:?} {:?} {:?}",
1949 sz,
1950 a,
1951 b,
1952 c,
1953 dst
1954 ),
1955 }
1956 Ok(())
1957 }
1958
1959 fn emit_sxtb(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1960 match (sz, src, dst) {
1961 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1962 dynasm!(self ; sxtb W(dst), W(src));
1963 }
1964 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1965 dynasm!(self ; sxtb X(dst), W(src));
1966 }
1967 _ => codegen_error!("singlepass can't emit SXTB {:?} {:?} {:?}", sz, src, dst),
1968 }
1969 Ok(())
1970 }
1971 fn emit_sxth(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1972 match (sz, src, dst) {
1973 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
1974 dynasm!(self ; sxth W(dst), W(src));
1975 }
1976 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
1977 dynasm!(self ; sxth X(dst), W(src));
1978 }
1979 _ => codegen_error!("singlepass can't emit SXTH {:?} {:?} {:?}", sz, src, dst),
1980 }
1981 Ok(())
1982 }
1983 fn emit_sxtw(&mut self, _sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1984 match (src, dst) {
1985 (Location::GPR(src), Location::GPR(dst)) => {
1986 dynasm!(self ; sxtw X(dst), W(src));
1987 }
1988 _ => codegen_error!("singlepass can't emit SXTW {:?} {:?}", src, dst),
1989 }
1990 Ok(())
1991 }
1992 fn emit_uxtb(&mut self, _sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1993 match (src, dst) {
1994 (Location::GPR(src), Location::GPR(dst)) => {
1995 dynasm!(self ; uxtb W(dst), W(src));
1996 }
1997 _ => codegen_error!("singlepass can't emit UXTB {:?} {:?}", src, dst),
1998 }
1999 Ok(())
2000 }
2001 fn emit_uxth(&mut self, _sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2002 match (src, dst) {
2003 (Location::GPR(src), Location::GPR(dst)) => {
2004 dynasm!(self ; uxth W(dst), W(src));
2005 }
2006 _ => codegen_error!("singlepass can't emit UXTH {:?} {:?}", src, dst),
2007 }
2008 Ok(())
2009 }
2010
2011 fn emit_cset(&mut self, sz: Size, dst: Location, cond: Condition) -> Result<(), CompileError> {
2012 match (sz, dst) {
2013 (Size::S32, Location::GPR(reg)) => match cond {
2014 Condition::Eq => dynasm!(self ; cset W(reg), eq),
2015 Condition::Ne => dynasm!(self ; cset W(reg), ne),
2016 Condition::Cs => dynasm!(self ; cset W(reg), cs),
2017 Condition::Cc => dynasm!(self ; cset W(reg), cc),
2018 Condition::Mi => dynasm!(self ; cset W(reg), mi),
2019 Condition::Pl => dynasm!(self ; cset W(reg), pl),
2020 Condition::Vs => dynasm!(self ; cset W(reg), vs),
2021 Condition::Vc => dynasm!(self ; cset W(reg), vc),
2022 Condition::Hi => dynasm!(self ; cset W(reg), hi),
2023 Condition::Ls => dynasm!(self ; cset W(reg), ls),
2024 Condition::Ge => dynasm!(self ; cset W(reg), ge),
2025 Condition::Lt => dynasm!(self ; cset W(reg), lt),
2026 Condition::Gt => dynasm!(self ; cset W(reg), gt),
2027 Condition::Le => dynasm!(self ; cset W(reg), le),
2028 Condition::Al => dynasm!(self ; cset W(reg), al),
2029 },
2030 (Size::S64, Location::GPR(reg)) => match cond {
2031 Condition::Eq => dynasm!(self ; cset X(reg), eq),
2032 Condition::Ne => dynasm!(self ; cset X(reg), ne),
2033 Condition::Cs => dynasm!(self ; cset X(reg), cs),
2034 Condition::Cc => dynasm!(self ; cset X(reg), cc),
2035 Condition::Mi => dynasm!(self ; cset X(reg), mi),
2036 Condition::Pl => dynasm!(self ; cset X(reg), pl),
2037 Condition::Vs => dynasm!(self ; cset X(reg), vs),
2038 Condition::Vc => dynasm!(self ; cset X(reg), vc),
2039 Condition::Hi => dynasm!(self ; cset X(reg), hi),
2040 Condition::Ls => dynasm!(self ; cset X(reg), ls),
2041 Condition::Ge => dynasm!(self ; cset X(reg), ge),
2042 Condition::Lt => dynasm!(self ; cset X(reg), lt),
2043 Condition::Gt => dynasm!(self ; cset X(reg), gt),
2044 Condition::Le => dynasm!(self ; cset X(reg), le),
2045 Condition::Al => dynasm!(self ; cset X(reg), al),
2046 },
2047 _ => codegen_error!("singlepass can't emit CSET {:?} {:?} {:?}", sz, dst, cond),
2048 }
2049 Ok(())
2050 }
2051 fn emit_csetm(&mut self, sz: Size, dst: Location, cond: Condition) -> Result<(), CompileError> {
2052 match (sz, dst) {
2053 (Size::S32, Location::GPR(reg)) => match cond {
2054 Condition::Eq => dynasm!(self ; csetm W(reg), eq),
2055 Condition::Ne => dynasm!(self ; csetm W(reg), ne),
2056 Condition::Cs => dynasm!(self ; csetm W(reg), cs),
2057 Condition::Cc => dynasm!(self ; csetm W(reg), cc),
2058 Condition::Mi => dynasm!(self ; csetm W(reg), mi),
2059 Condition::Pl => dynasm!(self ; csetm W(reg), pl),
2060 Condition::Vs => dynasm!(self ; csetm W(reg), vs),
2061 Condition::Vc => dynasm!(self ; csetm W(reg), vc),
2062 Condition::Hi => dynasm!(self ; csetm W(reg), hi),
2063 Condition::Ls => dynasm!(self ; csetm W(reg), ls),
2064 Condition::Ge => dynasm!(self ; csetm W(reg), ge),
2065 Condition::Lt => dynasm!(self ; csetm W(reg), lt),
2066 Condition::Gt => dynasm!(self ; csetm W(reg), gt),
2067 Condition::Le => dynasm!(self ; csetm W(reg), le),
2068 Condition::Al => dynasm!(self ; csetm W(reg), al),
2069 },
2070 (Size::S64, Location::GPR(reg)) => match cond {
2071 Condition::Eq => dynasm!(self ; csetm X(reg), eq),
2072 Condition::Ne => dynasm!(self ; csetm X(reg), ne),
2073 Condition::Cs => dynasm!(self ; csetm X(reg), cs),
2074 Condition::Cc => dynasm!(self ; csetm X(reg), cc),
2075 Condition::Mi => dynasm!(self ; csetm X(reg), mi),
2076 Condition::Pl => dynasm!(self ; csetm X(reg), pl),
2077 Condition::Vs => dynasm!(self ; csetm X(reg), vs),
2078 Condition::Vc => dynasm!(self ; csetm X(reg), vc),
2079 Condition::Hi => dynasm!(self ; csetm X(reg), hi),
2080 Condition::Ls => dynasm!(self ; csetm X(reg), ls),
2081 Condition::Ge => dynasm!(self ; csetm X(reg), ge),
2082 Condition::Lt => dynasm!(self ; csetm X(reg), lt),
2083 Condition::Gt => dynasm!(self ; csetm X(reg), gt),
2084 Condition::Le => dynasm!(self ; csetm X(reg), le),
2085 Condition::Al => dynasm!(self ; csetm X(reg), al),
2086 },
2087 _ => codegen_error!("singlepass can't emit CSETM {:?} {:?} {:?}", sz, dst, cond),
2088 }
2089 Ok(())
2090 }
2091 fn emit_cinc(
2092 &mut self,
2093 sz: Size,
2094 src: Location,
2095 dst: Location,
2096 cond: Condition,
2097 ) -> Result<(), CompileError> {
2098 match (sz, src, dst) {
2099 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
2100 match cond {
2101 Condition::Eq => dynasm!(self ; cinc W(dst), W(src), eq),
2102 Condition::Ne => dynasm!(self ; cinc W(dst), W(src), ne),
2103 Condition::Cs => dynasm!(self ; cinc W(dst), W(src), cs),
2104 Condition::Cc => dynasm!(self ; cinc W(dst), W(src), cc),
2105 Condition::Mi => dynasm!(self ; cinc W(dst), W(src), mi),
2106 Condition::Pl => dynasm!(self ; cinc W(dst), W(src), pl),
2107 Condition::Vs => dynasm!(self ; cinc W(dst), W(src), vs),
2108 Condition::Vc => dynasm!(self ; cinc W(dst), W(src), vc),
2109 Condition::Hi => dynasm!(self ; cinc W(dst), W(src), hi),
2110 Condition::Ls => dynasm!(self ; cinc W(dst), W(src), ls),
2111 Condition::Ge => dynasm!(self ; cinc W(dst), W(src), ge),
2112 Condition::Lt => dynasm!(self ; cinc W(dst), W(src), lt),
2113 Condition::Gt => dynasm!(self ; cinc W(dst), W(src), gt),
2114 Condition::Le => dynasm!(self ; cinc W(dst), W(src), le),
2115 Condition::Al => dynasm!(self ; cinc W(dst), W(src), al),
2116 };
2117 }
2118 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
2119 match cond {
2120 Condition::Eq => dynasm!(self ; cinc X(src), X(dst), eq),
2121 Condition::Ne => dynasm!(self ; cinc X(src), X(dst), ne),
2122 Condition::Cs => dynasm!(self ; cinc X(src), X(dst), cs),
2123 Condition::Cc => dynasm!(self ; cinc X(src), X(dst), cc),
2124 Condition::Mi => dynasm!(self ; cinc X(src), X(dst), mi),
2125 Condition::Pl => dynasm!(self ; cinc X(src), X(dst), pl),
2126 Condition::Vs => dynasm!(self ; cinc X(src), X(dst), vs),
2127 Condition::Vc => dynasm!(self ; cinc X(src), X(dst), vc),
2128 Condition::Hi => dynasm!(self ; cinc X(src), X(dst), hi),
2129 Condition::Ls => dynasm!(self ; cinc X(src), X(dst), ls),
2130 Condition::Ge => dynasm!(self ; cinc X(src), X(dst), ge),
2131 Condition::Lt => dynasm!(self ; cinc X(src), X(dst), lt),
2132 Condition::Gt => dynasm!(self ; cinc X(src), X(dst), gt),
2133 Condition::Le => dynasm!(self ; cinc X(src), X(dst), le),
2134 Condition::Al => dynasm!(self ; cinc X(src), X(dst), al),
2135 };
2136 }
2137 _ => codegen_error!("singlepass can't emit CINC"),
2138 }
2139 Ok(())
2140 }
2141
2142 fn emit_clz(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2143 match (sz, src, dst) {
2144 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
2145 dynasm!(self ; clz X(dst), X(src));
2146 }
2147 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
2148 dynasm!(self ; clz W(dst), W(src));
2149 }
2150 _ => codegen_error!("singlepass can't emit CLS {:?} {:?} {:?}", sz, src, dst),
2151 }
2152 Ok(())
2153 }
2154 fn emit_rbit(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2155 match (sz, src, dst) {
2156 (Size::S64, Location::GPR(src), Location::GPR(dst)) => {
2157 dynasm!(self ; rbit X(dst), X(src));
2158 }
2159 (Size::S32, Location::GPR(src), Location::GPR(dst)) => {
2160 dynasm!(self ; rbit W(dst), W(src));
2161 }
2162 _ => codegen_error!("singlepass can't emit CLS {:?} {:?} {:?}", sz, src, dst),
2163 }
2164 Ok(())
2165 }
2166
2167 fn emit_label(&mut self, label: Label) -> Result<(), CompileError> {
2168 dynasm!(self ; => label);
2169 Ok(())
2170 }
2171 fn emit_load_label(&mut self, reg: GPR, label: Label) -> Result<(), CompileError> {
2172 dynasm!(self ; adr X(reg), =>label);
2173 Ok(())
2174 }
2175
2176 fn emit_b_label(&mut self, label: Label) -> Result<(), CompileError> {
2177 dynasm!(self ; b =>label);
2178 Ok(())
2179 }
2180 fn emit_cbz_label(
2181 &mut self,
2182 sz: Size,
2183 reg: Location,
2184 label: Label,
2185 ) -> Result<(), CompileError> {
2186 match (sz, reg) {
2187 (Size::S32, Location::GPR(reg)) => {
2188 dynasm!(self ; cbz W(reg), =>label);
2189 }
2190 (Size::S64, Location::GPR(reg)) => {
2191 dynasm!(self ; cbz X(reg), =>label);
2192 }
2193 _ => codegen_error!("singlepass can't emit CBZ {:?} {:?} {:?}", sz, reg, label),
2194 }
2195 Ok(())
2196 }
2197 fn emit_cbnz_label(
2198 &mut self,
2199 sz: Size,
2200 reg: Location,
2201 label: Label,
2202 ) -> Result<(), CompileError> {
2203 match (sz, reg) {
2204 (Size::S32, Location::GPR(reg)) => {
2205 dynasm!(self ; cbnz W(reg), =>label);
2206 }
2207 (Size::S64, Location::GPR(reg)) => {
2208 dynasm!(self ; cbnz X(reg), =>label);
2209 }
2210 _ => codegen_error!("singlepass can't emit CBNZ {:?} {:?} {:?}", sz, reg, label),
2211 }
2212 Ok(())
2213 }
2214 fn emit_cbz_label_far(
2215 &mut self,
2216 sz: Size,
2217 reg: Location,
2218 label: Label,
2219 ) -> Result<(), CompileError> {
2220 let near_label: Label = self.get_label();
2221 let continue_label: Label = self.get_label();
2222
2223 match (sz, reg) {
2224 (Size::S32, Location::GPR(reg)) => {
2225 dynasm!(self ; cbz W(reg), => near_label);
2226 dynasm!(self ; b => continue_label);
2227 }
2228 (Size::S64, Location::GPR(reg)) => {
2229 dynasm!(self ; cbz X(reg), => near_label);
2230 dynasm!(self ; b => continue_label);
2231 }
2232 _ => codegen_error!("singlepass can't emit CBZ {:?} {:?} {:?}", sz, reg, label),
2233 }
2234 self.emit_label(near_label)?;
2235 dynasm!(self ; b => label );
2236
2237 self.emit_label(continue_label)?;
2238
2239 Ok(())
2240 }
2241 fn emit_tbz_label(
2242 &mut self,
2243 sz: Size,
2244 reg: Location,
2245 n: u32,
2246 label: Label,
2247 ) -> Result<(), CompileError> {
2248 match (sz, reg) {
2249 (Size::S32, Location::GPR(reg)) => {
2250 dynasm!(self ; tbz W(reg), n, =>label);
2251 }
2252 (Size::S64, Location::GPR(reg)) => {
2253 dynasm!(self ; tbz X(reg), n, =>label);
2254 }
2255 _ => codegen_error!(
2256 "singlepass can't emit TBZ {:?} {:?} {:?} {:?}",
2257 sz,
2258 reg,
2259 n,
2260 label
2261 ),
2262 }
2263 Ok(())
2264 }
2265 fn emit_tbnz_label(
2266 &mut self,
2267 sz: Size,
2268 reg: Location,
2269 n: u32,
2270 label: Label,
2271 ) -> Result<(), CompileError> {
2272 match (sz, reg) {
2273 (Size::S32, Location::GPR(reg)) => {
2274 dynasm!(self ; tbnz W(reg), n, =>label);
2275 }
2276 (Size::S64, Location::GPR(reg)) => {
2277 dynasm!(self ; tbnz X(reg), n, =>label);
2278 }
2279 _ => codegen_error!(
2280 "singlepass can't emit TBNZ {:?} {:?} {:?} {:?}",
2281 sz,
2282 reg,
2283 n,
2284 label
2285 ),
2286 }
2287 Ok(())
2288 }
2289 fn emit_bcond_label(&mut self, condition: Condition, label: Label) -> Result<(), CompileError> {
2290 match condition {
2291 Condition::Eq => dynasm!(self ; b.eq => label),
2292 Condition::Ne => dynasm!(self ; b.ne => label),
2293 Condition::Cs => dynasm!(self ; b.cs => label),
2294 Condition::Cc => dynasm!(self ; b.cc => label),
2295 Condition::Mi => dynasm!(self ; b.mi => label),
2296 Condition::Pl => dynasm!(self ; b.pl => label),
2297 Condition::Vs => dynasm!(self ; b.vs => label),
2298 Condition::Vc => dynasm!(self ; b.vc => label),
2299 Condition::Hi => dynasm!(self ; b.hi => label),
2300 Condition::Ls => dynasm!(self ; b.ls => label),
2301 Condition::Ge => dynasm!(self ; b.ge => label),
2302 Condition::Lt => dynasm!(self ; b.lt => label),
2303 Condition::Gt => dynasm!(self ; b.gt => label),
2304 Condition::Le => dynasm!(self ; b.le => label),
2305 Condition::Al => dynasm!(self ; b => label),
2306 }
2307 Ok(())
2308 }
2309 fn emit_bcond_label_far(
2310 &mut self,
2311 condition: Condition,
2312 label: Label,
2313 ) -> Result<(), CompileError> {
2314 let cont: Label = self.get_label();
2315 match condition {
2316 Condition::Eq => dynasm!(self ; b.ne => cont),
2318 Condition::Ne => dynasm!(self ; b.eq => cont),
2319 Condition::Cs => dynasm!(self ; b.cc => cont),
2320 Condition::Cc => dynasm!(self ; b.cs => cont),
2321 Condition::Mi => dynasm!(self ; b.pl => cont),
2322 Condition::Pl => dynasm!(self ; b.mi => cont),
2323 Condition::Vs => dynasm!(self ; b.vc => cont),
2324 Condition::Vc => dynasm!(self ; b.vs => cont),
2325 Condition::Hi => dynasm!(self ; b.ls => cont),
2326 Condition::Ls => dynasm!(self ; b.hi => cont),
2327 Condition::Ge => dynasm!(self ; b.lt => cont),
2328 Condition::Lt => dynasm!(self ; b.ge => cont),
2329 Condition::Gt => dynasm!(self ; b.le => cont),
2330 Condition::Le => dynasm!(self ; b.gt => cont),
2331 Condition::Al => { }
2332 }
2333 dynasm!(self ; b => label);
2334 self.emit_label(cont)?;
2335 Ok(())
2336 }
2337 fn emit_b_register(&mut self, reg: GPR) -> Result<(), CompileError> {
2338 dynasm!(self ; br X(reg));
2339 Ok(())
2340 }
2341 fn emit_call_label(&mut self, label: Label) -> Result<(), CompileError> {
2342 dynasm!(self ; bl =>label);
2343 Ok(())
2344 }
2345 fn emit_call_register(&mut self, reg: GPR) -> Result<(), CompileError> {
2346 dynasm!(self ; blr X(reg));
2347 Ok(())
2348 }
2349 fn emit_ret(&mut self) -> Result<(), CompileError> {
2350 dynasm!(self ; ret);
2351 Ok(())
2352 }
2353
2354 fn emit_udf(&mut self, payload: u16) -> Result<(), CompileError> {
2355 dynasm!(self ; udf payload as u32);
2356 Ok(())
2357 }
2358 fn emit_dmb(&mut self) -> Result<(), CompileError> {
2359 dynasm!(self ; dmb ish);
2360 Ok(())
2361 }
2362 fn emit_brk(&mut self) -> Result<(), CompileError> {
2363 dynasm!(self ; brk 0);
2364 Ok(())
2365 }
2366
2367 fn emit_fcmp(&mut self, sz: Size, src1: Location, src2: Location) -> Result<(), CompileError> {
2368 match (sz, src1, src2) {
2369 (Size::S32, Location::SIMD(src1), Location::SIMD(src2)) => {
2370 dynasm!(self ; fcmp S(src1), S(src2));
2371 }
2372 (Size::S64, Location::SIMD(src1), Location::SIMD(src2)) => {
2373 dynasm!(self ; fcmp D(src1), D(src2));
2374 }
2375 _ => codegen_error!("singlepass can't emit FCMP {:?} {:?} {:?}", sz, src1, src2),
2376 }
2377 Ok(())
2378 }
2379
2380 fn emit_fneg(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2381 match (sz, src, dst) {
2382 (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2383 dynasm!(self ; fneg S(dst), S(src));
2384 }
2385 (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2386 dynasm!(self ; fneg D(dst), D(src));
2387 }
2388 _ => codegen_error!("singlepass can't emit FNEG {:?} {:?} {:?}", sz, src, dst),
2389 }
2390 Ok(())
2391 }
2392 fn emit_fsqrt(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2393 match (sz, src, dst) {
2394 (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2395 dynasm!(self ; fsqrt S(dst), S(src));
2396 }
2397 (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2398 dynasm!(self ; fsqrt D(dst), D(src));
2399 }
2400 _ => codegen_error!("singlepass can't emit FSQRT {:?} {:?} {:?}", sz, src, dst),
2401 }
2402 Ok(())
2403 }
2404
2405 fn emit_fadd(
2406 &mut self,
2407 sz: Size,
2408 src1: Location,
2409 src2: Location,
2410 dst: Location,
2411 ) -> Result<(), CompileError> {
2412 match (sz, src1, src2, dst) {
2413 (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2414 dynasm!(self ; fadd S(dst), S(src1), S(src2));
2415 }
2416 (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2417 dynasm!(self ; fadd D(dst), D(src1), D(src2));
2418 }
2419 _ => codegen_error!(
2420 "singlepass can't emit FADD {:?} {:?} {:?} {:?}",
2421 sz,
2422 src1,
2423 src2,
2424 dst
2425 ),
2426 }
2427 Ok(())
2428 }
2429 fn emit_fsub(
2430 &mut self,
2431 sz: Size,
2432 src1: Location,
2433 src2: Location,
2434 dst: Location,
2435 ) -> Result<(), CompileError> {
2436 match (sz, src1, src2, dst) {
2437 (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2438 dynasm!(self ; fsub S(dst), S(src1), S(src2));
2439 }
2440 (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2441 dynasm!(self ; fsub D(dst), D(src1), D(src2));
2442 }
2443 _ => codegen_error!(
2444 "singlepass can't emit FSUB {:?} {:?} {:?} {:?}",
2445 sz,
2446 src1,
2447 src2,
2448 dst
2449 ),
2450 }
2451 Ok(())
2452 }
2453 fn emit_fmul(
2454 &mut self,
2455 sz: Size,
2456 src1: Location,
2457 src2: Location,
2458 dst: Location,
2459 ) -> Result<(), CompileError> {
2460 match (sz, src1, src2, dst) {
2461 (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2462 dynasm!(self ; fmul S(dst), S(src1), S(src2));
2463 }
2464 (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2465 dynasm!(self ; fmul D(dst), D(src1), D(src2));
2466 }
2467 _ => codegen_error!(
2468 "singlepass can't emit FMUL {:?} {:?} {:?} {:?}",
2469 sz,
2470 src1,
2471 src2,
2472 dst
2473 ),
2474 }
2475 Ok(())
2476 }
2477 fn emit_fdiv(
2478 &mut self,
2479 sz: Size,
2480 src1: Location,
2481 src2: Location,
2482 dst: Location,
2483 ) -> Result<(), CompileError> {
2484 match (sz, src1, src2, dst) {
2485 (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2486 dynasm!(self ; fdiv S(dst), S(src1), S(src2));
2487 }
2488 (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2489 dynasm!(self ; fdiv D(dst), D(src1), D(src2));
2490 }
2491 _ => codegen_error!(
2492 "singlepass can't emit FDIV {:?} {:?} {:?} {:?}",
2493 sz,
2494 src1,
2495 src2,
2496 dst
2497 ),
2498 }
2499 Ok(())
2500 }
2501
2502 fn emit_fmin(
2503 &mut self,
2504 sz: Size,
2505 src1: Location,
2506 src2: Location,
2507 dst: Location,
2508 ) -> Result<(), CompileError> {
2509 match (sz, src1, src2, dst) {
2510 (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2511 dynasm!(self ; fmin S(dst), S(src1), S(src2));
2512 }
2513 (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2514 dynasm!(self ; fmin D(dst), D(src1), D(src2));
2515 }
2516 _ => codegen_error!(
2517 "singlepass can't emit FMIN {:?} {:?} {:?} {:?}",
2518 sz,
2519 src1,
2520 src2,
2521 dst
2522 ),
2523 }
2524 Ok(())
2525 }
2526 fn emit_fmax(
2527 &mut self,
2528 sz: Size,
2529 src1: Location,
2530 src2: Location,
2531 dst: Location,
2532 ) -> Result<(), CompileError> {
2533 match (sz, src1, src2, dst) {
2534 (Size::S32, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2535 dynasm!(self ; fmax S(dst), S(src1), S(src2));
2536 }
2537 (Size::S64, Location::SIMD(src1), Location::SIMD(src2), Location::SIMD(dst)) => {
2538 dynasm!(self ; fmax D(dst), D(src1), D(src2));
2539 }
2540 _ => codegen_error!(
2541 "singlepass can't emit FMAX {:?} {:?} {:?} {:?}",
2542 sz,
2543 src1,
2544 src2,
2545 dst
2546 ),
2547 }
2548 Ok(())
2549 }
2550
2551 fn emit_frintz(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2552 match (sz, src, dst) {
2553 (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2554 dynasm!(self ; frintz S(dst), S(src));
2555 }
2556 (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2557 dynasm!(self ; frintz D(dst), D(src));
2558 }
2559 _ => codegen_error!("singlepass can't emit FRINTZ {:?} {:?} {:?}", sz, src, dst),
2560 }
2561 Ok(())
2562 }
2563 fn emit_frintn(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2564 match (sz, src, dst) {
2565 (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2566 dynasm!(self ; frintn S(dst), S(src));
2567 }
2568 (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2569 dynasm!(self ; frintn D(dst), D(src));
2570 }
2571 _ => codegen_error!("singlepass can't emit FRINTN {:?} {:?} {:?}", sz, src, dst),
2572 }
2573 Ok(())
2574 }
2575 fn emit_frintm(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2576 match (sz, src, dst) {
2577 (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2578 dynasm!(self ; frintm S(dst), S(src));
2579 }
2580 (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2581 dynasm!(self ; frintm D(dst), D(src));
2582 }
2583 _ => codegen_error!("singlepass can't emit FRINTM {:?} {:?} {:?}", sz, src, dst),
2584 }
2585 Ok(())
2586 }
2587 fn emit_frintp(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2588 match (sz, src, dst) {
2589 (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2590 dynasm!(self ; frintp S(dst), S(src));
2591 }
2592 (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2593 dynasm!(self ; frintp D(dst), D(src));
2594 }
2595 _ => codegen_error!("singlepass can't emit FRINTP {:?} {:?} {:?}", sz, src, dst),
2596 }
2597 Ok(())
2598 }
2599
2600 fn emit_scvtf(
2601 &mut self,
2602 sz_in: Size,
2603 src: Location,
2604 sz_out: Size,
2605 dst: Location,
2606 ) -> Result<(), CompileError> {
2607 match (sz_in, src, sz_out, dst) {
2608 (Size::S32, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
2609 dynasm!(self ; scvtf S(dst), W(src));
2610 }
2611 (Size::S64, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
2612 dynasm!(self ; scvtf S(dst), X(src));
2613 }
2614 (Size::S32, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
2615 dynasm!(self ; scvtf D(dst), W(src));
2616 }
2617 (Size::S64, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
2618 dynasm!(self ; scvtf D(dst), X(src));
2619 }
2620 _ => codegen_error!(
2621 "singlepass can't emit SCVTF {:?} {:?} {:?} {:?}",
2622 sz_in,
2623 src,
2624 sz_out,
2625 dst
2626 ),
2627 }
2628 Ok(())
2629 }
2630 fn emit_ucvtf(
2631 &mut self,
2632 sz_in: Size,
2633 src: Location,
2634 sz_out: Size,
2635 dst: Location,
2636 ) -> Result<(), CompileError> {
2637 match (sz_in, src, sz_out, dst) {
2638 (Size::S32, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
2639 dynasm!(self ; ucvtf S(dst), W(src));
2640 }
2641 (Size::S64, Location::GPR(src), Size::S32, Location::SIMD(dst)) => {
2642 dynasm!(self ; ucvtf S(dst), X(src));
2643 }
2644 (Size::S32, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
2645 dynasm!(self ; ucvtf D(dst), W(src));
2646 }
2647 (Size::S64, Location::GPR(src), Size::S64, Location::SIMD(dst)) => {
2648 dynasm!(self ; ucvtf D(dst), X(src));
2649 }
2650 _ => codegen_error!(
2651 "singlepass can't emit UCVTF {:?} {:?} {:?} {:?}",
2652 sz_in,
2653 src,
2654 sz_out,
2655 dst
2656 ),
2657 }
2658 Ok(())
2659 }
2660 fn emit_fcvt(&mut self, sz_in: Size, src: Location, dst: Location) -> Result<(), CompileError> {
2661 match (sz_in, src, dst) {
2662 (Size::S32, Location::SIMD(src), Location::SIMD(dst)) => {
2663 dynasm!(self ; fcvt D(dst), S(src));
2664 }
2665 (Size::S64, Location::SIMD(src), Location::SIMD(dst)) => {
2666 dynasm!(self ; fcvt S(dst), D(src));
2667 }
2668 _ => codegen_error!(
2669 "singlepass can't emit UCVTF {:?} {:?} {:?}",
2670 sz_in,
2671 src,
2672 dst
2673 ),
2674 }
2675 Ok(())
2676 }
2677 fn emit_fcvtzs(
2678 &mut self,
2679 sz_in: Size,
2680 src: Location,
2681 sz_out: Size,
2682 dst: Location,
2683 ) -> Result<(), CompileError> {
2684 match (sz_in, src, sz_out, dst) {
2685 (Size::S32, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
2686 dynasm!(self ; fcvtzs W(dst), S(src));
2687 }
2688 (Size::S64, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
2689 dynasm!(self ; fcvtzs W(dst), D(src));
2690 }
2691 (Size::S32, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
2692 dynasm!(self ; fcvtzs X(dst), S(src));
2693 }
2694 (Size::S64, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
2695 dynasm!(self ; fcvtzs X(dst), D(src));
2696 }
2697 _ => codegen_error!(
2698 "singlepass can't emit FCVTZS {:?} {:?} {:?} {:?}",
2699 sz_in,
2700 src,
2701 sz_out,
2702 dst
2703 ),
2704 }
2705 Ok(())
2706 }
2707 fn emit_fcvtzu(
2708 &mut self,
2709 sz_in: Size,
2710 src: Location,
2711 sz_out: Size,
2712 dst: Location,
2713 ) -> Result<(), CompileError> {
2714 match (sz_in, src, sz_out, dst) {
2715 (Size::S32, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
2716 dynasm!(self ; fcvtzu W(dst), S(src));
2717 }
2718 (Size::S64, Location::SIMD(src), Size::S32, Location::GPR(dst)) => {
2719 dynasm!(self ; fcvtzu W(dst), D(src));
2720 }
2721 (Size::S32, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
2722 dynasm!(self ; fcvtzu X(dst), S(src));
2723 }
2724 (Size::S64, Location::SIMD(src), Size::S64, Location::GPR(dst)) => {
2725 dynasm!(self ; fcvtzu X(dst), D(src));
2726 }
2727 _ => codegen_error!(
2728 "singlepass can't emit FCVTZU {:?} {:?} {:?} {:?}",
2729 sz_in,
2730 src,
2731 sz_out,
2732 dst
2733 ),
2734 }
2735 Ok(())
2736 }
2737
2738 fn emit_read_fpcr(&mut self, reg: GPR) -> Result<(), CompileError> {
2740 dynasm!(self ; mrs X(reg), 0b1_011_0100_0100_000);
2741 Ok(())
2742 }
2743 fn emit_write_fpcr(&mut self, reg: GPR) -> Result<(), CompileError> {
2744 dynasm!(self ; msr 0b1_011_0100_0100_000, X(reg));
2745 Ok(())
2746 }
2747 fn emit_read_fpsr(&mut self, reg: GPR) -> Result<(), CompileError> {
2749 dynasm!(self ; mrs X(reg), 0b1_011_0100_0100_001);
2750 Ok(())
2751 }
2752 fn emit_write_fpsr(&mut self, reg: GPR) -> Result<(), CompileError> {
2753 dynasm!(self ; msr 0b1_011_0100_0100_001, X(reg));
2754 Ok(())
2755 }
2756
2757 fn emit_cnt(&mut self, src: NEON, dst: NEON) -> Result<(), CompileError> {
2758 dynasm!(self ; cnt V(dst).B8, V(src).B8);
2759 Ok(())
2760 }
2761
2762 fn emit_addv(&mut self, src: NEON, dst: NEON) -> Result<(), CompileError> {
2763 dynasm!(self ; addv B(dst), V(src).B8);
2764 Ok(())
2765 }
2766
2767 #[allow(clippy::unit_arg)]
2768 fn emit_fmov(
2769 &mut self,
2770 src_size: Size,
2771 src: Location,
2772 dst_size: Size,
2773 dst: Location,
2774 ) -> Result<(), CompileError> {
2775 let mut err = false;
2776 match (src, dst) {
2777 (AbstractLocation::GPR(src), AbstractLocation::SIMD(dst)) => {
2778 match (src_size, dst_size) {
2779 (Size::S32, Size::S32) => dynasm!(self ; fmov S(dst), W(src)),
2780 (Size::S64, Size::S64) => dynasm!(self ; fmov D(dst), X(src)),
2781 _ => {
2782 err = true;
2783 }
2784 }
2785 }
2786 (AbstractLocation::SIMD(src), AbstractLocation::GPR(dst)) => {
2787 match (src_size, dst_size) {
2788 (Size::S32, Size::S32) => dynasm!(self ; fmov W(dst), S(src)),
2789 (Size::S64, Size::S64) => dynasm!(self ; fmov X(dst), D(src)),
2790 _ => {
2791 err = true;
2792 }
2793 }
2794 }
2795 _ => {
2799 err = true;
2800 }
2801 };
2802
2803 if err {
2804 codegen_error!(
2805 "singlepass can't generate fmov instruction for src_size: {:?}, src: {:?}, dst_size: {:?}, dst: {:?}",
2806 src_size,
2807 src,
2808 dst_size,
2809 dst,
2810 )
2811 }
2812 Ok(())
2813 }
2814}
2815
2816pub fn gen_std_trampoline_arm64(
2817 sig: &FunctionType,
2818 calling_convention: CallingConvention,
2819) -> Result<FunctionBody, CompileError> {
2820 let mut a = Assembler::new(0);
2821
2822 let fptr = GPR::X27;
2823 let args = GPR::X28;
2824
2825 dynasm!(a
2826 ; sub sp, sp, 32
2827 ; stp x29, x30, [sp]
2828 ; stp X(fptr), X(args), [sp, 16]
2829 ; mov x29, sp
2830 ; mov X(fptr), x1
2831 ; mov X(args), x2
2832 );
2833
2834 let stack_args = sig.params().len().saturating_sub(7); let stack_return_slots = sig
2836 .results()
2837 .len()
2838 .saturating_sub(ARM64_RETURN_VALUE_REGISTERS.len());
2839 let mut stack_offset = (stack_args + stack_return_slots) as u32 * 8;
2840 if stack_offset > 0 {
2841 if !stack_offset.is_multiple_of(16) {
2842 stack_offset += 8;
2843 assert!(stack_offset.is_multiple_of(16));
2844 }
2845 if stack_offset < 0x1000 {
2846 dynasm!(a ; sub sp, sp, stack_offset);
2847 } else {
2848 a.emit_mov_imm(Location::GPR(GPR::X26), stack_offset as u64)?;
2849 dynasm!(a ; sub sp, sp, x26);
2850 }
2851 }
2852
2853 let mut caller_stack_offset = stack_return_slots as u32 * 8;
2856 for (i, param) in sig.params().iter().enumerate() {
2857 let sz = match *param {
2858 Type::I32 | Type::F32 => Size::S32,
2859 Type::I64 | Type::F64 => Size::S64,
2860 Type::ExternRef => Size::S64,
2861 Type::FuncRef => Size::S64,
2862 _ => codegen_error!(
2863 "singlepass unsupported param type for trampoline {:?}",
2864 *param
2865 ),
2866 };
2867 match i {
2868 0..=6 => {
2869 a.emit_ldr(
2870 sz,
2871 Location::GPR(GPR::from_index(i + 1).unwrap()),
2872 Location::Memory(args, (i * 16) as i32),
2873 )?;
2874 }
2875 _ => {
2876 #[allow(clippy::single_match)]
2877 match calling_convention {
2878 CallingConvention::AppleAarch64 => {
2879 caller_stack_offset = caller_stack_offset.next_multiple_of(sz.bytes());
2881 }
2882 _ => (),
2883 };
2884 a.emit_ldr(
2886 sz,
2887 Location::GPR(GPR::X16),
2888 Location::Memory(args, (i * 16) as i32),
2889 )?;
2890 a.emit_str(
2891 sz,
2892 Location::GPR(GPR::X16),
2893 Location::Memory(GPR::XzrSp, caller_stack_offset as _),
2894 )?;
2895 match calling_convention {
2896 CallingConvention::AppleAarch64 => {
2897 caller_stack_offset += sz.bytes();
2898 }
2899 _ => {
2900 caller_stack_offset += 8;
2901 }
2902 }
2903 }
2904 }
2905 }
2906
2907 dynasm!(a ; blr X(fptr));
2908
2909 let mut n_stack_return_slots: usize = 0;
2911 for i in 0..sig.results().len() {
2912 let src = if let Some(®) = ARM64_RETURN_VALUE_REGISTERS.get(i) {
2913 reg
2914 } else {
2915 let loc = GPR::X16;
2916 a.emit_ldr(
2917 Size::S64,
2918 Location::GPR(GPR::X16),
2919 Location::Memory(GPR::XzrSp, (n_stack_return_slots as u32 * 8) as _),
2920 )?;
2921 n_stack_return_slots += 1;
2922 loc
2923 };
2924 a.emit_str(
2925 Size::S64,
2926 Location::GPR(src),
2927 Location::Memory(args, (i * 16) as _),
2928 )?;
2929 }
2930
2931 dynasm!(a
2933 ; ldp X(fptr), X(args), [x29, 16]
2934 ; ldp x29, x30, [x29]
2935 );
2936 let restored_stack_offset = 32 + stack_offset;
2937 if restored_stack_offset < 0x1000 {
2938 dynasm!(a; add sp, sp, restored_stack_offset);
2939 } else {
2940 a.emit_mov_imm(Location::GPR(GPR::X26), restored_stack_offset as u64)?;
2941 dynasm!(a; add sp, sp, x26);
2942 }
2943 dynasm!(a; ret);
2944
2945 let mut body = a.finalize().unwrap();
2946 body.shrink_to_fit();
2947 Ok(FunctionBody {
2948 body,
2949 unwind_info: None,
2950 })
2951}
2952pub fn gen_std_dynamic_import_trampoline_arm64(
2954 vmoffsets: &VMOffsets,
2955 sig: &FunctionType,
2956 calling_convention: CallingConvention,
2957) -> Result<FunctionBody, CompileError> {
2958 let mut a = Assembler::new(0);
2959 let stack_offset: usize = 16 * std::cmp::max(sig.params().len(), sig.results().len());
2961 a.emit_stpdb(
2963 Size::S64,
2964 Location::GPR(GPR::X30),
2965 Location::GPR(GPR::X26),
2966 GPR::XzrSp,
2967 16,
2968 )?;
2969
2970 if stack_offset != 0 {
2971 if stack_offset < 0x1000 {
2972 a.emit_sub(
2973 Size::S64,
2974 Location::GPR(GPR::XzrSp),
2975 Location::Imm32(stack_offset as _),
2976 Location::GPR(GPR::XzrSp),
2977 )?;
2978 } else {
2979 a.emit_mov_imm(Location::GPR(GPR::X26), stack_offset as u64)?;
2980 a.emit_sub(
2981 Size::S64,
2982 Location::GPR(GPR::XzrSp),
2983 Location::GPR(GPR::X26),
2984 Location::GPR(GPR::XzrSp),
2985 )?;
2986 }
2987 }
2988
2989 if !sig.params().is_empty() {
2991 let mut argalloc = ArgumentRegisterAllocator::default();
2992 argalloc.next(Type::I64, calling_convention).unwrap(); let mut stack_param_count: usize = 0;
2995
2996 for (i, ty) in sig.params().iter().enumerate() {
2997 let source_loc = match argalloc.next(*ty, calling_convention)? {
2998 Some(ARM64Register::GPR(gpr)) => Location::GPR(gpr),
2999 Some(ARM64Register::NEON(neon)) => Location::SIMD(neon),
3000 None => {
3001 let sz = match calling_convention {
3002 CallingConvention::AppleAarch64 => match *ty {
3003 Type::I32 | Type::F32 => Size::S32,
3004 _ => {
3005 stack_param_count = stack_param_count.next_multiple_of(8);
3006 Size::S64
3007 }
3008 },
3009 _ => Size::S64,
3010 };
3011 a.emit_ldr(
3012 sz,
3013 Location::GPR(GPR::X26),
3014 Location::Memory(GPR::XzrSp, (stack_offset + 16 + stack_param_count) as _),
3015 )?;
3016 stack_param_count += match sz {
3017 Size::S32 => 4,
3018 Size::S64 => 8,
3019 _ => codegen_error!(
3020 "singlepass unreachable in gen_std_dynamic_import_trampoline_arm64"
3021 ),
3022 };
3023 Location::GPR(GPR::X26)
3024 }
3025 };
3026 a.emit_str(
3027 Size::S64,
3028 source_loc,
3029 Location::Memory(GPR::XzrSp, (i * 16) as _),
3030 )?;
3031
3032 a.emit_str(
3034 Size::S64,
3035 Location::GPR(GPR::XzrSp), Location::Memory(GPR::XzrSp, (i * 16 + 8) as _), )?;
3038 }
3039 }
3040
3041 #[allow(clippy::match_single_binding)]
3042 match calling_convention {
3043 _ => {
3044 let offset = vmoffsets.vmdynamicfunction_import_context_address();
3046 a.emit_ldur(Size::S64, Location::GPR(GPR::X26), GPR::X0, offset as i32)?;
3047 a.emit_add(
3049 Size::S64,
3050 Location::GPR(GPR::XzrSp),
3051 Location::Imm8(0),
3052 Location::GPR(GPR::X1),
3053 )?;
3054 }
3055 };
3056
3057 a.emit_call_register(GPR::X26)?;
3059
3060 if !sig.results().is_empty() {
3062 assert_eq!(sig.results().len(), 1);
3063 a.emit_ldr(
3064 Size::S64,
3065 Location::GPR(GPR::X0),
3066 Location::Memory(GPR::XzrSp, 0),
3067 )?;
3068 }
3069
3070 if stack_offset != 0 {
3072 if stack_offset < 0x1000 {
3073 a.emit_add(
3074 Size::S64,
3075 Location::GPR(GPR::XzrSp),
3076 Location::Imm32(stack_offset as _),
3077 Location::GPR(GPR::XzrSp),
3078 )?;
3079 } else {
3080 a.emit_mov_imm(Location::GPR(GPR::X26), stack_offset as u64)?;
3081 a.emit_add(
3082 Size::S64,
3083 Location::GPR(GPR::XzrSp),
3084 Location::GPR(GPR::X26),
3085 Location::GPR(GPR::XzrSp),
3086 )?;
3087 }
3088 }
3089 a.emit_ldpia(
3090 Size::S64,
3091 Location::GPR(GPR::X30),
3092 Location::GPR(GPR::X26),
3093 GPR::XzrSp,
3094 16,
3095 )?;
3096
3097 a.emit_ret()?;
3099
3100 let mut body = a.finalize().unwrap();
3101 body.shrink_to_fit();
3102 Ok(FunctionBody {
3103 body,
3104 unwind_info: None,
3105 })
3106}
3107pub fn gen_import_call_trampoline_arm64(
3109 vmoffsets: &VMOffsets,
3110 index: FunctionIndex,
3111 sig: &FunctionType,
3112 calling_convention: CallingConvention,
3113) -> Result<CustomSection, CompileError> {
3114 let mut a = Assembler::new(0);
3115
3116 if sig
3121 .params()
3122 .iter()
3123 .any(|&x| x == Type::F32 || x == Type::F64)
3124 {
3125 #[allow(clippy::match_single_binding)]
3126 match calling_convention {
3127 _ => {
3128 let mut stack_offset: u32 = if sig.params().len() > 7 {
3130 7 * 8
3131 } else {
3132 (sig.params().len() as u32) * 8
3133 };
3134 stack_offset = stack_offset.next_multiple_of(16);
3135 if stack_offset != 0 {
3136 if stack_offset < 0x1000 {
3137 a.emit_sub(
3138 Size::S64,
3139 Location::GPR(GPR::XzrSp),
3140 Location::Imm32(stack_offset as u32),
3141 Location::GPR(GPR::XzrSp),
3142 )?;
3143 } else {
3144 a.emit_mov_imm(Location::GPR(GPR::X16), stack_offset as u64)?;
3145 a.emit_sub(
3146 Size::S64,
3147 Location::GPR(GPR::XzrSp),
3148 Location::GPR(GPR::X16),
3149 Location::GPR(GPR::XzrSp),
3150 )?;
3151 }
3152 }
3153
3154 static PARAM_REGS: &[GPR] = &[
3156 GPR::X1,
3157 GPR::X2,
3158 GPR::X3,
3159 GPR::X4,
3160 GPR::X5,
3161 GPR::X6,
3162 GPR::X7,
3163 ];
3164 let mut param_locations = vec![];
3165 for (i, param_reg) in PARAM_REGS.iter().enumerate().take(sig.params().len()) {
3166 let loc = match i {
3167 0..=6 => {
3168 let loc = Location::Memory(GPR::XzrSp, (i * 8) as i32);
3169 a.emit_str(Size::S64, Location::GPR(*param_reg), loc)?;
3170 loc
3171 }
3172 _ => {
3173 Location::Memory(GPR::XzrSp, (stack_offset + (i as u32 - 7) * 8) as i32)
3174 }
3175 };
3176 param_locations.push(loc);
3177 }
3178
3179 let mut caller_stack_offset: u32 = 0;
3181 let mut argalloc = ArgumentRegisterAllocator::default();
3182 argalloc.next(Type::I64, calling_convention).unwrap(); for (i, ty) in sig.params().iter().enumerate() {
3184 let prev_loc = param_locations[i];
3185 let targ = match argalloc.next(*ty, calling_convention)? {
3186 Some(ARM64Register::GPR(gpr)) => Location::GPR(gpr),
3187 Some(ARM64Register::NEON(neon)) => Location::SIMD(neon),
3188 None => {
3189 a.emit_ldr(Size::S64, Location::GPR(GPR::X16), prev_loc)?;
3191 a.emit_str(
3192 Size::S64,
3193 Location::GPR(GPR::X16),
3194 Location::Memory(
3195 GPR::XzrSp,
3196 (stack_offset + caller_stack_offset) as i32,
3197 ),
3198 )?;
3199 caller_stack_offset += 8;
3200 continue;
3201 }
3202 };
3203 a.emit_ldr(Size::S64, targ, prev_loc)?;
3204 }
3205
3206 if stack_offset > 0 {
3208 if stack_offset < 0x1000 {
3209 a.emit_add(
3210 Size::S64,
3211 Location::GPR(GPR::XzrSp),
3212 Location::Imm32(stack_offset as u32),
3213 Location::GPR(GPR::XzrSp),
3214 )?;
3215 } else {
3216 a.emit_mov_imm(Location::GPR(GPR::X16), stack_offset as u64)?;
3217 a.emit_add(
3218 Size::S64,
3219 Location::GPR(GPR::XzrSp),
3220 Location::GPR(GPR::X16),
3221 Location::GPR(GPR::XzrSp),
3222 )?;
3223 }
3224 }
3225 }
3226 }
3227 }
3228
3229 let offset = vmoffsets.vmctx_vmfunction_import(index);
3233 let offset =
3236 if (offset > 0) && ((offset < 0xF8) || (offset < 0x7FF8 && offset.is_multiple_of(8))) {
3237 offset
3238 } else {
3239 a.emit_mov_imm(Location::GPR(GPR::X16), (offset as i64) as u64)?;
3240 a.emit_add(
3241 Size::S64,
3242 Location::GPR(GPR::X0),
3243 Location::GPR(GPR::X16),
3244 Location::GPR(GPR::X0),
3245 )?;
3246 0
3247 };
3248 #[allow(clippy::match_single_binding)]
3249 match calling_convention {
3250 _ => {
3251 if offset.is_multiple_of(8) {
3252 a.emit_ldr(
3253 Size::S64,
3254 Location::GPR(GPR::X16),
3255 Location::Memory(GPR::X0, offset as i32), )?;
3257 a.emit_ldr(
3258 Size::S64,
3259 Location::GPR(GPR::X0),
3260 Location::Memory(GPR::X0, offset as i32 + 8), )?;
3262 } else {
3263 a.emit_ldur(
3264 Size::S64,
3265 Location::GPR(GPR::X16),
3266 GPR::X0,
3267 offset as i32, )?;
3269 a.emit_ldur(
3270 Size::S64,
3271 Location::GPR(GPR::X0),
3272 GPR::X0,
3273 offset as i32 + 8, )?;
3275 }
3276 }
3277 }
3278 a.emit_b_register(GPR::X16)?;
3279
3280 let mut contents = a.finalize().unwrap();
3281 contents.shrink_to_fit();
3282 let section_body = SectionBody::new_with_vec(contents);
3283
3284 Ok(CustomSection {
3285 protection: CustomSectionProtection::ReadExecute,
3286 alignment: None,
3287 bytes: section_body,
3288 relocations: vec![],
3289 })
3290}