1use std::collections::HashMap;
2
3use dynasmrt::{VecAssembler, aarch64::Aarch64Relocation};
4#[cfg(feature = "unwind")]
5use gimli::{AArch64, write::CallFrameInstruction};
6
7use wasmer_compiler::{
8 types::{
9 address_map::InstructionAddressMap,
10 function::FunctionBody,
11 relocation::{Relocation, RelocationKind, RelocationTarget},
12 section::CustomSection,
13 },
14 wasmparser::MemArg,
15};
16use wasmer_types::{
17 CompileError, FunctionIndex, FunctionType, SourceLoc, TrapCode, TrapInformation, VMOffsets,
18 target::{CallingConvention, CpuFeature, Target},
19};
20
21use crate::{
22 arm64_decl::{GPR, NEON},
23 codegen_error,
24 common_decl::*,
25 emitter_arm64::*,
26 location::{Location as AbstractLocation, Reg},
27 machine::*,
28 unwind::{UnwindInstructions, UnwindOps, UnwindRegister},
29};
30
31type Assembler = VecAssembler<Aarch64Relocation>;
32type Location = AbstractLocation<GPR, NEON>;
33
34pub struct MachineARM64 {
35 assembler: Assembler,
36 used_gprs: u32,
37 used_simd: u32,
38 trap_table: TrapTable,
39 instructions_address_map: Vec<InstructionAddressMap>,
42 src_loc: u32,
44 pushed: bool,
46 unwind_ops: Vec<(usize, UnwindOps<GPR, NEON>)>,
48 has_neon: bool,
50}
51
52pub(crate) const ARM64_RETURN_VALUE_REGISTERS: [GPR; 8] = [
55 GPR::X0,
56 GPR::X1,
57 GPR::X2,
58 GPR::X3,
59 GPR::X4,
60 GPR::X5,
61 GPR::X6,
62 GPR::X7,
63];
64
65#[allow(dead_code)]
66#[derive(PartialEq)]
67enum ImmType {
68 None,
69 NoneXzr,
70 Bits8,
71 Bits12,
72 Shift32,
73 Shift32No0,
74 Shift64,
75 Shift64No0,
76 Logical32,
77 Logical64,
78 UnscaledOffset,
79 OffsetByte,
80 OffsetHWord,
81 OffsetWord,
82 OffsetDWord,
83}
84
85#[allow(dead_code)]
86impl MachineARM64 {
87 pub fn new(target: Option<Target>) -> Self {
88 let has_neon = match target {
92 Some(ref target) => target.cpu_features().contains(CpuFeature::NEON),
93 None => false,
94 };
95
96 MachineARM64 {
97 assembler: Assembler::new(0),
98 used_gprs: 0,
99 used_simd: 0,
100 trap_table: TrapTable::default(),
101 instructions_address_map: vec![],
102 src_loc: 0,
103 pushed: false,
104 unwind_ops: vec![],
105 has_neon,
106 }
107 }
108 fn compatible_imm(&self, imm: i64, ty: ImmType) -> bool {
109 match ty {
110 ImmType::None => false,
111 ImmType::NoneXzr => false,
112 ImmType::Bits8 => (0..256).contains(&imm),
113 ImmType::Bits12 => (0..0x1000).contains(&imm),
114 ImmType::Shift32 => (0..32).contains(&imm),
115 ImmType::Shift32No0 => (1..32).contains(&imm),
116 ImmType::Shift64 => (0..64).contains(&imm),
117 ImmType::Shift64No0 => (1..64).contains(&imm),
118 ImmType::Logical32 => encode_logical_immediate_32bit(imm as u32).is_some(),
119 ImmType::Logical64 => encode_logical_immediate_64bit(imm as u64).is_some(),
120 ImmType::UnscaledOffset => (imm > -256) && (imm < 256),
121 ImmType::OffsetByte => (0..0x1000).contains(&imm),
122 ImmType::OffsetHWord => (imm & 1 == 0) && (0..0x2000).contains(&imm),
123 ImmType::OffsetWord => (imm & 3 == 0) && (0..0x4000).contains(&imm),
124 ImmType::OffsetDWord => (imm & 7 == 0) && (0..0x8000).contains(&imm),
125 }
126 }
127
128 fn location_to_reg(
129 &mut self,
130 sz: Size,
131 src: Location,
132 temps: &mut Vec<GPR>,
133 allow_imm: ImmType,
134 read_val: bool,
135 wanted: Option<GPR>,
136 ) -> Result<Location, CompileError> {
137 match src {
138 Location::GPR(_) | Location::SIMD(_) => Ok(src),
139 Location::Imm8(val) => {
140 if allow_imm == ImmType::NoneXzr && val == 0 {
141 Ok(Location::GPR(GPR::XzrSp))
142 } else if self.compatible_imm(val as i64, allow_imm) {
143 Ok(src)
144 } else {
145 let tmp = if let Some(wanted) = wanted {
146 wanted
147 } else {
148 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
149 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
150 })?;
151 temps.push(tmp);
152 tmp
153 };
154 self.assembler
155 .emit_mov_imm(Location::GPR(tmp), val as u64)?;
156 Ok(Location::GPR(tmp))
157 }
158 }
159 Location::Imm32(val) => {
160 if allow_imm == ImmType::NoneXzr && val == 0 {
161 Ok(Location::GPR(GPR::XzrSp))
162 } else if self.compatible_imm(val as i64, allow_imm) {
163 Ok(src)
164 } else {
165 let tmp = if let Some(wanted) = wanted {
166 wanted
167 } else {
168 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
169 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
170 })?;
171 temps.push(tmp);
172 tmp
173 };
174 self.assembler
175 .emit_mov_imm(Location::GPR(tmp), (val as i64) as u64)?;
176 Ok(Location::GPR(tmp))
177 }
178 }
179 Location::Imm64(val) => {
180 if allow_imm == ImmType::NoneXzr && val == 0 {
181 Ok(Location::GPR(GPR::XzrSp))
182 } else if self.compatible_imm(val as i64, allow_imm) {
183 Ok(src)
184 } else {
185 let tmp = if let Some(wanted) = wanted {
186 wanted
187 } else {
188 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
189 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
190 })?;
191 temps.push(tmp);
192 tmp
193 };
194 self.assembler
195 .emit_mov_imm(Location::GPR(tmp), val as u64)?;
196 Ok(Location::GPR(tmp))
197 }
198 }
199 Location::Memory(reg, val) => {
200 let tmp = if let Some(wanted) = wanted {
201 wanted
202 } else {
203 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
204 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
205 })?;
206 temps.push(tmp);
207 tmp
208 };
209 if read_val {
210 let offsize = match sz {
211 Size::S8 => ImmType::OffsetByte,
212 Size::S16 => ImmType::OffsetHWord,
213 Size::S32 => ImmType::OffsetWord,
214 Size::S64 => ImmType::OffsetDWord,
215 };
216 if sz == Size::S8 {
217 if self.compatible_imm(val as i64, offsize) {
218 self.assembler.emit_ldrb(
219 sz,
220 Location::GPR(tmp),
221 Location::Memory(reg, val as _),
222 )?;
223 } else {
224 if reg == tmp {
225 codegen_error!("singlepass reg==tmp unreachable");
226 }
227 self.assembler
228 .emit_mov_imm(Location::GPR(tmp), (val as i64) as u64)?;
229 self.assembler.emit_ldrb(
230 sz,
231 Location::GPR(tmp),
232 Location::Memory2(reg, tmp, Multiplier::One, 0),
233 )?;
234 }
235 } else if sz == Size::S16 {
236 if self.compatible_imm(val as i64, offsize) {
237 self.assembler.emit_ldrh(
238 sz,
239 Location::GPR(tmp),
240 Location::Memory(reg, val as _),
241 )?;
242 } else {
243 if reg == tmp {
244 codegen_error!("singlepass reg==tmp unreachable");
245 }
246 self.assembler
247 .emit_mov_imm(Location::GPR(tmp), (val as i64) as u64)?;
248 self.assembler.emit_ldrh(
249 sz,
250 Location::GPR(tmp),
251 Location::Memory2(reg, tmp, Multiplier::One, 0),
252 )?;
253 }
254 } else if self.compatible_imm(val as i64, offsize) {
255 self.assembler.emit_ldr(
256 sz,
257 Location::GPR(tmp),
258 Location::Memory(reg, val as _),
259 )?;
260 } else if self.compatible_imm(val as i64, ImmType::UnscaledOffset) {
261 self.assembler.emit_ldur(sz, Location::GPR(tmp), reg, val)?;
262 } else {
263 if reg == tmp {
264 codegen_error!("singlepass reg == tmp unreachable");
265 }
266 self.assembler
267 .emit_mov_imm(Location::GPR(tmp), (val as i64) as u64)?;
268 self.assembler.emit_ldr(
269 sz,
270 Location::GPR(tmp),
271 Location::Memory2(reg, tmp, Multiplier::One, 0),
272 )?;
273 }
274 }
275 Ok(Location::GPR(tmp))
276 }
277 _ => codegen_error!("singlepass can't emit location_to_reg {:?} {:?}", sz, src),
278 }
279 }
280 fn location_to_neon(
281 &mut self,
282 sz: Size,
283 src: Location,
284 temps: &mut Vec<NEON>,
285 allow_imm: ImmType,
286 read_val: bool,
287 ) -> Result<Location, CompileError> {
288 match src {
289 Location::SIMD(_) => Ok(src),
290 Location::GPR(_) => {
291 let tmp = self.acquire_temp_simd().ok_or_else(|| {
292 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
293 })?;
294 temps.push(tmp);
295 if read_val {
296 self.assembler.emit_mov(sz, src, Location::SIMD(tmp))?;
297 }
298 Ok(Location::SIMD(tmp))
299 }
300 Location::Imm8(val) => {
301 if self.compatible_imm(val as i64, allow_imm) {
302 Ok(src)
303 } else {
304 let gpr = self.acquire_temp_gpr().ok_or_else(|| {
305 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
306 })?;
307 let tmp = self.acquire_temp_simd().ok_or_else(|| {
308 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
309 })?;
310 temps.push(tmp);
311 self.assembler
312 .emit_mov_imm(Location::GPR(gpr), val as u64)?;
313 self.assembler
314 .emit_mov(sz, Location::GPR(gpr), Location::SIMD(tmp))?;
315 self.release_gpr(gpr);
316 Ok(Location::SIMD(tmp))
317 }
318 }
319 Location::Imm32(val) => {
320 if self.compatible_imm(val as i64, allow_imm) {
321 Ok(src)
322 } else {
323 let gpr = self.acquire_temp_gpr().ok_or_else(|| {
324 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
325 })?;
326 let tmp = self.acquire_temp_simd().ok_or_else(|| {
327 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
328 })?;
329 temps.push(tmp);
330 self.assembler
331 .emit_mov_imm(Location::GPR(gpr), (val as i64) as u64)?;
332 self.assembler
333 .emit_mov(sz, Location::GPR(gpr), Location::SIMD(tmp))?;
334 self.release_gpr(gpr);
335 Ok(Location::SIMD(tmp))
336 }
337 }
338 Location::Imm64(val) => {
339 if self.compatible_imm(val as i64, allow_imm) {
340 Ok(src)
341 } else {
342 let gpr = self.acquire_temp_gpr().ok_or_else(|| {
343 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
344 })?;
345 let tmp = self.acquire_temp_simd().ok_or_else(|| {
346 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
347 })?;
348 temps.push(tmp);
349 self.assembler
350 .emit_mov_imm(Location::GPR(gpr), val as u64)?;
351 self.assembler
352 .emit_mov(sz, Location::GPR(gpr), Location::SIMD(tmp))?;
353 self.release_gpr(gpr);
354 Ok(Location::SIMD(tmp))
355 }
356 }
357 Location::Memory(reg, val) => {
358 let tmp = self.acquire_temp_simd().ok_or_else(|| {
359 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
360 })?;
361 temps.push(tmp);
362 if read_val {
363 let offsize = if sz == Size::S32 {
364 ImmType::OffsetWord
365 } else {
366 ImmType::OffsetDWord
367 };
368 if self.compatible_imm(val as i64, offsize) {
369 self.assembler.emit_ldr(
370 sz,
371 Location::SIMD(tmp),
372 Location::Memory(reg, val as _),
373 )?;
374 } else if self.compatible_imm(val as i64, ImmType::UnscaledOffset) {
375 self.assembler
376 .emit_ldur(sz, Location::SIMD(tmp), reg, val)?;
377 } else {
378 let gpr = self.acquire_temp_gpr().ok_or_else(|| {
379 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
380 })?;
381 self.assembler
382 .emit_mov_imm(Location::GPR(gpr), (val as i64) as u64)?;
383 self.assembler.emit_ldr(
384 sz,
385 Location::SIMD(tmp),
386 Location::Memory2(reg, gpr, Multiplier::One, 0),
387 )?;
388 self.release_gpr(gpr);
389 }
390 }
391 Ok(Location::SIMD(tmp))
392 }
393 _ => codegen_error!("singlepass can't emit location_to_neon {:?} {:?}", sz, src),
394 }
395 }
396
397 fn emit_relaxed_binop(
398 &mut self,
399 op: fn(&mut Assembler, Size, Location, Location) -> Result<(), CompileError>,
400 sz: Size,
401 src: Location,
402 dst: Location,
403 putback: bool,
404 ) -> Result<(), CompileError> {
405 let mut temps = vec![];
406 let src_imm = if putback {
407 ImmType::None
408 } else {
409 ImmType::Bits12
410 };
411 let src = self.location_to_reg(sz, src, &mut temps, src_imm, true, None)?;
412 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, !putback, None)?;
413 op(&mut self.assembler, sz, src, dest)?;
414 if dst != dest && putback {
415 self.move_location(sz, dest, dst)?;
416 }
417 for r in temps {
418 self.release_gpr(r);
419 }
420 Ok(())
421 }
422 fn emit_relaxed_binop_neon(
423 &mut self,
424 op: fn(&mut Assembler, Size, Location, Location) -> Result<(), CompileError>,
425 sz: Size,
426 src: Location,
427 dst: Location,
428 putback: bool,
429 ) -> Result<(), CompileError> {
430 let mut temps = vec![];
431 let src = self.location_to_neon(sz, src, &mut temps, ImmType::None, true)?;
432 let dest = self.location_to_neon(sz, dst, &mut temps, ImmType::None, !putback)?;
433 op(&mut self.assembler, sz, src, dest)?;
434 if dst != dest && putback {
435 self.move_location(sz, dest, dst)?;
436 }
437 for r in temps {
438 self.release_simd(r);
439 }
440 Ok(())
441 }
442 fn emit_relaxed_binop3(
443 &mut self,
444 op: fn(&mut Assembler, Size, Location, Location, Location) -> Result<(), CompileError>,
445 sz: Size,
446 src1: Location,
447 src2: Location,
448 dst: Location,
449 allow_imm: ImmType,
450 ) -> Result<(), CompileError> {
451 let mut temps = vec![];
452 let src1 = self.location_to_reg(sz, src1, &mut temps, ImmType::None, true, None)?;
453 let src2 = self.location_to_reg(sz, src2, &mut temps, allow_imm, true, None)?;
454 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
455 op(&mut self.assembler, sz, src1, src2, dest)?;
456 if dst != dest {
457 self.move_location(sz, dest, dst)?;
458 }
459 for r in temps {
460 self.release_gpr(r);
461 }
462 Ok(())
463 }
464 fn emit_relaxed_binop3_neon(
465 &mut self,
466 op: fn(&mut Assembler, Size, Location, Location, Location) -> Result<(), CompileError>,
467 sz: Size,
468 src1: Location,
469 src2: Location,
470 dst: Location,
471 allow_imm: ImmType,
472 ) -> Result<(), CompileError> {
473 let mut temps = vec![];
474 let src1 = self.location_to_neon(sz, src1, &mut temps, ImmType::None, true)?;
475 let src2 = self.location_to_neon(sz, src2, &mut temps, allow_imm, true)?;
476 let dest = self.location_to_neon(sz, dst, &mut temps, ImmType::None, false)?;
477 op(&mut self.assembler, sz, src1, src2, dest)?;
478 if dst != dest {
479 self.move_location(sz, dest, dst)?;
480 }
481 for r in temps {
482 self.release_simd(r);
483 }
484 Ok(())
485 }
486 fn emit_relaxed_ldr64(
487 &mut self,
488 sz: Size,
489 dst: Location,
490 src: Location,
491 ) -> Result<(), CompileError> {
492 let mut temps = vec![];
493 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
494 match src {
495 Location::Memory(addr, offset) => {
496 if self.compatible_imm(offset as i64, ImmType::OffsetDWord) {
497 self.assembler.emit_ldr(Size::S64, dest, src)?;
498 } else if self.compatible_imm(offset as i64, ImmType::UnscaledOffset) {
499 self.assembler.emit_ldur(Size::S64, dest, addr, offset)?;
500 } else {
501 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
502 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
503 })?;
504 self.assembler
505 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
506 self.assembler.emit_ldr(
507 Size::S64,
508 dest,
509 Location::Memory2(addr, tmp, Multiplier::One, 0),
510 )?;
511 temps.push(tmp);
512 }
513 }
514 _ => codegen_error!("singplass emit_relaxed_ldr64 unreachable"),
515 }
516 if dst != dest {
517 self.move_location(sz, dest, dst)?;
518 }
519 for r in temps {
520 self.release_gpr(r);
521 }
522 Ok(())
523 }
524 fn emit_relaxed_ldr32(
525 &mut self,
526 sz: Size,
527 dst: Location,
528 src: Location,
529 ) -> Result<(), CompileError> {
530 let mut temps = vec![];
531 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
532 match src {
533 Location::Memory(addr, offset) => {
534 if self.compatible_imm(offset as i64, ImmType::OffsetWord) {
535 self.assembler.emit_ldr(Size::S32, dest, src)?;
536 } else if self.compatible_imm(offset as i64, ImmType::UnscaledOffset) {
537 self.assembler.emit_ldur(Size::S32, dest, addr, offset)?;
538 } else {
539 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
540 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
541 })?;
542 self.assembler
543 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
544 self.assembler.emit_ldr(
545 Size::S32,
546 dest,
547 Location::Memory2(addr, tmp, Multiplier::One, 0),
548 )?;
549 temps.push(tmp);
550 }
551 }
552 _ => codegen_error!("singlepass emit_relaxed_ldr32 unreachable"),
553 }
554 if dst != dest {
555 self.move_location(sz, dest, dst)?;
556 }
557 for r in temps {
558 self.release_gpr(r);
559 }
560 Ok(())
561 }
562 fn emit_relaxed_ldr32s(
563 &mut self,
564 sz: Size,
565 dst: Location,
566 src: Location,
567 ) -> Result<(), CompileError> {
568 let mut temps = vec![];
569 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
570 match src {
571 Location::Memory(addr, offset) => {
572 if self.compatible_imm(offset as i64, ImmType::OffsetWord) {
573 self.assembler.emit_ldrsw(Size::S64, dest, src)?;
574 } else {
575 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
576 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
577 })?;
578 self.assembler
579 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
580 self.assembler.emit_ldrsw(
581 Size::S64,
582 dest,
583 Location::Memory2(addr, tmp, Multiplier::One, 0),
584 )?;
585 temps.push(tmp);
586 }
587 }
588 _ => codegen_error!("singplepass emit_relaxed_ldr32s unreachable"),
589 }
590 if dst != dest {
591 self.move_location(sz, dest, dst)?;
592 }
593 for r in temps {
594 self.release_gpr(r);
595 }
596 Ok(())
597 }
598 fn emit_relaxed_ldr16(
599 &mut self,
600 sz: Size,
601 dst: Location,
602 src: Location,
603 ) -> Result<(), CompileError> {
604 let mut temps = vec![];
605 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
606 match src {
607 Location::Memory(addr, offset) => {
608 if self.compatible_imm(offset as i64, ImmType::OffsetHWord) {
609 self.assembler.emit_ldrh(Size::S32, dest, src)?;
610 } else {
611 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
612 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
613 })?;
614 self.assembler
615 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
616 self.assembler.emit_ldrh(
617 Size::S32,
618 dest,
619 Location::Memory2(addr, tmp, Multiplier::One, 0),
620 )?;
621 temps.push(tmp);
622 }
623 }
624 _ => codegen_error!("singlpass emit_relaxed_ldr16 unreachable"),
625 }
626 if dst != dest {
627 self.move_location(sz, dest, dst)?;
628 }
629 for r in temps {
630 self.release_gpr(r);
631 }
632 Ok(())
633 }
634 fn emit_relaxed_ldr16s(
635 &mut self,
636 sz: Size,
637 dst: Location,
638 src: Location,
639 ) -> Result<(), CompileError> {
640 let mut temps = vec![];
641 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
642 match src {
643 Location::Memory(addr, offset) => {
644 if self.compatible_imm(offset as i64, ImmType::OffsetHWord) {
645 self.assembler.emit_ldrsh(sz, dest, src)?;
646 } else {
647 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
648 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
649 })?;
650 self.assembler
651 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
652 self.assembler.emit_ldrsh(
653 sz,
654 dest,
655 Location::Memory2(addr, tmp, Multiplier::One, 0),
656 )?;
657 temps.push(tmp);
658 }
659 }
660 _ => codegen_error!("singlepass emit_relaxed_ldr16s unreachable"),
661 }
662 if dst != dest {
663 self.move_location(sz, dest, dst)?;
664 }
665 for r in temps {
666 self.release_gpr(r);
667 }
668 Ok(())
669 }
670 fn emit_relaxed_ldr8(
671 &mut self,
672 sz: Size,
673 dst: Location,
674 src: Location,
675 ) -> Result<(), CompileError> {
676 let mut temps = vec![];
677 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
678 match src {
679 Location::Memory(addr, offset) => {
680 if self.compatible_imm(offset as i64, ImmType::OffsetByte) {
681 self.assembler.emit_ldrb(Size::S32, dest, src)?;
682 } else {
683 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
684 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
685 })?;
686 self.assembler
687 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
688 self.assembler.emit_ldrb(
689 Size::S32,
690 dest,
691 Location::Memory2(addr, tmp, Multiplier::One, 0),
692 )?;
693 temps.push(tmp);
694 }
695 }
696 _ => codegen_error!("singplepass emit_relaxed_ldr8 unreachable"),
697 }
698 if dst != dest {
699 self.move_location(sz, dest, dst)?;
700 }
701 for r in temps {
702 self.release_gpr(r);
703 }
704 Ok(())
705 }
706 fn emit_relaxed_ldr8s(
707 &mut self,
708 sz: Size,
709 dst: Location,
710 src: Location,
711 ) -> Result<(), CompileError> {
712 let mut temps = vec![];
713 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
714 match src {
715 Location::Memory(addr, offset) => {
716 if self.compatible_imm(offset as i64, ImmType::OffsetByte) {
717 self.assembler.emit_ldrsb(sz, dest, src)?;
718 } else {
719 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
720 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
721 })?;
722 self.assembler
723 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
724 self.assembler.emit_ldrsb(
725 sz,
726 dest,
727 Location::Memory2(addr, tmp, Multiplier::One, 0),
728 )?;
729 temps.push(tmp);
730 }
731 }
732 _ => codegen_error!("singlepass emit_relaxed_ldr8s unreachable"),
733 }
734 if dst != dest {
735 self.move_location(sz, dest, dst)?;
736 }
737 for r in temps {
738 self.release_gpr(r);
739 }
740 Ok(())
741 }
742 fn emit_relaxed_str64(&mut self, dst: Location, src: Location) -> Result<(), CompileError> {
743 let mut temps = vec![];
744 let dst = self.location_to_reg(Size::S64, dst, &mut temps, ImmType::NoneXzr, true, None)?;
745 match src {
746 Location::Memory(addr, offset) => {
747 if self.compatible_imm(offset as i64, ImmType::OffsetDWord) {
748 self.assembler.emit_str(Size::S64, dst, src)?;
749 } else if self.compatible_imm(offset as i64, ImmType::UnscaledOffset) {
750 self.assembler.emit_stur(Size::S64, dst, addr, offset)?;
751 } else {
752 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
753 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
754 })?;
755 self.assembler
756 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
757 self.assembler.emit_str(
758 Size::S64,
759 dst,
760 Location::Memory2(addr, tmp, Multiplier::One, 0),
761 )?;
762 temps.push(tmp);
763 }
764 }
765 _ => codegen_error!("singlepass can't emit str64 {:?} {:?}", dst, src),
766 }
767 for r in temps {
768 self.release_gpr(r);
769 }
770 Ok(())
771 }
772 fn emit_relaxed_str32(&mut self, dst: Location, src: Location) -> Result<(), CompileError> {
773 let mut temps = vec![];
774 let dst = self.location_to_reg(Size::S64, dst, &mut temps, ImmType::NoneXzr, true, None)?;
775 match src {
776 Location::Memory(addr, offset) => {
777 if self.compatible_imm(offset as i64, ImmType::OffsetWord) {
778 self.assembler.emit_str(Size::S32, dst, src)?;
779 } else if self.compatible_imm(offset as i64, ImmType::UnscaledOffset) {
780 self.assembler.emit_stur(Size::S32, dst, addr, offset)?;
781 } else {
782 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
783 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
784 })?;
785 self.assembler
786 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
787 self.assembler.emit_str(
788 Size::S32,
789 dst,
790 Location::Memory2(addr, tmp, Multiplier::One, 0),
791 )?;
792 temps.push(tmp);
793 }
794 }
795 _ => codegen_error!("singplepass emit_relaxed_str32 unreachable"),
796 }
797 for r in temps {
798 self.release_gpr(r);
799 }
800 Ok(())
801 }
802 fn emit_relaxed_str16(&mut self, dst: Location, src: Location) -> Result<(), CompileError> {
803 let mut temps = vec![];
804 let dst = self.location_to_reg(Size::S64, dst, &mut temps, ImmType::NoneXzr, true, None)?;
805 match src {
806 Location::Memory(addr, offset) => {
807 if self.compatible_imm(offset as i64, ImmType::OffsetHWord) {
808 self.assembler.emit_strh(Size::S32, dst, src)?;
809 } else {
810 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
811 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
812 })?;
813 self.assembler
814 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
815 self.assembler.emit_strh(
816 Size::S32,
817 dst,
818 Location::Memory2(addr, tmp, Multiplier::One, 0),
819 )?;
820 temps.push(tmp);
821 }
822 }
823 _ => codegen_error!("singlepass emit_relaxed_str16 unreachable"),
824 }
825 for r in temps {
826 self.release_gpr(r);
827 }
828 Ok(())
829 }
830 fn emit_relaxed_str8(&mut self, dst: Location, src: Location) -> Result<(), CompileError> {
831 let mut temps = vec![];
832 let dst = self.location_to_reg(Size::S64, dst, &mut temps, ImmType::NoneXzr, true, None)?;
833 match src {
834 Location::Memory(addr, offset) => {
835 if self.compatible_imm(offset as i64, ImmType::OffsetByte) {
836 self.assembler
837 .emit_strb(Size::S32, dst, Location::Memory(addr, offset))?;
838 } else {
839 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
840 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
841 })?;
842 self.assembler
843 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
844 self.assembler.emit_strb(
845 Size::S32,
846 dst,
847 Location::Memory2(addr, tmp, Multiplier::One, 0),
848 )?;
849 temps.push(tmp);
850 }
851 }
852 _ => codegen_error!("singlepass emit_relaxed_str8 unreachable"),
853 }
854 for r in temps {
855 self.release_gpr(r);
856 }
857 Ok(())
858 }
859 fn emit_cmpop_i64_dynamic_b(
861 &mut self,
862 c: Condition,
863 loc_a: Location,
864 loc_b: Location,
865 ret: Location,
866 ) -> Result<(), CompileError> {
867 match ret {
868 Location::GPR(_) => {
869 self.emit_relaxed_cmp(Size::S64, loc_b, loc_a)?;
870 self.assembler.emit_cset(Size::S32, ret, c)?;
871 }
872 Location::Memory(_, _) => {
873 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
874 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
875 })?;
876 self.emit_relaxed_cmp(Size::S64, loc_b, loc_a)?;
877 self.assembler.emit_cset(Size::S32, Location::GPR(tmp), c)?;
878 self.move_location(Size::S32, Location::GPR(tmp), ret)?;
879 self.release_gpr(tmp);
880 }
881 _ => {
882 codegen_error!("singlepass emit_compop_i64_dynamic_b unreachable");
883 }
884 }
885 Ok(())
886 }
887 fn emit_cmpop_i32_dynamic_b(
889 &mut self,
890 c: Condition,
891 loc_a: Location,
892 loc_b: Location,
893 ret: Location,
894 ) -> Result<(), CompileError> {
895 match ret {
896 Location::GPR(_) => {
897 self.emit_relaxed_cmp(Size::S32, loc_b, loc_a)?;
898 self.assembler.emit_cset(Size::S32, ret, c)?;
899 }
900 Location::Memory(_, _) => {
901 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
902 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
903 })?;
904 self.emit_relaxed_cmp(Size::S32, loc_b, loc_a)?;
905 self.assembler.emit_cset(Size::S32, Location::GPR(tmp), c)?;
906 self.move_location(Size::S32, Location::GPR(tmp), ret)?;
907 self.release_gpr(tmp);
908 }
909 _ => {
910 codegen_error!("singlepass emit_cmpop_i32_dynamic_b unreachable");
911 }
912 }
913 Ok(())
914 }
915
916 #[allow(clippy::too_many_arguments)]
917 fn memory_op<F: FnOnce(&mut Self, GPR) -> Result<(), CompileError>>(
918 &mut self,
919 addr: Location,
920 memarg: &MemArg,
921 check_alignment: bool,
922 value_size: usize,
923 need_check: bool,
924 imported_memories: bool,
925 offset: i32,
926 heap_access_oob: Label,
927 unaligned_atomic: Label,
928 cb: F,
929 ) -> Result<(), CompileError> {
930 let tmp_addr = self.acquire_temp_gpr().ok_or_else(|| {
931 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
932 })?;
933
934 let (base_loc, bound_loc) = if imported_memories {
936 self.emit_relaxed_binop(
938 Assembler::emit_mov,
939 Size::S64,
940 Location::Memory(self.get_vmctx_reg(), offset),
941 Location::GPR(tmp_addr),
942 true,
943 )?;
944 (Location::Memory(tmp_addr, 0), Location::Memory(tmp_addr, 8))
945 } else {
946 (
947 Location::Memory(self.get_vmctx_reg(), offset),
948 Location::Memory(self.get_vmctx_reg(), offset + 8),
949 )
950 };
951
952 let tmp_base = self.acquire_temp_gpr().ok_or_else(|| {
953 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
954 })?;
955 let tmp_bound = self.acquire_temp_gpr().ok_or_else(|| {
956 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
957 })?;
958
959 self.emit_relaxed_ldr64(Size::S64, Location::GPR(tmp_base), base_loc)?;
961
962 if need_check {
964 self.emit_relaxed_ldr64(Size::S64, Location::GPR(tmp_bound), bound_loc)?;
965
966 self.assembler.emit_add(
970 Size::S64,
971 Location::GPR(tmp_bound),
972 Location::GPR(tmp_base),
973 Location::GPR(tmp_bound),
974 )?;
975 if self.compatible_imm(value_size as _, ImmType::Bits12) {
976 self.assembler.emit_sub(
977 Size::S64,
978 Location::GPR(tmp_bound),
979 Location::Imm32(value_size as _),
980 Location::GPR(tmp_bound),
981 )?;
982 } else {
983 let tmp2 = self.acquire_temp_gpr().ok_or_else(|| {
984 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
985 })?;
986 self.assembler
987 .emit_mov_imm(Location::GPR(tmp2), value_size as u64)?;
988 self.assembler.emit_sub(
989 Size::S64,
990 Location::GPR(tmp_bound),
991 Location::GPR(tmp2),
992 Location::GPR(tmp_bound),
993 )?;
994 self.release_gpr(tmp2);
995 }
996 }
997
998 self.move_location(Size::S32, addr, Location::GPR(tmp_addr))?;
1002
1003 if memarg.offset != 0 {
1005 if self.compatible_imm(memarg.offset as _, ImmType::Bits12) {
1006 self.assembler.emit_adds(
1007 Size::S32,
1008 Location::Imm32(memarg.offset as u32),
1009 Location::GPR(tmp_addr),
1010 Location::GPR(tmp_addr),
1011 )?;
1012 } else {
1013 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
1014 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1015 })?;
1016 self.assembler
1017 .emit_mov_imm(Location::GPR(tmp), memarg.offset as _)?;
1018 self.assembler.emit_adds(
1019 Size::S32,
1020 Location::GPR(tmp_addr),
1021 Location::GPR(tmp),
1022 Location::GPR(tmp_addr),
1023 )?;
1024 self.release_gpr(tmp);
1025 }
1026
1027 self.assembler
1029 .emit_bcond_label_far(Condition::Cs, heap_access_oob)?;
1030 }
1031
1032 self.assembler.emit_add(
1034 Size::S64,
1035 Location::GPR(tmp_base),
1036 Location::GPR(tmp_addr),
1037 Location::GPR(tmp_addr),
1038 )?;
1039
1040 if need_check {
1041 self.assembler.emit_cmp(
1043 Size::S64,
1044 Location::GPR(tmp_bound),
1045 Location::GPR(tmp_addr),
1046 )?;
1047
1048 self.assembler
1050 .emit_bcond_label_far(Condition::Hi, heap_access_oob)?;
1051 }
1052
1053 self.release_gpr(tmp_bound);
1054 self.release_gpr(tmp_base);
1055
1056 let align = value_size as u32;
1057 if check_alignment && align != 1 {
1058 self.assembler.emit_tst(
1059 Size::S64,
1060 Location::Imm32(align - 1),
1061 Location::GPR(tmp_addr),
1062 )?;
1063 self.assembler
1064 .emit_bcond_label_far(Condition::Ne, unaligned_atomic)?;
1065 }
1066 let begin = self.assembler.get_offset().0;
1067 cb(self, tmp_addr)?;
1068 let end = self.assembler.get_offset().0;
1069 self.mark_address_range_with_trap_code(TrapCode::HeapAccessOutOfBounds, begin, end);
1070
1071 self.release_gpr(tmp_addr);
1072 Ok(())
1073 }
1074
1075 fn offset_is_ok(&self, size: Size, offset: i32) -> bool {
1076 if offset < 0 {
1077 return false;
1078 }
1079 let shift = size.bytes().trailing_zeros() as i32;
1080 if offset >= 0x1000 << shift {
1081 return false;
1082 }
1083 if (offset & ((1 << shift) - 1)) != 0 {
1084 return false;
1085 }
1086 true
1087 }
1088
1089 fn emit_push(&mut self, sz: Size, src: Location) -> Result<(), CompileError> {
1090 match (sz, src) {
1091 (Size::S64, Location::GPR(_)) | (Size::S64, Location::SIMD(_)) => {
1092 let offset = if self.pushed {
1093 0
1094 } else {
1095 self.assembler.emit_sub(
1096 Size::S64,
1097 Location::GPR(GPR::XzrSp),
1098 Location::Imm8(16),
1099 Location::GPR(GPR::XzrSp),
1100 )?;
1101 8
1102 };
1103 self.assembler
1104 .emit_stur(Size::S64, src, GPR::XzrSp, offset)?;
1105 self.pushed = !self.pushed;
1106 }
1107 (Size::S64, _) => {
1108 let mut temps = vec![];
1109 let src = self.location_to_reg(sz, src, &mut temps, ImmType::None, true, None)?;
1110 let offset = if self.pushed {
1111 0
1112 } else {
1113 self.assembler.emit_sub(
1114 Size::S64,
1115 Location::GPR(GPR::XzrSp),
1116 Location::Imm8(16),
1117 Location::GPR(GPR::XzrSp),
1118 )?;
1119 8
1120 };
1121 self.assembler
1122 .emit_stur(Size::S64, src, GPR::XzrSp, offset)?;
1123 self.pushed = !self.pushed;
1124 for r in temps {
1125 self.release_gpr(r);
1126 }
1127 }
1128 _ => codegen_error!("singlepass can't emit PUSH {:?} {:?}", sz, src),
1129 }
1130 Ok(())
1131 }
1132 fn emit_double_push(
1133 &mut self,
1134 sz: Size,
1135 src1: Location,
1136 src2: Location,
1137 ) -> Result<(), CompileError> {
1138 if !self.pushed {
1139 match (sz, src1, src2) {
1140 (Size::S64, Location::GPR(_), Location::GPR(_)) => {
1141 self.assembler
1142 .emit_stpdb(Size::S64, src1, src2, GPR::XzrSp, 16)?;
1143 }
1144 _ => {
1145 self.emit_push(sz, src1)?;
1146 self.emit_push(sz, src2)?;
1147 }
1148 }
1149 } else {
1150 self.emit_push(sz, src1)?;
1151 self.emit_push(sz, src2)?;
1152 }
1153 Ok(())
1154 }
1155 fn emit_pop(&mut self, sz: Size, dst: Location) -> Result<(), CompileError> {
1156 match (sz, dst) {
1157 (Size::S64, Location::GPR(_)) | (Size::S64, Location::SIMD(_)) => {
1158 let offset = if self.pushed { 8 } else { 0 };
1159 self.assembler
1160 .emit_ldur(Size::S64, dst, GPR::XzrSp, offset)?;
1161 if self.pushed {
1162 self.assembler.emit_add(
1163 Size::S64,
1164 Location::GPR(GPR::XzrSp),
1165 Location::Imm8(16),
1166 Location::GPR(GPR::XzrSp),
1167 )?;
1168 }
1169 self.pushed = !self.pushed;
1170 }
1171 _ => codegen_error!("singlepass can't emit POP {:?} {:?}", sz, dst),
1172 }
1173 Ok(())
1174 }
1175 fn emit_double_pop(
1176 &mut self,
1177 sz: Size,
1178 dst1: Location,
1179 dst2: Location,
1180 ) -> Result<(), CompileError> {
1181 if !self.pushed {
1182 match (sz, dst1, dst2) {
1183 (Size::S64, Location::GPR(_), Location::GPR(_)) => {
1184 self.assembler
1185 .emit_ldpia(Size::S64, dst1, dst2, GPR::XzrSp, 16)?;
1186 }
1187 _ => {
1188 self.emit_pop(sz, dst2)?;
1189 self.emit_pop(sz, dst1)?;
1190 }
1191 }
1192 } else {
1193 self.emit_pop(sz, dst2)?;
1194 self.emit_pop(sz, dst1)?;
1195 }
1196 Ok(())
1197 }
1198
1199 fn set_default_nan(&mut self, temps: &mut Vec<GPR>) -> Result<GPR, CompileError> {
1200 let old_fpcr = self.acquire_temp_gpr().ok_or_else(|| {
1202 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1203 })?;
1204 temps.push(old_fpcr);
1205 self.assembler.emit_read_fpcr(old_fpcr)?;
1206 let new_fpcr = self.acquire_temp_gpr().ok_or_else(|| {
1207 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1208 })?;
1209 temps.push(new_fpcr);
1210 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
1211 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1212 })?;
1213 temps.push(tmp);
1214 self.assembler
1215 .emit_mov(Size::S32, Location::Imm32(1), Location::GPR(tmp))?;
1216 self.assembler
1217 .emit_mov(Size::S64, Location::GPR(old_fpcr), Location::GPR(new_fpcr))?;
1218 self.assembler.emit_bfi(
1220 Size::S64,
1221 Location::GPR(tmp),
1222 25,
1223 1,
1224 Location::GPR(new_fpcr),
1225 )?;
1226 self.assembler.emit_write_fpcr(new_fpcr)?;
1227 Ok(old_fpcr)
1228 }
1229 fn set_trap_enabled(&mut self, temps: &mut Vec<GPR>) -> Result<GPR, CompileError> {
1230 let old_fpcr = self.acquire_temp_gpr().ok_or_else(|| {
1232 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1233 })?;
1234 temps.push(old_fpcr);
1235 self.assembler.emit_read_fpcr(old_fpcr)?;
1236 let new_fpcr = self.acquire_temp_gpr().ok_or_else(|| {
1237 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1238 })?;
1239 temps.push(new_fpcr);
1240 self.assembler
1241 .emit_mov(Size::S64, Location::GPR(old_fpcr), Location::GPR(new_fpcr))?;
1242 self.assembler
1244 .emit_bfc(Size::S64, 8, 1, Location::GPR(new_fpcr))?;
1245 self.assembler.emit_write_fpcr(new_fpcr)?;
1246 Ok(old_fpcr)
1247 }
1248 fn restore_fpcr(&mut self, old_fpcr: GPR) -> Result<(), CompileError> {
1249 self.assembler.emit_write_fpcr(old_fpcr)
1250 }
1251
1252 fn reset_exception_fpsr(&mut self) -> Result<(), CompileError> {
1253 let fpsr = self.acquire_temp_gpr().ok_or_else(|| {
1255 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1256 })?;
1257 self.assembler.emit_read_fpsr(fpsr)?;
1258 self.assembler
1260 .emit_bfc(Size::S64, 0, 1, Location::GPR(fpsr))?;
1261 self.assembler.emit_write_fpsr(fpsr)?;
1262 self.release_gpr(fpsr);
1263 Ok(())
1264 }
1265 fn read_fpsr(&mut self) -> Result<GPR, CompileError> {
1266 let fpsr = self.acquire_temp_gpr().ok_or_else(|| {
1267 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1268 })?;
1269 self.assembler.emit_read_fpsr(fpsr)?;
1270 Ok(fpsr)
1271 }
1272
1273 fn trap_float_convertion_errors(
1274 &mut self,
1275 old_fpcr: GPR,
1276 sz: Size,
1277 f: Location,
1278 temps: &mut Vec<GPR>,
1279 ) -> Result<(), CompileError> {
1280 let trap_badconv = self.assembler.get_label();
1281 let end = self.assembler.get_label();
1282
1283 let fpsr = self.read_fpsr()?;
1284 temps.push(fpsr);
1285 self.assembler
1287 .emit_tbz_label(Size::S32, Location::GPR(fpsr), 0, end)?;
1288 self.assembler
1290 .emit_bfc(Size::S64, 0, 4, Location::GPR(fpsr))?;
1291 self.restore_fpcr(old_fpcr)?;
1292 self.assembler.emit_fcmp(sz, f, f)?;
1293 self.assembler
1294 .emit_bcond_label(Condition::Vs, trap_badconv)?;
1295 self.emit_illegal_op_internal(TrapCode::IntegerOverflow)?;
1297
1298 self.emit_label(trap_badconv)?;
1299 self.emit_illegal_op_internal(TrapCode::BadConversionToInteger)?;
1300
1301 self.emit_label(end)?;
1302 self.restore_fpcr(old_fpcr)
1303 }
1304
1305 fn used_gprs_contains(&self, r: &GPR) -> bool {
1306 self.used_gprs & (1 << r.into_index()) != 0
1307 }
1308 fn used_simd_contains(&self, r: &NEON) -> bool {
1309 self.used_simd & (1 << r.into_index()) != 0
1310 }
1311 fn used_gprs_insert(&mut self, r: GPR) {
1312 self.used_gprs |= 1 << r.into_index();
1313 }
1314 fn used_simd_insert(&mut self, r: NEON) {
1315 self.used_simd |= 1 << r.into_index();
1316 }
1317 fn used_gprs_remove(&mut self, r: &GPR) -> bool {
1318 let ret = self.used_gprs_contains(r);
1319 self.used_gprs &= !(1 << r.into_index());
1320 ret
1321 }
1322 fn used_simd_remove(&mut self, r: &NEON) -> bool {
1323 let ret = self.used_simd_contains(r);
1324 self.used_simd &= !(1 << r.into_index());
1325 ret
1326 }
1327 fn emit_unwind_op(&mut self, op: UnwindOps<GPR, NEON>) {
1328 self.unwind_ops.push((self.get_offset().0, op));
1329 }
1330 fn emit_illegal_op_internal(&mut self, trap: TrapCode) -> Result<(), CompileError> {
1331 self.assembler.emit_udf(0xc0 | (trap as u8) as u16)
1332 }
1333}
1334
1335impl Machine for MachineARM64 {
1336 type GPR = GPR;
1337 type SIMD = NEON;
1338 fn assembler_get_offset(&self) -> Offset {
1339 self.assembler.get_offset()
1340 }
1341
1342 fn get_vmctx_reg(&self) -> GPR {
1343 GPR::X28
1344 }
1345
1346 fn get_used_gprs(&self) -> Vec<GPR> {
1347 GPR::iterator()
1348 .filter(|x| self.used_gprs & (1 << x.into_index()) != 0)
1349 .cloned()
1350 .collect()
1351 }
1352
1353 fn get_used_simd(&self) -> Vec<NEON> {
1354 NEON::iterator()
1355 .filter(|x| self.used_simd & (1 << x.into_index()) != 0)
1356 .cloned()
1357 .collect()
1358 }
1359
1360 fn pick_gpr(&self) -> Option<GPR> {
1361 use GPR::*;
1362 static REGS: &[GPR] = &[X9, X10, X11, X12, X13, X14, X15];
1363 for r in REGS {
1364 if !self.used_gprs_contains(r) {
1365 return Some(*r);
1366 }
1367 }
1368 None
1369 }
1370
1371 fn pick_temp_gpr(&self) -> Option<GPR> {
1373 use GPR::*;
1374 static REGS: &[GPR] = &[X8, X7, X6, X5, X4, X3, X2, X1];
1375 for r in REGS {
1376 if !self.used_gprs_contains(r) {
1377 return Some(*r);
1378 }
1379 }
1380 None
1381 }
1382
1383 fn acquire_temp_gpr(&mut self) -> Option<GPR> {
1384 let gpr = self.pick_temp_gpr();
1385 if let Some(x) = gpr {
1386 self.used_gprs_insert(x);
1387 }
1388 gpr
1389 }
1390
1391 fn release_gpr(&mut self, gpr: GPR) {
1392 assert!(self.used_gprs_remove(&gpr));
1393 }
1394
1395 fn reserve_unused_temp_gpr(&mut self, gpr: GPR) -> GPR {
1396 assert!(!self.used_gprs_contains(&gpr));
1397 self.used_gprs_insert(gpr);
1398 gpr
1399 }
1400
1401 fn reserve_gpr(&mut self, gpr: GPR) {
1402 self.used_gprs_insert(gpr);
1403 }
1404
1405 fn push_used_gpr(&mut self, used_gprs: &[GPR]) -> Result<usize, CompileError> {
1406 if used_gprs.len() % 2 == 1 {
1407 self.emit_push(Size::S64, Location::GPR(GPR::XzrSp))?;
1408 }
1409 for r in used_gprs.iter() {
1410 self.emit_push(Size::S64, Location::GPR(*r))?;
1411 }
1412 Ok(used_gprs.len().div_ceil(2) * 16)
1413 }
1414 fn pop_used_gpr(&mut self, used_gprs: &[GPR]) -> Result<(), CompileError> {
1415 for r in used_gprs.iter().rev() {
1416 self.emit_pop(Size::S64, Location::GPR(*r))?;
1417 }
1418 if used_gprs.len() % 2 == 1 {
1419 self.emit_pop(Size::S64, Location::GPR(GPR::XzrSp))?;
1420 }
1421 Ok(())
1422 }
1423
1424 fn pick_simd(&self) -> Option<NEON> {
1426 use NEON::*;
1427 static REGS: &[NEON] = &[V8, V9, V10, V11, V12];
1428 for r in REGS {
1429 if !self.used_simd_contains(r) {
1430 return Some(*r);
1431 }
1432 }
1433 None
1434 }
1435
1436 fn pick_temp_simd(&self) -> Option<NEON> {
1438 use NEON::*;
1439 static REGS: &[NEON] = &[V0, V1, V2, V3, V4, V5, V6, V7];
1440 for r in REGS {
1441 if !self.used_simd_contains(r) {
1442 return Some(*r);
1443 }
1444 }
1445 None
1446 }
1447
1448 fn acquire_temp_simd(&mut self) -> Option<NEON> {
1450 let simd = self.pick_temp_simd();
1451 if let Some(x) = simd {
1452 self.used_simd_insert(x);
1453 }
1454 simd
1455 }
1456
1457 fn reserve_simd(&mut self, simd: NEON) {
1458 self.used_simd_insert(simd);
1459 }
1460
1461 fn release_simd(&mut self, simd: NEON) {
1463 assert!(self.used_simd_remove(&simd));
1464 }
1465
1466 fn push_used_simd(&mut self, used_neons: &[NEON]) -> Result<usize, CompileError> {
1467 let stack_adjust = if used_neons.len() % 2 == 1 {
1468 (used_neons.len() * 8) as u32 + 8
1469 } else {
1470 (used_neons.len() * 8) as u32
1471 };
1472 self.extend_stack(stack_adjust)?;
1473
1474 for (i, r) in used_neons.iter().enumerate() {
1475 self.assembler.emit_str(
1476 Size::S64,
1477 Location::SIMD(*r),
1478 Location::Memory(GPR::XzrSp, (i * 8) as i32),
1479 )?;
1480 }
1481 Ok(stack_adjust as usize)
1482 }
1483 fn pop_used_simd(&mut self, used_neons: &[NEON]) -> Result<(), CompileError> {
1484 for (i, r) in used_neons.iter().enumerate() {
1485 self.assembler.emit_ldr(
1486 Size::S64,
1487 Location::SIMD(*r),
1488 Location::Memory(GPR::XzrSp, (i * 8) as i32),
1489 )?;
1490 }
1491 let stack_adjust = if used_neons.len() % 2 == 1 {
1492 (used_neons.len() * 8) as u32 + 8
1493 } else {
1494 (used_neons.len() * 8) as u32
1495 };
1496 self.assembler.emit_add(
1497 Size::S64,
1498 Location::GPR(GPR::XzrSp),
1499 Location::Imm32(stack_adjust as _),
1500 Location::GPR(GPR::XzrSp),
1501 )
1502 }
1503
1504 fn set_srcloc(&mut self, offset: u32) {
1506 self.src_loc = offset;
1507 }
1508 fn mark_address_range_with_trap_code(&mut self, code: TrapCode, begin: usize, end: usize) {
1510 for i in begin..end {
1511 self.trap_table.offset_to_code.insert(i, code);
1512 }
1513 self.mark_instruction_address_end(begin);
1514 }
1515
1516 fn mark_address_with_trap_code(&mut self, code: TrapCode) {
1518 let offset = self.assembler.get_offset().0;
1519 self.trap_table.offset_to_code.insert(offset, code);
1520 self.mark_instruction_address_end(offset);
1521 }
1522 fn mark_instruction_with_trap_code(&mut self, code: TrapCode) -> usize {
1524 let offset = self.assembler.get_offset().0;
1525 self.trap_table.offset_to_code.insert(offset, code);
1526 offset
1527 }
1528 fn mark_instruction_address_end(&mut self, begin: usize) {
1531 self.instructions_address_map.push(InstructionAddressMap {
1532 srcloc: SourceLoc::new(self.src_loc),
1533 code_offset: begin,
1534 code_len: self.assembler.get_offset().0 - begin,
1535 });
1536 }
1537
1538 fn insert_stackoverflow(&mut self) {
1540 let offset = 0;
1541 self.trap_table
1542 .offset_to_code
1543 .insert(offset, TrapCode::StackOverflow);
1544 self.mark_instruction_address_end(offset);
1545 }
1546
1547 fn collect_trap_information(&self) -> Vec<TrapInformation> {
1549 self.trap_table
1550 .offset_to_code
1551 .clone()
1552 .into_iter()
1553 .map(|(offset, code)| TrapInformation {
1554 code_offset: offset as u32,
1555 trap_code: code,
1556 })
1557 .collect()
1558 }
1559
1560 fn instructions_address_map(&self) -> Vec<InstructionAddressMap> {
1561 self.instructions_address_map.clone()
1562 }
1563
1564 fn round_stack_adjust(&self, value: usize) -> usize {
1566 value.next_multiple_of(16)
1567 }
1568
1569 fn local_on_stack(&mut self, stack_offset: i32) -> Location {
1571 Location::Memory(GPR::X29, -stack_offset)
1572 }
1573
1574 fn extend_stack(&mut self, delta_stack_offset: u32) -> Result<(), CompileError> {
1575 let delta = if self.compatible_imm(delta_stack_offset as _, ImmType::Bits12) {
1576 Location::Imm32(delta_stack_offset as _)
1577 } else {
1578 let tmp = GPR::X17;
1579 self.assembler
1580 .emit_mov_imm(Location::GPR(tmp), delta_stack_offset as u64)?;
1581 Location::GPR(tmp)
1582 };
1583 self.assembler.emit_sub(
1584 Size::S64,
1585 Location::GPR(GPR::XzrSp),
1586 delta,
1587 Location::GPR(GPR::XzrSp),
1588 )
1589 }
1590
1591 fn truncate_stack(&mut self, delta_stack_offset: u32) -> Result<(), CompileError> {
1592 let delta = if self.compatible_imm(delta_stack_offset as _, ImmType::Bits12) {
1593 Location::Imm32(delta_stack_offset as _)
1594 } else {
1595 let tmp = GPR::X17;
1596 self.assembler
1597 .emit_mov_imm(Location::GPR(tmp), delta_stack_offset as u64)?;
1598 Location::GPR(tmp)
1599 };
1600 self.assembler.emit_add(
1601 Size::S64,
1602 Location::GPR(GPR::XzrSp),
1603 delta,
1604 Location::GPR(GPR::XzrSp),
1605 )
1606 }
1607
1608 fn move_location_for_native(
1610 &mut self,
1611 size: Size,
1612 loc: Location,
1613 dest: Location,
1614 ) -> Result<(), CompileError> {
1615 match loc {
1616 Location::Imm64(_)
1617 | Location::Imm32(_)
1618 | Location::Imm8(_)
1619 | Location::Memory(_, _)
1620 | Location::Memory2(_, _, _, _) => {
1621 self.move_location(size, loc, Location::GPR(GPR::X17))?;
1622 self.move_location(size, Location::GPR(GPR::X17), dest)
1623 }
1624 _ => self.move_location(size, loc, dest),
1625 }
1626 }
1627
1628 fn zero_location(&mut self, size: Size, location: Location) -> Result<(), CompileError> {
1630 self.move_location(size, Location::GPR(GPR::XzrSp), location)
1631 }
1632
1633 fn local_pointer(&self) -> GPR {
1635 GPR::X29
1636 }
1637
1638 fn is_local_on_stack(&self, idx: usize) -> bool {
1640 idx > 7
1641 }
1642
1643 fn get_local_location(&self, idx: usize, callee_saved_regs_size: usize) -> Location {
1645 match idx {
1647 0 => Location::GPR(GPR::X19),
1648 1 => Location::GPR(GPR::X20),
1649 2 => Location::GPR(GPR::X21),
1650 3 => Location::GPR(GPR::X22),
1651 4 => Location::GPR(GPR::X23),
1652 5 => Location::GPR(GPR::X24),
1653 6 => Location::GPR(GPR::X25),
1654 7 => Location::GPR(GPR::X26),
1655 _ => Location::Memory(GPR::X29, -(((idx - 7) * 8 + callee_saved_regs_size) as i32)),
1656 }
1657 }
1658 fn move_local(&mut self, stack_offset: i32, location: Location) -> Result<(), CompileError> {
1660 if stack_offset < 256 {
1661 self.assembler
1662 .emit_stur(Size::S64, location, GPR::X29, -stack_offset)?;
1663 } else {
1664 let tmp = GPR::X17;
1665 if stack_offset < 0x1_0000 {
1666 self.assembler
1667 .emit_mov_imm(Location::GPR(tmp), (-stack_offset as i64) as u64)?;
1668 self.assembler.emit_str(
1669 Size::S64,
1670 location,
1671 Location::Memory2(GPR::X29, tmp, Multiplier::One, 0),
1672 )?;
1673 } else {
1674 self.assembler
1675 .emit_mov_imm(Location::GPR(tmp), (stack_offset as i64) as u64)?;
1676 self.assembler.emit_sub(
1677 Size::S64,
1678 Location::GPR(GPR::X29),
1679 Location::GPR(tmp),
1680 Location::GPR(tmp),
1681 )?;
1682 self.assembler
1683 .emit_str(Size::S64, location, Location::GPR(tmp))?;
1684 }
1685 }
1686 match location {
1687 Location::GPR(x) => self.emit_unwind_op(UnwindOps::SaveRegister {
1688 reg: UnwindRegister::<GPR, NEON>::GPR(x),
1689 bp_neg_offset: stack_offset,
1690 }),
1691 Location::SIMD(x) => self.emit_unwind_op(UnwindOps::SaveRegister {
1692 reg: UnwindRegister::FPR(x),
1693 bp_neg_offset: stack_offset,
1694 }),
1695 _ => (),
1696 }
1697 Ok(())
1698 }
1699
1700 fn list_to_save(&self, _calling_convention: CallingConvention) -> Vec<Location> {
1702 vec![]
1703 }
1704
1705 fn get_param_registers(&self, _calling_convention: CallingConvention) -> &'static [Self::GPR] {
1707 &[
1708 GPR::X0,
1709 GPR::X1,
1710 GPR::X2,
1711 GPR::X3,
1712 GPR::X4,
1713 GPR::X5,
1714 GPR::X6,
1715 GPR::X7,
1716 ]
1717 }
1718
1719 fn get_param_location(
1721 &self,
1722 idx: usize,
1723 sz: Size,
1724 stack_args: &mut usize,
1725 calling_convention: CallingConvention,
1726 ) -> Location {
1727 let register_params = self.get_param_registers(calling_convention);
1728 match calling_convention {
1729 CallingConvention::AppleAarch64 => register_params.get(idx).map_or_else(
1730 || {
1731 let sz = sz.bytes() as usize;
1733 *stack_args = (*stack_args).next_multiple_of(sz);
1734 let loc = Location::Memory(GPR::XzrSp, *stack_args as i32);
1735 *stack_args += sz;
1736 loc
1737 },
1738 |reg| Location::GPR(*reg),
1739 ),
1740 _ => {
1741 if let Some(reg) = register_params.get(idx) {
1742 Location::GPR(*reg)
1743 } else {
1744 let loc = Location::Memory(GPR::XzrSp, *stack_args as i32);
1745 *stack_args += 8;
1746 loc
1747 }
1748 }
1749 }
1750 }
1751 fn get_call_param_location(
1753 &self,
1754 return_slots: usize,
1755 idx: usize,
1756 sz: Size,
1757 stack_args: &mut usize,
1758 calling_convention: CallingConvention,
1759 ) -> Location {
1760 let register_params = self.get_param_registers(calling_convention);
1761 let return_values_memory_size =
1762 8 * return_slots.saturating_sub(ARM64_RETURN_VALUE_REGISTERS.len()) as i32;
1763
1764 match calling_convention {
1765 CallingConvention::AppleAarch64 => register_params.get(idx).map_or_else(
1766 || {
1767 let sz = sz.bytes() as usize;
1768 *stack_args = (*stack_args).next_multiple_of(sz);
1770 let loc = Location::Memory(
1771 GPR::X29,
1772 16 * 2 + return_values_memory_size + *stack_args as i32,
1773 );
1774 *stack_args += sz;
1775 loc
1776 },
1777 |reg| Location::GPR(*reg),
1778 ),
1779 _ => register_params.get(idx).map_or_else(
1780 || {
1781 let loc = Location::Memory(
1782 GPR::X29,
1783 16 * 2 + return_values_memory_size + *stack_args as i32,
1784 );
1785 *stack_args += 8;
1786 loc
1787 },
1788 |reg| Location::GPR(*reg),
1789 ),
1790 }
1791 }
1792
1793 fn get_simple_param_location(
1794 &self,
1795 idx: usize,
1796 calling_convention: CallingConvention,
1797 ) -> Self::GPR {
1798 self.get_param_registers(calling_convention)[idx]
1799 }
1800
1801 fn get_return_value_location(
1802 &self,
1803 idx: usize,
1804 stack_location: &mut usize,
1805 _calling_convention: CallingConvention,
1806 ) -> AbstractLocation<Self::GPR, Self::SIMD> {
1807 ARM64_RETURN_VALUE_REGISTERS.get(idx).map_or_else(
1808 || {
1809 let loc = Location::Memory(GPR::XzrSp, *stack_location as i32);
1810 *stack_location += 8;
1811 loc
1812 },
1813 |reg| Location::GPR(*reg),
1814 )
1815 }
1816
1817 fn get_call_return_value_location(
1818 &self,
1819 idx: usize,
1820 _calling_convention: CallingConvention,
1821 ) -> AbstractLocation<Self::GPR, Self::SIMD> {
1822 ARM64_RETURN_VALUE_REGISTERS.get(idx).map_or_else(
1823 || {
1824 Location::Memory(
1825 GPR::X29,
1826 (16 * 2 + (idx - ARM64_RETURN_VALUE_REGISTERS.len()) * 8) as i32,
1827 )
1828 },
1829 |reg| Location::GPR(*reg),
1830 )
1831 }
1832
1833 fn move_location(
1835 &mut self,
1836 size: Size,
1837 source: Location,
1838 dest: Location,
1839 ) -> Result<(), CompileError> {
1840 match source {
1841 Location::GPR(_) | Location::SIMD(_) => match dest {
1842 Location::GPR(_) | Location::SIMD(_) => self.assembler.emit_mov(size, source, dest),
1843 Location::Memory(addr, offs) => {
1844 if self.offset_is_ok(size, offs) {
1845 self.assembler.emit_str(size, source, dest)
1846 } else if self.compatible_imm(offs as i64, ImmType::UnscaledOffset) {
1847 self.assembler.emit_stur(size, source, addr, offs)
1848 } else {
1849 let tmp = GPR::X17;
1850 if offs < 0 {
1851 self.assembler
1852 .emit_mov_imm(Location::GPR(tmp), (-offs) as u64)?;
1853 self.assembler.emit_sub(
1854 Size::S64,
1855 Location::GPR(addr),
1856 Location::GPR(tmp),
1857 Location::GPR(tmp),
1858 )?;
1859 } else {
1860 self.assembler
1861 .emit_mov_imm(Location::GPR(tmp), offs as u64)?;
1862 self.assembler.emit_add(
1863 Size::S64,
1864 Location::GPR(addr),
1865 Location::GPR(tmp),
1866 Location::GPR(tmp),
1867 )?;
1868 }
1869 self.assembler
1870 .emit_str(size, source, Location::Memory(tmp, 0))
1871 }
1872 }
1873 _ => codegen_error!(
1874 "singlepass can't emit move_location {:?} {:?} => {:?}",
1875 size,
1876 source,
1877 dest
1878 ),
1879 },
1880 Location::Imm8(_) => match dest {
1881 Location::GPR(_) => self.assembler.emit_mov(size, source, dest),
1882 Location::Memory(_, _) => match size {
1883 Size::S64 => self.emit_relaxed_str64(source, dest),
1884 Size::S32 => self.emit_relaxed_str32(source, dest),
1885 Size::S16 => self.emit_relaxed_str16(source, dest),
1886 Size::S8 => self.emit_relaxed_str8(source, dest),
1887 },
1888 _ => codegen_error!(
1889 "singlepass can't emit move_location {:?} {:?} => {:?}",
1890 size,
1891 source,
1892 dest
1893 ),
1894 },
1895 Location::Imm32(val) => match dest {
1896 Location::GPR(_) => self.assembler.emit_mov_imm(dest, val as u64),
1897 Location::Memory(_, _) => match size {
1898 Size::S64 => self.emit_relaxed_str64(source, dest),
1899 Size::S32 => self.emit_relaxed_str32(source, dest),
1900 Size::S16 => self.emit_relaxed_str16(source, dest),
1901 Size::S8 => self.emit_relaxed_str8(source, dest),
1902 },
1903 _ => codegen_error!(
1904 "singlepass can't emit move_location {:?} {:?} => {:?}",
1905 size,
1906 source,
1907 dest
1908 ),
1909 },
1910 Location::Imm64(val) => match dest {
1911 Location::GPR(_) => self.assembler.emit_mov_imm(dest, val),
1912 Location::Memory(_, _) => match size {
1913 Size::S64 => self.emit_relaxed_str64(source, dest),
1914 Size::S32 => self.emit_relaxed_str32(source, dest),
1915 Size::S16 => self.emit_relaxed_str16(source, dest),
1916 Size::S8 => self.emit_relaxed_str8(source, dest),
1917 },
1918 _ => codegen_error!(
1919 "singlepass can't emit move_location {:?} {:?} => {:?}",
1920 size,
1921 source,
1922 dest
1923 ),
1924 },
1925 Location::Memory(addr, offs) => match dest {
1926 Location::GPR(_) | Location::SIMD(_) => {
1927 if self.offset_is_ok(size, offs) {
1928 self.assembler.emit_ldr(size, dest, source)
1929 } else if offs > -256 && offs < 256 {
1930 self.assembler.emit_ldur(size, dest, addr, offs)
1931 } else {
1932 let tmp = GPR::X17;
1933 if offs < 0 {
1934 self.assembler
1935 .emit_mov_imm(Location::GPR(tmp), (-offs) as u64)?;
1936 self.assembler.emit_sub(
1937 Size::S64,
1938 Location::GPR(addr),
1939 Location::GPR(tmp),
1940 Location::GPR(tmp),
1941 )?;
1942 } else {
1943 self.assembler
1944 .emit_mov_imm(Location::GPR(tmp), offs as u64)?;
1945 self.assembler.emit_add(
1946 Size::S64,
1947 Location::GPR(addr),
1948 Location::GPR(tmp),
1949 Location::GPR(tmp),
1950 )?;
1951 }
1952 self.assembler
1953 .emit_ldr(size, dest, Location::Memory(tmp, 0))
1954 }
1955 }
1956 _ => {
1957 let mut temps = vec![];
1958 let src =
1959 self.location_to_reg(size, source, &mut temps, ImmType::None, true, None)?;
1960 self.move_location(size, src, dest)?;
1961 for r in temps {
1962 self.release_gpr(r);
1963 }
1964 Ok(())
1965 }
1966 },
1967 _ => codegen_error!(
1968 "singlepass can't emit move_location {:?} {:?} => {:?}",
1969 size,
1970 source,
1971 dest
1972 ),
1973 }
1974 }
1975 fn move_location_extend(
1977 &mut self,
1978 size_val: Size,
1979 signed: bool,
1980 source: Location,
1981 size_op: Size,
1982 dest: Location,
1983 ) -> Result<(), CompileError> {
1984 if size_op != Size::S64 {
1985 codegen_error!("singlepass move_location_extend unreachable");
1986 }
1987 let mut temps = vec![];
1988 let dst = self.location_to_reg(size_op, dest, &mut temps, ImmType::None, false, None)?;
1989 let src = match (size_val, signed, source) {
1990 (Size::S64, _, _) => source,
1991 (Size::S32, false, Location::GPR(_)) => {
1992 self.assembler.emit_mov(size_val, source, dst)?;
1993 dst
1994 }
1995 (Size::S8, false, Location::GPR(_)) => {
1996 self.assembler.emit_uxtb(size_op, source, dst)?;
1997 dst
1998 }
1999 (Size::S16, false, Location::GPR(_)) => {
2000 self.assembler.emit_uxth(size_op, source, dst)?;
2001 dst
2002 }
2003 (Size::S8, true, Location::GPR(_)) => {
2004 self.assembler.emit_sxtb(size_op, source, dst)?;
2005 dst
2006 }
2007 (Size::S16, true, Location::GPR(_)) => {
2008 self.assembler.emit_sxth(size_op, source, dst)?;
2009 dst
2010 }
2011 (Size::S32, true, Location::GPR(_)) => {
2012 self.assembler.emit_sxtw(size_op, source, dst)?;
2013 dst
2014 }
2015 (Size::S32, false, Location::Memory(_, _)) => {
2016 self.emit_relaxed_ldr32(size_op, dst, source)?;
2017 dst
2018 }
2019 (Size::S32, true, Location::Memory(_, _)) => {
2020 self.emit_relaxed_ldr32s(size_op, dst, source)?;
2021 dst
2022 }
2023 (Size::S16, false, Location::Memory(_, _)) => {
2024 self.emit_relaxed_ldr16(size_op, dst, source)?;
2025 dst
2026 }
2027 (Size::S16, true, Location::Memory(_, _)) => {
2028 self.emit_relaxed_ldr16s(size_op, dst, source)?;
2029 dst
2030 }
2031 (Size::S8, false, Location::Memory(_, _)) => {
2032 self.emit_relaxed_ldr8(size_op, dst, source)?;
2033 dst
2034 }
2035 (Size::S8, true, Location::Memory(_, _)) => {
2036 self.emit_relaxed_ldr8s(size_op, dst, source)?;
2037 dst
2038 }
2039 _ => codegen_error!(
2040 "singlepass can't emit move_location_extend {:?} {:?} {:?} => {:?} {:?}",
2041 size_val,
2042 signed,
2043 source,
2044 size_op,
2045 dest
2046 ),
2047 };
2048 if src != dst {
2049 self.move_location(size_op, src, dst)?;
2050 }
2051 if dst != dest {
2052 self.move_location(size_op, dst, dest)?;
2053 }
2054 for r in temps {
2055 self.release_gpr(r);
2056 }
2057 Ok(())
2058 }
2059
2060 fn init_stack_loc(
2062 &mut self,
2063 init_stack_loc_cnt: u64,
2064 last_stack_loc: Location,
2065 ) -> Result<(), CompileError> {
2066 let label = self.assembler.get_label();
2067 let mut temps = vec![];
2068 let dest = self.acquire_temp_gpr().ok_or_else(|| {
2069 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2070 })?;
2071 temps.push(dest);
2072 let cnt = self.location_to_reg(
2073 Size::S64,
2074 Location::Imm64(init_stack_loc_cnt),
2075 &mut temps,
2076 ImmType::None,
2077 true,
2078 None,
2079 )?;
2080 let dest = match last_stack_loc {
2081 Location::GPR(_) => codegen_error!("singlepass init_stack_loc unreachable"),
2082 Location::SIMD(_) => codegen_error!("singlepass init_stack_loc unreachable"),
2083 Location::Memory(reg, offset) => {
2084 if offset < 0 {
2085 let offset = (-offset) as u32;
2086 if self.compatible_imm(offset as i64, ImmType::Bits12) {
2087 self.assembler.emit_sub(
2088 Size::S64,
2089 Location::GPR(reg),
2090 Location::Imm32(offset),
2091 Location::GPR(dest),
2092 )?;
2093 } else {
2094 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2095 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2096 })?;
2097 self.assembler
2098 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
2099 self.assembler.emit_sub(
2100 Size::S64,
2101 Location::GPR(reg),
2102 Location::GPR(tmp),
2103 Location::GPR(dest),
2104 )?;
2105 temps.push(tmp);
2106 }
2107 dest
2108 } else {
2109 let offset = offset as u32;
2110 if self.compatible_imm(offset as i64, ImmType::Bits12) {
2111 self.assembler.emit_add(
2112 Size::S64,
2113 Location::GPR(reg),
2114 Location::Imm32(offset),
2115 Location::GPR(dest),
2116 )?;
2117 } else {
2118 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2119 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2120 })?;
2121 self.assembler
2122 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
2123 self.assembler.emit_add(
2124 Size::S64,
2125 Location::GPR(reg),
2126 Location::GPR(tmp),
2127 Location::GPR(dest),
2128 )?;
2129 temps.push(tmp);
2130 }
2131 dest
2132 }
2133 }
2134 _ => codegen_error!("singlepass can't emit init_stack_loc {:?}", last_stack_loc),
2135 };
2136 self.assembler.emit_label(label)?;
2137 self.assembler
2138 .emit_stria(Size::S64, Location::GPR(GPR::XzrSp), dest, 8)?;
2139 self.assembler
2140 .emit_sub(Size::S64, cnt, Location::Imm8(1), cnt)?;
2141 self.assembler.emit_cbnz_label(Size::S64, cnt, label)?;
2142 for r in temps {
2143 self.release_gpr(r);
2144 }
2145 Ok(())
2146 }
2147 fn restore_saved_area(&mut self, saved_area_offset: i32) -> Result<(), CompileError> {
2149 let real_delta = if saved_area_offset & 15 != 0 {
2150 self.pushed = true;
2151 saved_area_offset + 8
2152 } else {
2153 self.pushed = false;
2154 saved_area_offset
2155 };
2156 if self.compatible_imm(real_delta as _, ImmType::Bits12) {
2157 self.assembler.emit_sub(
2158 Size::S64,
2159 Location::GPR(GPR::X29),
2160 Location::Imm32(real_delta as _),
2161 Location::GPR(GPR::XzrSp),
2162 )?;
2163 } else {
2164 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2165 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2166 })?;
2167 self.assembler
2168 .emit_mov_imm(Location::GPR(tmp), real_delta as u64)?;
2169 self.assembler.emit_sub(
2170 Size::S64,
2171 Location::GPR(GPR::X29),
2172 Location::GPR(tmp),
2173 Location::GPR(GPR::XzrSp),
2174 )?;
2175 self.release_gpr(tmp);
2176 }
2177 Ok(())
2178 }
2179 fn pop_location(&mut self, location: Location) -> Result<(), CompileError> {
2181 self.emit_pop(Size::S64, location)
2182 }
2183
2184 fn assembler_finalize(
2186 self,
2187 assembly_comments: HashMap<usize, AssemblyComment>,
2188 ) -> Result<FinalizedAssembly, CompileError> {
2189 Ok(FinalizedAssembly {
2190 body: self.assembler.finalize().map_err(|e| {
2191 CompileError::Codegen(format!("Assembler failed finalization with: {e:?}"))
2192 })?,
2193 assembly_comments,
2194 })
2195 }
2196
2197 fn get_offset(&self) -> Offset {
2198 self.assembler.get_offset()
2199 }
2200
2201 fn finalize_function(&mut self) -> Result<(), CompileError> {
2202 self.assembler.finalize_function();
2203 Ok(())
2204 }
2205
2206 fn emit_function_prolog(&mut self) -> Result<(), CompileError> {
2207 self.emit_double_push(Size::S64, Location::GPR(GPR::X29), Location::GPR(GPR::X30))?; self.emit_unwind_op(UnwindOps::Push2Regs {
2209 reg1: UnwindRegister::GPR(GPR::X29),
2210 reg2: UnwindRegister::GPR(GPR::X30),
2211 up_to_sp: 16,
2212 });
2213 self.emit_double_push(Size::S64, Location::GPR(GPR::X27), Location::GPR(GPR::X28))?;
2214 self.emit_unwind_op(UnwindOps::Push2Regs {
2215 reg1: UnwindRegister::GPR(GPR::X27),
2216 reg2: UnwindRegister::GPR(GPR::X28),
2217 up_to_sp: 32,
2218 });
2219 self.assembler.emit_add(
2221 Size::S64,
2222 Location::GPR(GPR::XzrSp),
2223 Location::Imm8(0),
2224 Location::GPR(GPR::X29),
2225 )?;
2226 self.emit_unwind_op(UnwindOps::DefineNewFrame);
2227 Ok(())
2228 }
2229
2230 fn emit_function_epilog(&mut self) -> Result<(), CompileError> {
2231 self.assembler.emit_add(
2233 Size::S64,
2234 Location::GPR(GPR::X29),
2235 Location::Imm8(0),
2236 Location::GPR(GPR::XzrSp),
2237 )?;
2238 self.pushed = false; self.emit_double_pop(Size::S64, Location::GPR(GPR::X27), Location::GPR(GPR::X28))?;
2240 self.emit_double_pop(Size::S64, Location::GPR(GPR::X29), Location::GPR(GPR::X30))?;
2241 Ok(())
2242 }
2243
2244 fn emit_function_return_float(&mut self) -> Result<(), CompileError> {
2245 self.assembler
2246 .emit_mov(Size::S64, Location::GPR(GPR::X0), Location::SIMD(NEON::V0))
2247 }
2248
2249 fn canonicalize_nan(
2250 &mut self,
2251 sz: Size,
2252 input: Location,
2253 output: Location,
2254 ) -> Result<(), CompileError> {
2255 let mut tempn = vec![];
2256 let mut temps = vec![];
2257 let old_fpcr = self.set_default_nan(&mut temps)?;
2258 match (sz, input, output) {
2260 (Size::S32, Location::SIMD(_), Location::SIMD(_)) => {
2261 self.assembler.emit_fmax(sz, input, input, output)?;
2262 }
2263 (Size::S64, Location::SIMD(_), Location::SIMD(_)) => {
2264 self.assembler.emit_fmax(sz, input, input, output)?;
2265 }
2266 (Size::S32, Location::SIMD(_), _) | (Size::S64, Location::SIMD(_), _) => {
2267 let tmp = self.location_to_neon(sz, output, &mut tempn, ImmType::None, false)?;
2268 self.assembler.emit_fmax(sz, input, input, tmp)?;
2269 self.move_location(sz, tmp, output)?;
2270 }
2271 (Size::S32, Location::Memory(_, _), _) | (Size::S64, Location::Memory(_, _), _) => {
2272 let src = self.location_to_neon(sz, input, &mut tempn, ImmType::None, true)?;
2273 let tmp = self.location_to_neon(sz, output, &mut tempn, ImmType::None, false)?;
2274 self.assembler.emit_fmax(sz, src, src, tmp)?;
2275 if tmp != output {
2276 self.move_location(sz, tmp, output)?;
2277 }
2278 }
2279 _ => codegen_error!(
2280 "singlepass can't emit canonicalize_nan {:?} {:?} {:?}",
2281 sz,
2282 input,
2283 output
2284 ),
2285 }
2286
2287 self.restore_fpcr(old_fpcr)?;
2288 for r in temps {
2289 self.release_gpr(r);
2290 }
2291 for r in tempn {
2292 self.release_simd(r);
2293 }
2294 Ok(())
2295 }
2296
2297 fn emit_illegal_op(&mut self, trap: TrapCode) -> Result<(), CompileError> {
2298 let offset = self.assembler.get_offset().0;
2299 self.assembler.emit_udf(0xc0 | (trap as u8) as u16)?;
2300 self.mark_instruction_address_end(offset);
2301 Ok(())
2302 }
2303 fn get_label(&mut self) -> Label {
2304 self.assembler.new_dynamic_label()
2305 }
2306 fn emit_label(&mut self, label: Label) -> Result<(), CompileError> {
2307 self.assembler.emit_label(label)
2308 }
2309 fn get_gpr_for_call(&self) -> GPR {
2310 GPR::X27
2311 }
2312 fn emit_call_register(&mut self, reg: GPR) -> Result<(), CompileError> {
2313 self.assembler.emit_call_register(reg)
2314 }
2315 fn emit_call_label(&mut self, label: Label) -> Result<(), CompileError> {
2316 self.assembler.emit_call_label(label)
2317 }
2318
2319 fn arch_emit_indirect_call_with_trampoline(
2320 &mut self,
2321 location: Location,
2322 ) -> Result<(), CompileError> {
2323 self.assembler
2324 .arch_emit_indirect_call_with_trampoline(location)
2325 }
2326
2327 fn emit_debug_breakpoint(&mut self) -> Result<(), CompileError> {
2328 self.assembler.emit_brk()
2329 }
2330
2331 fn emit_call_location(&mut self, location: Location) -> Result<(), CompileError> {
2332 let mut temps = vec![];
2333 let loc = self.location_to_reg(
2334 Size::S64,
2335 location,
2336 &mut temps,
2337 ImmType::None,
2338 true,
2339 Some(self.get_gpr_for_call()),
2340 )?;
2341 match loc {
2342 Location::GPR(reg) => self.assembler.emit_call_register(reg),
2343 _ => codegen_error!("singlepass can't emit CALL Location"),
2344 }?;
2345 for r in temps {
2346 self.release_gpr(r);
2347 }
2348 Ok(())
2349 }
2350 fn location_add(
2352 &mut self,
2353 size: Size,
2354 source: Location,
2355 dest: Location,
2356 flags: bool,
2357 ) -> Result<(), CompileError> {
2358 let mut temps = vec![];
2359 let src = self.location_to_reg(size, source, &mut temps, ImmType::Bits12, true, None)?;
2360 let dst = self.location_to_reg(size, dest, &mut temps, ImmType::None, true, None)?;
2361 if flags {
2362 self.assembler.emit_adds(size, dst, src, dst)?;
2363 } else {
2364 self.assembler.emit_add(size, dst, src, dst)?;
2365 }
2366 if dst != dest {
2367 self.move_location(size, dst, dest)?;
2368 }
2369 for r in temps {
2370 self.release_gpr(r);
2371 }
2372 Ok(())
2373 }
2374 fn location_cmp(
2375 &mut self,
2376 size: Size,
2377 source: Location,
2378 dest: Location,
2379 ) -> Result<(), CompileError> {
2380 self.emit_relaxed_binop(Assembler::emit_cmp, size, source, dest, false)
2381 }
2382 fn jmp_unconditional(&mut self, label: Label) -> Result<(), CompileError> {
2383 self.assembler.emit_b_label(label)
2384 }
2385
2386 fn jmp_on_condition(
2387 &mut self,
2388 cond: UnsignedCondition,
2389 size: Size,
2390 loc_a: AbstractLocation<Self::GPR, Self::SIMD>,
2391 loc_b: AbstractLocation<Self::GPR, Self::SIMD>,
2392 label: Label,
2393 ) -> Result<(), CompileError> {
2394 self.emit_relaxed_binop(Assembler::emit_cmp, size, loc_b, loc_a, false)?;
2395 let cond = match cond {
2396 UnsignedCondition::Equal => Condition::Eq,
2397 UnsignedCondition::NotEqual => Condition::Ne,
2398 UnsignedCondition::Above => Condition::Hi,
2399 UnsignedCondition::AboveEqual => Condition::Cs,
2400 UnsignedCondition::Below => Condition::Cc,
2401 UnsignedCondition::BelowEqual => Condition::Ls,
2402 };
2403 self.assembler.emit_bcond_label_far(cond, label)
2404 }
2405
2406 fn emit_jmp_to_jumptable(&mut self, label: Label, cond: Location) -> Result<(), CompileError> {
2408 let tmp1 = self.acquire_temp_gpr().ok_or_else(|| {
2409 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2410 })?;
2411 let tmp2 = self.acquire_temp_gpr().ok_or_else(|| {
2412 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2413 })?;
2414
2415 self.assembler.emit_load_label(tmp1, label)?;
2416 self.move_location(Size::S32, cond, Location::GPR(tmp2))?;
2417
2418 self.assembler.emit_add_lsl(
2419 Size::S64,
2420 Location::GPR(tmp1),
2421 Location::GPR(tmp2),
2422 2,
2423 Location::GPR(tmp2),
2424 )?;
2425 self.assembler.emit_b_register(tmp2)?;
2426 self.release_gpr(tmp2);
2427 self.release_gpr(tmp1);
2428 Ok(())
2429 }
2430
2431 fn align_for_loop(&mut self) -> Result<(), CompileError> {
2432 Ok(())
2434 }
2435
2436 fn emit_ret(&mut self) -> Result<(), CompileError> {
2437 self.assembler.emit_ret()
2438 }
2439
2440 fn emit_push(&mut self, size: Size, loc: Location) -> Result<(), CompileError> {
2441 self.emit_push(size, loc)
2442 }
2443 fn emit_pop(&mut self, size: Size, loc: Location) -> Result<(), CompileError> {
2444 self.emit_pop(size, loc)
2445 }
2446
2447 fn emit_memory_fence(&mut self) -> Result<(), CompileError> {
2448 self.assembler.emit_dmb()
2449 }
2450
2451 fn emit_imul_imm32(&mut self, size: Size, imm32: u32, gpr: GPR) -> Result<(), CompileError> {
2452 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2453 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2454 })?;
2455 self.assembler
2456 .emit_mov_imm(Location::GPR(tmp), imm32 as u64)?;
2457 self.assembler.emit_mul(
2458 size,
2459 Location::GPR(gpr),
2460 Location::GPR(tmp),
2461 Location::GPR(gpr),
2462 )?;
2463 self.release_gpr(tmp);
2464 Ok(())
2465 }
2466
2467 fn emit_relaxed_mov(
2469 &mut self,
2470 sz: Size,
2471 src: Location,
2472 dst: Location,
2473 ) -> Result<(), CompileError> {
2474 self.emit_relaxed_binop(Assembler::emit_mov, sz, src, dst, true)
2475 }
2476 fn emit_relaxed_cmp(
2477 &mut self,
2478 sz: Size,
2479 src: Location,
2480 dst: Location,
2481 ) -> Result<(), CompileError> {
2482 self.emit_relaxed_binop(Assembler::emit_cmp, sz, src, dst, false)
2483 }
2484 fn emit_relaxed_sign_extension(
2485 &mut self,
2486 sz_src: Size,
2487 src: Location,
2488 sz_dst: Size,
2489 dst: Location,
2490 ) -> Result<(), CompileError> {
2491 match (src, dst) {
2492 (Location::Memory(_, _), Location::GPR(_)) => match sz_src {
2493 Size::S8 => self.emit_relaxed_ldr8s(sz_dst, dst, src),
2494 Size::S16 => self.emit_relaxed_ldr16s(sz_dst, dst, src),
2495 Size::S32 => self.emit_relaxed_ldr32s(sz_dst, dst, src),
2496 _ => codegen_error!("singlepass emit_relaxed_sign_extension unreachable"),
2497 },
2498 _ => {
2499 let mut temps = vec![];
2500 let src =
2501 self.location_to_reg(sz_src, src, &mut temps, ImmType::None, true, None)?;
2502 let dest =
2503 self.location_to_reg(sz_dst, dst, &mut temps, ImmType::None, false, None)?;
2504 match sz_src {
2505 Size::S8 => self.assembler.emit_sxtb(sz_dst, src, dest),
2506 Size::S16 => self.assembler.emit_sxth(sz_dst, src, dest),
2507 Size::S32 => self.assembler.emit_sxtw(sz_dst, src, dest),
2508 _ => codegen_error!("singlepass emit_relaxed_sign_extension unreachable"),
2509 }?;
2510 if dst != dest {
2511 self.move_location(sz_dst, dest, dst)?;
2512 }
2513 for r in temps {
2514 self.release_gpr(r);
2515 }
2516 Ok(())
2517 }
2518 }
2519 }
2520
2521 fn emit_binop_add32(
2522 &mut self,
2523 loc_a: Location,
2524 loc_b: Location,
2525 ret: Location,
2526 ) -> Result<(), CompileError> {
2527 self.emit_relaxed_binop3(
2528 Assembler::emit_add,
2529 Size::S32,
2530 loc_a,
2531 loc_b,
2532 ret,
2533 ImmType::Bits12,
2534 )
2535 }
2536 fn emit_binop_sub32(
2537 &mut self,
2538 loc_a: Location,
2539 loc_b: Location,
2540 ret: Location,
2541 ) -> Result<(), CompileError> {
2542 self.emit_relaxed_binop3(
2543 Assembler::emit_sub,
2544 Size::S32,
2545 loc_a,
2546 loc_b,
2547 ret,
2548 ImmType::Bits12,
2549 )
2550 }
2551 fn emit_binop_mul32(
2552 &mut self,
2553 loc_a: Location,
2554 loc_b: Location,
2555 ret: Location,
2556 ) -> Result<(), CompileError> {
2557 self.emit_relaxed_binop3(
2558 Assembler::emit_mul,
2559 Size::S32,
2560 loc_a,
2561 loc_b,
2562 ret,
2563 ImmType::None,
2564 )
2565 }
2566 fn emit_binop_udiv32(
2567 &mut self,
2568 loc_a: Location,
2569 loc_b: Location,
2570 ret: Location,
2571 integer_division_by_zero: Label,
2572 ) -> Result<usize, CompileError> {
2573 let mut temps = vec![];
2574 let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
2575 let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
2576 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
2577
2578 self.assembler
2579 .emit_cbz_label_far(Size::S32, src2, integer_division_by_zero)?;
2580 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
2581 self.assembler.emit_udiv(Size::S32, src1, src2, dest)?;
2582 if ret != dest {
2583 self.move_location(Size::S32, dest, ret)?;
2584 }
2585 for r in temps {
2586 self.release_gpr(r);
2587 }
2588 Ok(offset)
2589 }
2590 fn emit_binop_sdiv32(
2591 &mut self,
2592 loc_a: Location,
2593 loc_b: Location,
2594 ret: Location,
2595 integer_division_by_zero: Label,
2596 integer_overflow: Label,
2597 ) -> Result<usize, CompileError> {
2598 let mut temps = vec![];
2599 let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
2600 let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
2601 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
2602
2603 self.assembler
2604 .emit_cbz_label_far(Size::S32, src2, integer_division_by_zero)?;
2605 let label_nooverflow = self.assembler.get_label();
2606 let tmp = self.location_to_reg(
2607 Size::S32,
2608 Location::Imm32(0x80000000),
2609 &mut temps,
2610 ImmType::None,
2611 true,
2612 None,
2613 )?;
2614 self.assembler.emit_cmp(Size::S32, tmp, src1)?;
2615 self.assembler
2616 .emit_bcond_label(Condition::Ne, label_nooverflow)?;
2617 self.assembler.emit_movn(Size::S32, tmp, 0)?;
2618 self.assembler.emit_cmp(Size::S32, tmp, src2)?;
2619 self.assembler
2620 .emit_bcond_label_far(Condition::Eq, integer_overflow)?;
2621 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
2622 self.assembler.emit_label(label_nooverflow)?;
2623 self.assembler.emit_sdiv(Size::S32, src1, src2, dest)?;
2624 if ret != dest {
2625 self.move_location(Size::S32, dest, ret)?;
2626 }
2627 for r in temps {
2628 self.release_gpr(r);
2629 }
2630 Ok(offset)
2631 }
2632 fn emit_binop_urem32(
2633 &mut self,
2634 loc_a: Location,
2635 loc_b: Location,
2636 ret: Location,
2637 integer_division_by_zero: Label,
2638 ) -> Result<usize, CompileError> {
2639 let mut temps = vec![];
2640 let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
2641 let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
2642 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
2643 let dest = if dest == src1 || dest == src2 {
2644 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2645 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2646 })?;
2647 temps.push(tmp);
2648 self.assembler
2649 .emit_mov(Size::S32, dest, Location::GPR(tmp))?;
2650 Location::GPR(tmp)
2651 } else {
2652 dest
2653 };
2654 self.assembler
2655 .emit_cbz_label_far(Size::S32, src2, integer_division_by_zero)?;
2656 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
2657 self.assembler.emit_udiv(Size::S32, src1, src2, dest)?;
2658 self.assembler
2660 .emit_msub(Size::S32, dest, src2, src1, dest)?;
2661 if ret != dest {
2662 self.move_location(Size::S32, dest, ret)?;
2663 }
2664 for r in temps {
2665 self.release_gpr(r);
2666 }
2667 Ok(offset)
2668 }
2669 fn emit_binop_srem32(
2670 &mut self,
2671 loc_a: Location,
2672 loc_b: Location,
2673 ret: Location,
2674 integer_division_by_zero: Label,
2675 ) -> Result<usize, CompileError> {
2676 let mut temps = vec![];
2677 let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
2678 let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
2679 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
2680 let dest = if dest == src1 || dest == src2 {
2681 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2682 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2683 })?;
2684 temps.push(tmp);
2685 self.assembler
2686 .emit_mov(Size::S32, dest, Location::GPR(tmp))?;
2687 Location::GPR(tmp)
2688 } else {
2689 dest
2690 };
2691 self.assembler
2692 .emit_cbz_label_far(Size::S32, src2, integer_division_by_zero)?;
2693 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
2694 self.assembler.emit_sdiv(Size::S32, src1, src2, dest)?;
2695 self.assembler
2697 .emit_msub(Size::S32, dest, src2, src1, dest)?;
2698 if ret != dest {
2699 self.move_location(Size::S32, dest, ret)?;
2700 }
2701 for r in temps {
2702 self.release_gpr(r);
2703 }
2704 Ok(offset)
2705 }
2706 fn emit_binop_and32(
2707 &mut self,
2708 loc_a: Location,
2709 loc_b: Location,
2710 ret: Location,
2711 ) -> Result<(), CompileError> {
2712 self.emit_relaxed_binop3(
2713 Assembler::emit_and,
2714 Size::S32,
2715 loc_a,
2716 loc_b,
2717 ret,
2718 ImmType::Logical32,
2719 )
2720 }
2721 fn emit_binop_or32(
2722 &mut self,
2723 loc_a: Location,
2724 loc_b: Location,
2725 ret: Location,
2726 ) -> Result<(), CompileError> {
2727 self.emit_relaxed_binop3(
2728 Assembler::emit_or,
2729 Size::S32,
2730 loc_a,
2731 loc_b,
2732 ret,
2733 ImmType::Logical32,
2734 )
2735 }
2736 fn emit_binop_xor32(
2737 &mut self,
2738 loc_a: Location,
2739 loc_b: Location,
2740 ret: Location,
2741 ) -> Result<(), CompileError> {
2742 self.emit_relaxed_binop3(
2743 Assembler::emit_eor,
2744 Size::S32,
2745 loc_a,
2746 loc_b,
2747 ret,
2748 ImmType::Logical32,
2749 )
2750 }
2751 fn i32_cmp_ge_s(
2752 &mut self,
2753 loc_a: Location,
2754 loc_b: Location,
2755 ret: Location,
2756 ) -> Result<(), CompileError> {
2757 self.emit_cmpop_i32_dynamic_b(Condition::Ge, loc_a, loc_b, ret)
2758 }
2759 fn i32_cmp_gt_s(
2760 &mut self,
2761 loc_a: Location,
2762 loc_b: Location,
2763 ret: Location,
2764 ) -> Result<(), CompileError> {
2765 self.emit_cmpop_i32_dynamic_b(Condition::Gt, loc_a, loc_b, ret)
2766 }
2767 fn i32_cmp_le_s(
2768 &mut self,
2769 loc_a: Location,
2770 loc_b: Location,
2771 ret: Location,
2772 ) -> Result<(), CompileError> {
2773 self.emit_cmpop_i32_dynamic_b(Condition::Le, loc_a, loc_b, ret)
2774 }
2775 fn i32_cmp_lt_s(
2776 &mut self,
2777 loc_a: Location,
2778 loc_b: Location,
2779 ret: Location,
2780 ) -> Result<(), CompileError> {
2781 self.emit_cmpop_i32_dynamic_b(Condition::Lt, loc_a, loc_b, ret)
2782 }
2783 fn i32_cmp_ge_u(
2784 &mut self,
2785 loc_a: Location,
2786 loc_b: Location,
2787 ret: Location,
2788 ) -> Result<(), CompileError> {
2789 self.emit_cmpop_i32_dynamic_b(Condition::Cs, loc_a, loc_b, ret)
2790 }
2791 fn i32_cmp_gt_u(
2792 &mut self,
2793 loc_a: Location,
2794 loc_b: Location,
2795 ret: Location,
2796 ) -> Result<(), CompileError> {
2797 self.emit_cmpop_i32_dynamic_b(Condition::Hi, loc_a, loc_b, ret)
2798 }
2799 fn i32_cmp_le_u(
2800 &mut self,
2801 loc_a: Location,
2802 loc_b: Location,
2803 ret: Location,
2804 ) -> Result<(), CompileError> {
2805 self.emit_cmpop_i32_dynamic_b(Condition::Ls, loc_a, loc_b, ret)
2806 }
2807 fn i32_cmp_lt_u(
2808 &mut self,
2809 loc_a: Location,
2810 loc_b: Location,
2811 ret: Location,
2812 ) -> Result<(), CompileError> {
2813 self.emit_cmpop_i32_dynamic_b(Condition::Cc, loc_a, loc_b, ret)
2814 }
2815 fn i32_cmp_ne(
2816 &mut self,
2817 loc_a: Location,
2818 loc_b: Location,
2819 ret: Location,
2820 ) -> Result<(), CompileError> {
2821 self.emit_cmpop_i32_dynamic_b(Condition::Ne, loc_a, loc_b, ret)
2822 }
2823 fn i32_cmp_eq(
2824 &mut self,
2825 loc_a: Location,
2826 loc_b: Location,
2827 ret: Location,
2828 ) -> Result<(), CompileError> {
2829 self.emit_cmpop_i32_dynamic_b(Condition::Eq, loc_a, loc_b, ret)
2830 }
2831 fn i32_clz(&mut self, src: Location, dst: Location) -> Result<(), CompileError> {
2832 self.emit_relaxed_binop(Assembler::emit_clz, Size::S32, src, dst, true)
2833 }
2834 fn i32_ctz(&mut self, src: Location, dst: Location) -> Result<(), CompileError> {
2835 let mut temps = vec![];
2836 let src = self.location_to_reg(Size::S32, src, &mut temps, ImmType::None, true, None)?;
2837 let dest = self.location_to_reg(Size::S32, dst, &mut temps, ImmType::None, false, None)?;
2838 self.assembler.emit_rbit(Size::S32, src, dest)?;
2839 self.assembler.emit_clz(Size::S32, dest, dest)?;
2840 if dst != dest {
2841 self.move_location(Size::S32, dest, dst)?;
2842 }
2843 for r in temps {
2844 self.release_gpr(r);
2845 }
2846 Ok(())
2847 }
2848 fn i32_popcnt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
2849 if self.has_neon {
2850 let mut temps = vec![];
2851
2852 let src_gpr =
2853 self.location_to_reg(Size::S32, loc, &mut temps, ImmType::None, true, None)?;
2854 let dst_gpr =
2855 self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
2856
2857 let mut neon_temps = vec![];
2858 let neon_temp = self.acquire_temp_simd().ok_or_else(|| {
2859 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2860 })?;
2861 neon_temps.push(neon_temp);
2862
2863 self.assembler
2864 .emit_fmov(Size::S32, src_gpr, Size::S32, Location::SIMD(neon_temp))?;
2865 self.assembler.emit_cnt(neon_temp, neon_temp)?;
2866 self.assembler.emit_addv(neon_temp, neon_temp)?;
2867 self.assembler
2868 .emit_fmov(Size::S32, Location::SIMD(neon_temp), Size::S32, dst_gpr)?;
2869
2870 if ret != dst_gpr {
2871 self.move_location(Size::S32, dst_gpr, ret)?;
2872 }
2873
2874 for r in temps {
2875 self.release_gpr(r);
2876 }
2877
2878 for r in neon_temps {
2879 self.release_simd(r);
2880 }
2881 } else {
2882 let mut temps = vec![];
2883 let src =
2884 self.location_to_reg(Size::S32, loc, &mut temps, ImmType::None, true, None)?;
2885 let dest =
2886 self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
2887 let src = if src == loc {
2888 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2889 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2890 })?;
2891 temps.push(tmp);
2892 self.assembler
2893 .emit_mov(Size::S32, src, Location::GPR(tmp))?;
2894 Location::GPR(tmp)
2895 } else {
2896 src
2897 };
2898 let tmp = {
2899 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2900 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2901 })?;
2902 temps.push(tmp);
2903 Location::GPR(tmp)
2904 };
2905 let label_loop = self.assembler.get_label();
2906 let label_exit = self.assembler.get_label();
2907 self.assembler
2908 .emit_mov(Size::S32, Location::GPR(GPR::XzrSp), dest)?; self.assembler.emit_cbz_label(Size::S32, src, label_exit)?; self.assembler.emit_label(label_loop)?; self.assembler
2912 .emit_add(Size::S32, dest, Location::Imm8(1), dest)?; self.assembler.emit_clz(Size::S32, src, tmp)?; self.assembler.emit_lsl(Size::S32, src, tmp, src)?; self.assembler
2916 .emit_lsl(Size::S32, src, Location::Imm8(1), src)?; self.assembler.emit_cbnz_label(Size::S32, src, label_loop)?; self.assembler.emit_label(label_exit)?;
2919 if ret != dest {
2920 self.move_location(Size::S32, dest, ret)?;
2921 }
2922 for r in temps {
2923 self.release_gpr(r);
2924 }
2925 }
2926 Ok(())
2927 }
2928 fn i32_shl(
2929 &mut self,
2930 loc_a: Location,
2931 loc_b: Location,
2932 ret: Location,
2933 ) -> Result<(), CompileError> {
2934 self.emit_relaxed_binop3(
2935 Assembler::emit_lsl,
2936 Size::S32,
2937 loc_a,
2938 loc_b,
2939 ret,
2940 ImmType::Shift32No0,
2941 )
2942 }
2943 fn i32_shr(
2944 &mut self,
2945 loc_a: Location,
2946 loc_b: Location,
2947 ret: Location,
2948 ) -> Result<(), CompileError> {
2949 self.emit_relaxed_binop3(
2950 Assembler::emit_lsr,
2951 Size::S32,
2952 loc_a,
2953 loc_b,
2954 ret,
2955 ImmType::Shift32No0,
2956 )
2957 }
2958 fn i32_sar(
2959 &mut self,
2960 loc_a: Location,
2961 loc_b: Location,
2962 ret: Location,
2963 ) -> Result<(), CompileError> {
2964 self.emit_relaxed_binop3(
2965 Assembler::emit_asr,
2966 Size::S32,
2967 loc_a,
2968 loc_b,
2969 ret,
2970 ImmType::Shift32No0,
2971 )
2972 }
2973 fn i32_rol(
2974 &mut self,
2975 loc_a: Location,
2976 loc_b: Location,
2977 ret: Location,
2978 ) -> Result<(), CompileError> {
2979 let mut temps = vec![];
2980 let src2 = match loc_b {
2981 Location::Imm8(imm) => Location::Imm8(32 - (imm & 31)),
2982 Location::Imm32(imm) => Location::Imm8(32 - (imm & 31) as u8),
2983 Location::Imm64(imm) => Location::Imm8(32 - (imm & 31) as u8),
2984 _ => {
2985 let tmp1 = self.location_to_reg(
2986 Size::S32,
2987 Location::Imm32(32),
2988 &mut temps,
2989 ImmType::None,
2990 true,
2991 None,
2992 )?;
2993 let tmp2 =
2994 self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
2995 self.assembler.emit_sub(Size::S32, tmp1, tmp2, tmp1)?;
2996 tmp1
2997 }
2998 };
2999 self.emit_relaxed_binop3(
3000 Assembler::emit_ror,
3001 Size::S32,
3002 loc_a,
3003 src2,
3004 ret,
3005 ImmType::Shift32No0,
3006 )?;
3007 for r in temps {
3008 self.release_gpr(r);
3009 }
3010 Ok(())
3011 }
3012 fn i32_ror(
3013 &mut self,
3014 loc_a: Location,
3015 loc_b: Location,
3016 ret: Location,
3017 ) -> Result<(), CompileError> {
3018 self.emit_relaxed_binop3(
3019 Assembler::emit_ror,
3020 Size::S32,
3021 loc_a,
3022 loc_b,
3023 ret,
3024 ImmType::Shift32No0,
3025 )
3026 }
3027 fn i32_load(
3028 &mut self,
3029 addr: Location,
3030 memarg: &MemArg,
3031 ret: Location,
3032 need_check: bool,
3033 imported_memories: bool,
3034 offset: i32,
3035 heap_access_oob: Label,
3036 unaligned_atomic: Label,
3037 ) -> Result<(), CompileError> {
3038 self.memory_op(
3039 addr,
3040 memarg,
3041 false,
3042 4,
3043 need_check,
3044 imported_memories,
3045 offset,
3046 heap_access_oob,
3047 unaligned_atomic,
3048 |this, addr| this.emit_relaxed_ldr32(Size::S32, ret, Location::Memory(addr, 0)),
3049 )
3050 }
3051 fn i32_load_8u(
3052 &mut self,
3053 addr: Location,
3054 memarg: &MemArg,
3055 ret: Location,
3056 need_check: bool,
3057 imported_memories: bool,
3058 offset: i32,
3059 heap_access_oob: Label,
3060 unaligned_atomic: Label,
3061 ) -> Result<(), CompileError> {
3062 self.memory_op(
3063 addr,
3064 memarg,
3065 false,
3066 1,
3067 need_check,
3068 imported_memories,
3069 offset,
3070 heap_access_oob,
3071 unaligned_atomic,
3072 |this, addr| this.emit_relaxed_ldr8(Size::S32, ret, Location::Memory(addr, 0)),
3073 )
3074 }
3075 fn i32_load_8s(
3076 &mut self,
3077 addr: Location,
3078 memarg: &MemArg,
3079 ret: Location,
3080 need_check: bool,
3081 imported_memories: bool,
3082 offset: i32,
3083 heap_access_oob: Label,
3084 unaligned_atomic: Label,
3085 ) -> Result<(), CompileError> {
3086 self.memory_op(
3087 addr,
3088 memarg,
3089 false,
3090 1,
3091 need_check,
3092 imported_memories,
3093 offset,
3094 heap_access_oob,
3095 unaligned_atomic,
3096 |this, addr| this.emit_relaxed_ldr8s(Size::S32, ret, Location::Memory(addr, 0)),
3097 )
3098 }
3099 fn i32_load_16u(
3100 &mut self,
3101 addr: Location,
3102 memarg: &MemArg,
3103 ret: Location,
3104 need_check: bool,
3105 imported_memories: bool,
3106 offset: i32,
3107 heap_access_oob: Label,
3108 unaligned_atomic: Label,
3109 ) -> Result<(), CompileError> {
3110 self.memory_op(
3111 addr,
3112 memarg,
3113 false,
3114 2,
3115 need_check,
3116 imported_memories,
3117 offset,
3118 heap_access_oob,
3119 unaligned_atomic,
3120 |this, addr| this.emit_relaxed_ldr16(Size::S32, ret, Location::Memory(addr, 0)),
3121 )
3122 }
3123 fn i32_load_16s(
3124 &mut self,
3125 addr: Location,
3126 memarg: &MemArg,
3127 ret: Location,
3128 need_check: bool,
3129 imported_memories: bool,
3130 offset: i32,
3131 heap_access_oob: Label,
3132 unaligned_atomic: Label,
3133 ) -> Result<(), CompileError> {
3134 self.memory_op(
3135 addr,
3136 memarg,
3137 false,
3138 2,
3139 need_check,
3140 imported_memories,
3141 offset,
3142 heap_access_oob,
3143 unaligned_atomic,
3144 |this, addr| this.emit_relaxed_ldr16s(Size::S32, ret, Location::Memory(addr, 0)),
3145 )
3146 }
3147 fn i32_atomic_load(
3148 &mut self,
3149 addr: Location,
3150 memarg: &MemArg,
3151 ret: Location,
3152 need_check: bool,
3153 imported_memories: bool,
3154 offset: i32,
3155 heap_access_oob: Label,
3156 unaligned_atomic: Label,
3157 ) -> Result<(), CompileError> {
3158 self.memory_op(
3159 addr,
3160 memarg,
3161 true,
3162 4,
3163 need_check,
3164 imported_memories,
3165 offset,
3166 heap_access_oob,
3167 unaligned_atomic,
3168 |this, addr| this.emit_relaxed_ldr32(Size::S32, ret, Location::Memory(addr, 0)),
3169 )
3170 }
3171 fn i32_atomic_load_8u(
3172 &mut self,
3173 addr: Location,
3174 memarg: &MemArg,
3175 ret: Location,
3176 need_check: bool,
3177 imported_memories: bool,
3178 offset: i32,
3179 heap_access_oob: Label,
3180 unaligned_atomic: Label,
3181 ) -> Result<(), CompileError> {
3182 self.memory_op(
3183 addr,
3184 memarg,
3185 true,
3186 1,
3187 need_check,
3188 imported_memories,
3189 offset,
3190 heap_access_oob,
3191 unaligned_atomic,
3192 |this, addr| this.emit_relaxed_ldr8(Size::S32, ret, Location::Memory(addr, 0)),
3193 )
3194 }
3195 fn i32_atomic_load_16u(
3196 &mut self,
3197 addr: Location,
3198 memarg: &MemArg,
3199 ret: Location,
3200 need_check: bool,
3201 imported_memories: bool,
3202 offset: i32,
3203 heap_access_oob: Label,
3204 unaligned_atomic: Label,
3205 ) -> Result<(), CompileError> {
3206 self.memory_op(
3207 addr,
3208 memarg,
3209 true,
3210 2,
3211 need_check,
3212 imported_memories,
3213 offset,
3214 heap_access_oob,
3215 unaligned_atomic,
3216 |this, addr| this.emit_relaxed_ldr16(Size::S32, ret, Location::Memory(addr, 0)),
3217 )
3218 }
3219 fn i32_save(
3220 &mut self,
3221 target_value: Location,
3222 memarg: &MemArg,
3223 target_addr: Location,
3224 need_check: bool,
3225 imported_memories: bool,
3226 offset: i32,
3227 heap_access_oob: Label,
3228 unaligned_atomic: Label,
3229 ) -> Result<(), CompileError> {
3230 self.memory_op(
3231 target_addr,
3232 memarg,
3233 false,
3234 4,
3235 need_check,
3236 imported_memories,
3237 offset,
3238 heap_access_oob,
3239 unaligned_atomic,
3240 |this, addr| this.emit_relaxed_str32(target_value, Location::Memory(addr, 0)),
3241 )
3242 }
3243 fn i32_save_8(
3244 &mut self,
3245 target_value: Location,
3246 memarg: &MemArg,
3247 target_addr: Location,
3248 need_check: bool,
3249 imported_memories: bool,
3250 offset: i32,
3251 heap_access_oob: Label,
3252 unaligned_atomic: Label,
3253 ) -> Result<(), CompileError> {
3254 self.memory_op(
3255 target_addr,
3256 memarg,
3257 false,
3258 4,
3259 need_check,
3260 imported_memories,
3261 offset,
3262 heap_access_oob,
3263 unaligned_atomic,
3264 |this, addr| this.emit_relaxed_str8(target_value, Location::Memory(addr, 0)),
3265 )
3266 }
3267 fn i32_save_16(
3268 &mut self,
3269 target_value: Location,
3270 memarg: &MemArg,
3271 target_addr: Location,
3272 need_check: bool,
3273 imported_memories: bool,
3274 offset: i32,
3275 heap_access_oob: Label,
3276 unaligned_atomic: Label,
3277 ) -> Result<(), CompileError> {
3278 self.memory_op(
3279 target_addr,
3280 memarg,
3281 false,
3282 4,
3283 need_check,
3284 imported_memories,
3285 offset,
3286 heap_access_oob,
3287 unaligned_atomic,
3288 |this, addr| this.emit_relaxed_str16(target_value, Location::Memory(addr, 0)),
3289 )
3290 }
3291 fn i32_atomic_save(
3292 &mut self,
3293 target_value: Location,
3294 memarg: &MemArg,
3295 target_addr: Location,
3296 need_check: bool,
3297 imported_memories: bool,
3298 offset: i32,
3299 heap_access_oob: Label,
3300 unaligned_atomic: Label,
3301 ) -> Result<(), CompileError> {
3302 self.memory_op(
3303 target_addr,
3304 memarg,
3305 true,
3306 4,
3307 need_check,
3308 imported_memories,
3309 offset,
3310 heap_access_oob,
3311 unaligned_atomic,
3312 |this, addr| this.emit_relaxed_str32(target_value, Location::Memory(addr, 0)),
3313 )?;
3314 self.assembler.emit_dmb()
3315 }
3316 fn i32_atomic_save_8(
3317 &mut self,
3318 target_value: Location,
3319 memarg: &MemArg,
3320 target_addr: Location,
3321 need_check: bool,
3322 imported_memories: bool,
3323 offset: i32,
3324 heap_access_oob: Label,
3325 unaligned_atomic: Label,
3326 ) -> Result<(), CompileError> {
3327 self.memory_op(
3328 target_addr,
3329 memarg,
3330 true,
3331 1,
3332 need_check,
3333 imported_memories,
3334 offset,
3335 heap_access_oob,
3336 unaligned_atomic,
3337 |this, addr| this.emit_relaxed_str8(target_value, Location::Memory(addr, 0)),
3338 )?;
3339 self.assembler.emit_dmb()
3340 }
3341 fn i32_atomic_save_16(
3342 &mut self,
3343 target_value: Location,
3344 memarg: &MemArg,
3345 target_addr: Location,
3346 need_check: bool,
3347 imported_memories: bool,
3348 offset: i32,
3349 heap_access_oob: Label,
3350 unaligned_atomic: Label,
3351 ) -> Result<(), CompileError> {
3352 self.memory_op(
3353 target_addr,
3354 memarg,
3355 true,
3356 2,
3357 need_check,
3358 imported_memories,
3359 offset,
3360 heap_access_oob,
3361 unaligned_atomic,
3362 |this, addr| this.emit_relaxed_str16(target_value, Location::Memory(addr, 0)),
3363 )?;
3364 self.assembler.emit_dmb()
3365 }
3366 fn i32_atomic_add(
3368 &mut self,
3369 loc: Location,
3370 target: Location,
3371 memarg: &MemArg,
3372 ret: Location,
3373 need_check: bool,
3374 imported_memories: bool,
3375 offset: i32,
3376 heap_access_oob: Label,
3377 unaligned_atomic: Label,
3378 ) -> Result<(), CompileError> {
3379 self.memory_op(
3380 target,
3381 memarg,
3382 true,
3383 4,
3384 need_check,
3385 imported_memories,
3386 offset,
3387 heap_access_oob,
3388 unaligned_atomic,
3389 |this, addr| {
3390 let mut temps = vec![];
3391 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3392 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3393 })?;
3394 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3395 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3396 })?;
3397 let dst =
3398 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3399 let reread = this.get_label();
3400
3401 this.emit_label(reread)?;
3402 this.assembler
3403 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
3404 this.emit_binop_add32(dst, loc, Location::GPR(tmp1))?;
3405 this.assembler.emit_stlxr(
3406 Size::S32,
3407 Location::GPR(tmp2),
3408 Location::GPR(tmp1),
3409 Location::GPR(addr),
3410 )?;
3411 this.assembler
3412 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3413 this.assembler.emit_dmb()?;
3414
3415 if dst != ret {
3416 this.move_location(Size::S32, ret, dst)?;
3417 }
3418 for r in temps {
3419 this.release_gpr(r);
3420 }
3421 this.release_gpr(tmp1);
3422 this.release_gpr(tmp2);
3423 Ok(())
3424 },
3425 )
3426 }
3427 fn i32_atomic_add_8u(
3429 &mut self,
3430 loc: Location,
3431 target: Location,
3432 memarg: &MemArg,
3433 ret: Location,
3434 need_check: bool,
3435 imported_memories: bool,
3436 offset: i32,
3437 heap_access_oob: Label,
3438 unaligned_atomic: Label,
3439 ) -> Result<(), CompileError> {
3440 self.memory_op(
3441 target,
3442 memarg,
3443 true,
3444 1,
3445 need_check,
3446 imported_memories,
3447 offset,
3448 heap_access_oob,
3449 unaligned_atomic,
3450 |this, addr| {
3451 let mut temps = vec![];
3452 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3453 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3454 })?;
3455 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3456 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3457 })?;
3458 let dst =
3459 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3460 let reread = this.get_label();
3461
3462 this.emit_label(reread)?;
3463 this.assembler
3464 .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?;
3465 this.emit_binop_add32(dst, loc, Location::GPR(tmp1))?;
3466 this.assembler.emit_stlxrb(
3467 Size::S32,
3468 Location::GPR(tmp2),
3469 Location::GPR(tmp1),
3470 Location::GPR(addr),
3471 )?;
3472 this.assembler
3473 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3474 this.assembler.emit_dmb()?;
3475
3476 if dst != ret {
3477 this.move_location(Size::S32, ret, dst)?;
3478 }
3479 for r in temps {
3480 this.release_gpr(r);
3481 }
3482 this.release_gpr(tmp1);
3483 this.release_gpr(tmp2);
3484 Ok(())
3485 },
3486 )
3487 }
3488 fn i32_atomic_add_16u(
3490 &mut self,
3491 loc: Location,
3492 target: Location,
3493 memarg: &MemArg,
3494 ret: Location,
3495 need_check: bool,
3496 imported_memories: bool,
3497 offset: i32,
3498 heap_access_oob: Label,
3499 unaligned_atomic: Label,
3500 ) -> Result<(), CompileError> {
3501 self.memory_op(
3502 target,
3503 memarg,
3504 true,
3505 2,
3506 need_check,
3507 imported_memories,
3508 offset,
3509 heap_access_oob,
3510 unaligned_atomic,
3511 |this, addr| {
3512 let mut temps = vec![];
3513 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3514 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3515 })?;
3516 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3517 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3518 })?;
3519 let dst =
3520 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3521 let reread = this.get_label();
3522
3523 this.emit_label(reread)?;
3524 this.assembler
3525 .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?;
3526 this.emit_binop_add32(dst, loc, Location::GPR(tmp1))?;
3527 this.assembler.emit_stlxrh(
3528 Size::S32,
3529 Location::GPR(tmp2),
3530 Location::GPR(tmp1),
3531 Location::GPR(addr),
3532 )?;
3533 this.assembler
3534 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3535 this.assembler.emit_dmb()?;
3536
3537 if dst != ret {
3538 this.move_location(Size::S32, ret, dst)?;
3539 }
3540 for r in temps {
3541 this.release_gpr(r);
3542 }
3543 this.release_gpr(tmp1);
3544 this.release_gpr(tmp2);
3545 Ok(())
3546 },
3547 )
3548 }
3549 fn i32_atomic_sub(
3551 &mut self,
3552 loc: Location,
3553 target: Location,
3554 memarg: &MemArg,
3555 ret: Location,
3556 need_check: bool,
3557 imported_memories: bool,
3558 offset: i32,
3559 heap_access_oob: Label,
3560 unaligned_atomic: Label,
3561 ) -> Result<(), CompileError> {
3562 self.memory_op(
3563 target,
3564 memarg,
3565 true,
3566 4,
3567 need_check,
3568 imported_memories,
3569 offset,
3570 heap_access_oob,
3571 unaligned_atomic,
3572 |this, addr| {
3573 let mut temps = vec![];
3574 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3575 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3576 })?;
3577 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3578 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3579 })?;
3580 let dst =
3581 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3582 let reread = this.get_label();
3583
3584 this.emit_label(reread)?;
3585 this.assembler
3586 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
3587 this.emit_binop_sub32(dst, loc, Location::GPR(tmp1))?;
3588 this.assembler.emit_stlxr(
3589 Size::S32,
3590 Location::GPR(tmp2),
3591 Location::GPR(tmp1),
3592 Location::GPR(addr),
3593 )?;
3594 this.assembler
3595 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3596 this.assembler.emit_dmb()?;
3597
3598 if dst != ret {
3599 this.move_location(Size::S32, ret, dst)?;
3600 }
3601 for r in temps {
3602 this.release_gpr(r);
3603 }
3604 this.release_gpr(tmp1);
3605 this.release_gpr(tmp2);
3606 Ok(())
3607 },
3608 )
3609 }
3610 fn i32_atomic_sub_8u(
3612 &mut self,
3613 loc: Location,
3614 target: Location,
3615 memarg: &MemArg,
3616 ret: Location,
3617 need_check: bool,
3618 imported_memories: bool,
3619 offset: i32,
3620 heap_access_oob: Label,
3621 unaligned_atomic: Label,
3622 ) -> Result<(), CompileError> {
3623 self.memory_op(
3624 target,
3625 memarg,
3626 true,
3627 1,
3628 need_check,
3629 imported_memories,
3630 offset,
3631 heap_access_oob,
3632 unaligned_atomic,
3633 |this, addr| {
3634 let mut temps = vec![];
3635 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3636 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3637 })?;
3638 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3639 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3640 })?;
3641 let dst =
3642 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3643 let reread = this.get_label();
3644
3645 this.emit_label(reread)?;
3646 this.assembler
3647 .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?;
3648 this.emit_binop_sub32(dst, loc, Location::GPR(tmp1))?;
3649 this.assembler.emit_stlxrb(
3650 Size::S32,
3651 Location::GPR(tmp2),
3652 Location::GPR(tmp1),
3653 Location::GPR(addr),
3654 )?;
3655 this.assembler
3656 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3657 this.assembler.emit_dmb()?;
3658
3659 if dst != ret {
3660 this.move_location(Size::S32, ret, dst)?;
3661 }
3662 for r in temps {
3663 this.release_gpr(r);
3664 }
3665 this.release_gpr(tmp1);
3666 this.release_gpr(tmp2);
3667 Ok(())
3668 },
3669 )
3670 }
3671 fn i32_atomic_sub_16u(
3673 &mut self,
3674 loc: Location,
3675 target: Location,
3676 memarg: &MemArg,
3677 ret: Location,
3678 need_check: bool,
3679 imported_memories: bool,
3680 offset: i32,
3681 heap_access_oob: Label,
3682 unaligned_atomic: Label,
3683 ) -> Result<(), CompileError> {
3684 self.memory_op(
3685 target,
3686 memarg,
3687 true,
3688 2,
3689 need_check,
3690 imported_memories,
3691 offset,
3692 heap_access_oob,
3693 unaligned_atomic,
3694 |this, addr| {
3695 let mut temps = vec![];
3696 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3697 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3698 })?;
3699 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3700 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3701 })?;
3702 let dst =
3703 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3704 let reread = this.get_label();
3705
3706 this.emit_label(reread)?;
3707 this.assembler
3708 .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?;
3709 this.emit_binop_sub32(dst, loc, Location::GPR(tmp1))?;
3710 this.assembler.emit_stlxrh(
3711 Size::S32,
3712 Location::GPR(tmp2),
3713 Location::GPR(tmp1),
3714 Location::GPR(addr),
3715 )?;
3716 this.assembler
3717 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3718 this.assembler.emit_dmb()?;
3719
3720 if dst != ret {
3721 this.move_location(Size::S32, ret, dst)?;
3722 }
3723 for r in temps {
3724 this.release_gpr(r);
3725 }
3726 this.release_gpr(tmp1);
3727 this.release_gpr(tmp2);
3728 Ok(())
3729 },
3730 )
3731 }
3732 fn i32_atomic_and(
3734 &mut self,
3735 loc: Location,
3736 target: Location,
3737 memarg: &MemArg,
3738 ret: Location,
3739 need_check: bool,
3740 imported_memories: bool,
3741 offset: i32,
3742 heap_access_oob: Label,
3743 unaligned_atomic: Label,
3744 ) -> Result<(), CompileError> {
3745 self.memory_op(
3746 target,
3747 memarg,
3748 true,
3749 4,
3750 need_check,
3751 imported_memories,
3752 offset,
3753 heap_access_oob,
3754 unaligned_atomic,
3755 |this, addr| {
3756 let mut temps = vec![];
3757 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3758 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3759 })?;
3760 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3761 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3762 })?;
3763 let dst =
3764 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3765 let reread = this.get_label();
3766
3767 this.emit_label(reread)?;
3768 this.assembler
3769 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
3770 this.emit_binop_and32(dst, loc, Location::GPR(tmp1))?;
3771 this.assembler.emit_stlxr(
3772 Size::S32,
3773 Location::GPR(tmp2),
3774 Location::GPR(tmp1),
3775 Location::GPR(addr),
3776 )?;
3777 this.assembler
3778 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3779 this.assembler.emit_dmb()?;
3780
3781 if dst != ret {
3782 this.move_location(Size::S32, ret, dst)?;
3783 }
3784 for r in temps {
3785 this.release_gpr(r);
3786 }
3787 this.release_gpr(tmp1);
3788 this.release_gpr(tmp2);
3789 Ok(())
3790 },
3791 )
3792 }
3793 fn i32_atomic_and_8u(
3795 &mut self,
3796 loc: Location,
3797 target: Location,
3798 memarg: &MemArg,
3799 ret: Location,
3800 need_check: bool,
3801 imported_memories: bool,
3802 offset: i32,
3803 heap_access_oob: Label,
3804 unaligned_atomic: Label,
3805 ) -> Result<(), CompileError> {
3806 self.memory_op(
3807 target,
3808 memarg,
3809 true,
3810 1,
3811 need_check,
3812 imported_memories,
3813 offset,
3814 heap_access_oob,
3815 unaligned_atomic,
3816 |this, addr| {
3817 let mut temps = vec![];
3818 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3819 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3820 })?;
3821 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3822 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3823 })?;
3824 let dst =
3825 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3826 let reread = this.get_label();
3827
3828 this.emit_label(reread)?;
3829 this.assembler
3830 .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?;
3831 this.emit_binop_and32(dst, loc, Location::GPR(tmp1))?;
3832 this.assembler.emit_stlxrb(
3833 Size::S32,
3834 Location::GPR(tmp2),
3835 Location::GPR(tmp1),
3836 Location::GPR(addr),
3837 )?;
3838 this.assembler
3839 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3840 this.assembler.emit_dmb()?;
3841
3842 if dst != ret {
3843 this.move_location(Size::S32, ret, dst)?;
3844 }
3845 for r in temps {
3846 this.release_gpr(r);
3847 }
3848 this.release_gpr(tmp1);
3849 this.release_gpr(tmp2);
3850 Ok(())
3851 },
3852 )
3853 }
3854 fn i32_atomic_and_16u(
3856 &mut self,
3857 loc: Location,
3858 target: Location,
3859 memarg: &MemArg,
3860 ret: Location,
3861 need_check: bool,
3862 imported_memories: bool,
3863 offset: i32,
3864 heap_access_oob: Label,
3865 unaligned_atomic: Label,
3866 ) -> Result<(), CompileError> {
3867 self.memory_op(
3868 target,
3869 memarg,
3870 true,
3871 2,
3872 need_check,
3873 imported_memories,
3874 offset,
3875 heap_access_oob,
3876 unaligned_atomic,
3877 |this, addr| {
3878 let mut temps = vec![];
3879 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3880 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3881 })?;
3882 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3883 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3884 })?;
3885 let dst =
3886 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3887 let reread = this.get_label();
3888
3889 this.emit_label(reread)?;
3890 this.assembler
3891 .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?;
3892 this.emit_binop_and32(dst, loc, Location::GPR(tmp1))?;
3893 this.assembler.emit_stlxrh(
3894 Size::S32,
3895 Location::GPR(tmp2),
3896 Location::GPR(tmp1),
3897 Location::GPR(addr),
3898 )?;
3899 this.assembler
3900 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3901 this.assembler.emit_dmb()?;
3902
3903 if dst != ret {
3904 this.move_location(Size::S32, ret, dst)?;
3905 }
3906 for r in temps {
3907 this.release_gpr(r);
3908 }
3909 this.release_gpr(tmp1);
3910 this.release_gpr(tmp2);
3911 Ok(())
3912 },
3913 )
3914 }
3915 fn i32_atomic_or(
3917 &mut self,
3918 loc: Location,
3919 target: Location,
3920 memarg: &MemArg,
3921 ret: Location,
3922 need_check: bool,
3923 imported_memories: bool,
3924 offset: i32,
3925 heap_access_oob: Label,
3926 unaligned_atomic: Label,
3927 ) -> Result<(), CompileError> {
3928 self.memory_op(
3929 target,
3930 memarg,
3931 true,
3932 4,
3933 need_check,
3934 imported_memories,
3935 offset,
3936 heap_access_oob,
3937 unaligned_atomic,
3938 |this, addr| {
3939 let mut temps = vec![];
3940 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3941 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3942 })?;
3943 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3944 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3945 })?;
3946 let dst =
3947 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3948 let reread = this.get_label();
3949
3950 this.emit_label(reread)?;
3951 this.assembler
3952 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
3953 this.emit_binop_or32(dst, loc, Location::GPR(tmp1))?;
3954 this.assembler.emit_stlxr(
3955 Size::S32,
3956 Location::GPR(tmp2),
3957 Location::GPR(tmp1),
3958 Location::GPR(addr),
3959 )?;
3960 this.assembler
3961 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3962 this.assembler.emit_dmb()?;
3963
3964 if dst != ret {
3965 this.move_location(Size::S32, ret, dst)?;
3966 }
3967 for r in temps {
3968 this.release_gpr(r);
3969 }
3970 this.release_gpr(tmp1);
3971 this.release_gpr(tmp2);
3972 Ok(())
3973 },
3974 )
3975 }
3976 fn i32_atomic_or_8u(
3978 &mut self,
3979 loc: Location,
3980 target: Location,
3981 memarg: &MemArg,
3982 ret: Location,
3983 need_check: bool,
3984 imported_memories: bool,
3985 offset: i32,
3986 heap_access_oob: Label,
3987 unaligned_atomic: Label,
3988 ) -> Result<(), CompileError> {
3989 self.memory_op(
3990 target,
3991 memarg,
3992 true,
3993 1,
3994 need_check,
3995 imported_memories,
3996 offset,
3997 heap_access_oob,
3998 unaligned_atomic,
3999 |this, addr| {
4000 let mut temps = vec![];
4001 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
4002 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4003 })?;
4004 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
4005 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4006 })?;
4007 let dst =
4008 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4009 let reread = this.get_label();
4010
4011 this.emit_label(reread)?;
4012 this.assembler
4013 .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?;
4014 this.emit_binop_or32(dst, loc, Location::GPR(tmp1))?;
4015 this.assembler.emit_stlxrb(
4016 Size::S32,
4017 Location::GPR(tmp2),
4018 Location::GPR(tmp1),
4019 Location::GPR(addr),
4020 )?;
4021 this.assembler
4022 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
4023 this.assembler.emit_dmb()?;
4024
4025 if dst != ret {
4026 this.move_location(Size::S32, ret, dst)?;
4027 }
4028 for r in temps {
4029 this.release_gpr(r);
4030 }
4031 this.release_gpr(tmp1);
4032 this.release_gpr(tmp2);
4033 Ok(())
4034 },
4035 )
4036 }
4037 fn i32_atomic_or_16u(
4039 &mut self,
4040 loc: Location,
4041 target: Location,
4042 memarg: &MemArg,
4043 ret: Location,
4044 need_check: bool,
4045 imported_memories: bool,
4046 offset: i32,
4047 heap_access_oob: Label,
4048 unaligned_atomic: Label,
4049 ) -> Result<(), CompileError> {
4050 self.memory_op(
4051 target,
4052 memarg,
4053 true,
4054 2,
4055 need_check,
4056 imported_memories,
4057 offset,
4058 heap_access_oob,
4059 unaligned_atomic,
4060 |this, addr| {
4061 let mut temps = vec![];
4062 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
4063 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4064 })?;
4065 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
4066 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4067 })?;
4068 let dst =
4069 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4070 let reread = this.get_label();
4071
4072 this.emit_label(reread)?;
4073 this.assembler
4074 .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?;
4075 this.emit_binop_or32(dst, loc, Location::GPR(tmp1))?;
4076 this.assembler.emit_stlxrh(
4077 Size::S32,
4078 Location::GPR(tmp2),
4079 Location::GPR(tmp1),
4080 Location::GPR(addr),
4081 )?;
4082 this.assembler
4083 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
4084 this.assembler.emit_dmb()?;
4085
4086 if dst != ret {
4087 this.move_location(Size::S32, ret, dst)?;
4088 }
4089 for r in temps {
4090 this.release_gpr(r);
4091 }
4092 this.release_gpr(tmp1);
4093 this.release_gpr(tmp2);
4094 Ok(())
4095 },
4096 )
4097 }
4098 fn i32_atomic_xor(
4100 &mut self,
4101 loc: Location,
4102 target: Location,
4103 memarg: &MemArg,
4104 ret: Location,
4105 need_check: bool,
4106 imported_memories: bool,
4107 offset: i32,
4108 heap_access_oob: Label,
4109 unaligned_atomic: Label,
4110 ) -> Result<(), CompileError> {
4111 self.memory_op(
4112 target,
4113 memarg,
4114 true,
4115 4,
4116 need_check,
4117 imported_memories,
4118 offset,
4119 heap_access_oob,
4120 unaligned_atomic,
4121 |this, addr| {
4122 let mut temps = vec![];
4123 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
4124 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4125 })?;
4126 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
4127 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4128 })?;
4129 let dst =
4130 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4131 let reread = this.get_label();
4132
4133 this.emit_label(reread)?;
4134 this.assembler
4135 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
4136 this.emit_binop_xor32(dst, loc, Location::GPR(tmp1))?;
4137 this.assembler.emit_stlxr(
4138 Size::S32,
4139 Location::GPR(tmp2),
4140 Location::GPR(tmp1),
4141 Location::GPR(addr),
4142 )?;
4143 this.assembler
4144 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
4145 this.assembler.emit_dmb()?;
4146
4147 if dst != ret {
4148 this.move_location(Size::S32, ret, dst)?;
4149 }
4150 for r in temps {
4151 this.release_gpr(r);
4152 }
4153 this.release_gpr(tmp1);
4154 this.release_gpr(tmp2);
4155 Ok(())
4156 },
4157 )
4158 }
4159 fn i32_atomic_xor_8u(
4161 &mut self,
4162 loc: Location,
4163 target: Location,
4164 memarg: &MemArg,
4165 ret: Location,
4166 need_check: bool,
4167 imported_memories: bool,
4168 offset: i32,
4169 heap_access_oob: Label,
4170 unaligned_atomic: Label,
4171 ) -> Result<(), CompileError> {
4172 self.memory_op(
4173 target,
4174 memarg,
4175 true,
4176 1,
4177 need_check,
4178 imported_memories,
4179 offset,
4180 heap_access_oob,
4181 unaligned_atomic,
4182 |this, addr| {
4183 let mut temps = vec![];
4184 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
4185 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4186 })?;
4187 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
4188 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4189 })?;
4190 let dst =
4191 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4192 let reread = this.get_label();
4193
4194 this.emit_label(reread)?;
4195 this.assembler
4196 .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?;
4197 this.emit_binop_xor32(dst, loc, Location::GPR(tmp1))?;
4198 this.assembler.emit_stlxrb(
4199 Size::S32,
4200 Location::GPR(tmp2),
4201 Location::GPR(tmp1),
4202 Location::GPR(addr),
4203 )?;
4204 this.assembler
4205 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
4206 this.assembler.emit_dmb()?;
4207
4208 if dst != ret {
4209 this.move_location(Size::S32, ret, dst)?;
4210 }
4211 for r in temps {
4212 this.release_gpr(r);
4213 }
4214 this.release_gpr(tmp1);
4215 this.release_gpr(tmp2);
4216 Ok(())
4217 },
4218 )
4219 }
4220 fn i32_atomic_xor_16u(
4222 &mut self,
4223 loc: Location,
4224 target: Location,
4225 memarg: &MemArg,
4226 ret: Location,
4227 need_check: bool,
4228 imported_memories: bool,
4229 offset: i32,
4230 heap_access_oob: Label,
4231 unaligned_atomic: Label,
4232 ) -> Result<(), CompileError> {
4233 self.memory_op(
4234 target,
4235 memarg,
4236 true,
4237 2,
4238 need_check,
4239 imported_memories,
4240 offset,
4241 heap_access_oob,
4242 unaligned_atomic,
4243 |this, addr| {
4244 let mut temps = vec![];
4245 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
4246 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4247 })?;
4248 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
4249 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4250 })?;
4251 let dst =
4252 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4253 let reread = this.get_label();
4254
4255 this.emit_label(reread)?;
4256 this.assembler
4257 .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?;
4258 this.emit_binop_xor32(dst, loc, Location::GPR(tmp1))?;
4259 this.assembler.emit_stlxrh(
4260 Size::S32,
4261 Location::GPR(tmp2),
4262 Location::GPR(tmp1),
4263 Location::GPR(addr),
4264 )?;
4265 this.assembler
4266 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
4267 this.assembler.emit_dmb()?;
4268
4269 if dst != ret {
4270 this.move_location(Size::S32, ret, dst)?;
4271 }
4272 for r in temps {
4273 this.release_gpr(r);
4274 }
4275 this.release_gpr(tmp1);
4276 this.release_gpr(tmp2);
4277 Ok(())
4278 },
4279 )
4280 }
4281 fn i32_atomic_xchg(
4283 &mut self,
4284 loc: Location,
4285 target: Location,
4286 memarg: &MemArg,
4287 ret: Location,
4288 need_check: bool,
4289 imported_memories: bool,
4290 offset: i32,
4291 heap_access_oob: Label,
4292 unaligned_atomic: Label,
4293 ) -> Result<(), CompileError> {
4294 self.memory_op(
4295 target,
4296 memarg,
4297 true,
4298 4,
4299 need_check,
4300 imported_memories,
4301 offset,
4302 heap_access_oob,
4303 unaligned_atomic,
4304 |this, addr| {
4305 let mut temps = vec![];
4306 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
4307 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4308 })?;
4309 let dst =
4310 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4311 let org =
4312 this.location_to_reg(Size::S32, loc, &mut temps, ImmType::None, false, None)?;
4313 let reread = this.get_label();
4314
4315 this.emit_label(reread)?;
4316 this.assembler
4317 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
4318 this.assembler.emit_stlxr(
4319 Size::S32,
4320 Location::GPR(tmp),
4321 org,
4322 Location::GPR(addr),
4323 )?;
4324 this.assembler
4325 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
4326 this.assembler.emit_dmb()?;
4327
4328 if dst != ret {
4329 this.move_location(Size::S32, ret, dst)?;
4330 }
4331 for r in temps {
4332 this.release_gpr(r);
4333 }
4334 this.release_gpr(tmp);
4335 Ok(())
4336 },
4337 )
4338 }
4339 fn i32_atomic_xchg_8u(
4341 &mut self,
4342 loc: Location,
4343 target: Location,
4344 memarg: &MemArg,
4345 ret: Location,
4346 need_check: bool,
4347 imported_memories: bool,
4348 offset: i32,
4349 heap_access_oob: Label,
4350 unaligned_atomic: Label,
4351 ) -> Result<(), CompileError> {
4352 self.memory_op(
4353 target,
4354 memarg,
4355 true,
4356 1,
4357 need_check,
4358 imported_memories,
4359 offset,
4360 heap_access_oob,
4361 unaligned_atomic,
4362 |this, addr| {
4363 let mut temps = vec![];
4364 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
4365 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4366 })?;
4367 let dst =
4368 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4369 let org =
4370 this.location_to_reg(Size::S32, loc, &mut temps, ImmType::None, false, None)?;
4371 let reread = this.get_label();
4372
4373 this.emit_label(reread)?;
4374 this.assembler
4375 .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?;
4376 this.assembler.emit_stlxrb(
4377 Size::S32,
4378 Location::GPR(tmp),
4379 org,
4380 Location::GPR(addr),
4381 )?;
4382 this.assembler
4383 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
4384 this.assembler.emit_dmb()?;
4385
4386 if dst != ret {
4387 this.move_location(Size::S32, ret, dst)?;
4388 }
4389 for r in temps {
4390 this.release_gpr(r);
4391 }
4392 this.release_gpr(tmp);
4393 Ok(())
4394 },
4395 )
4396 }
4397 fn i32_atomic_xchg_16u(
4399 &mut self,
4400 loc: Location,
4401 target: Location,
4402 memarg: &MemArg,
4403 ret: Location,
4404 need_check: bool,
4405 imported_memories: bool,
4406 offset: i32,
4407 heap_access_oob: Label,
4408 unaligned_atomic: Label,
4409 ) -> Result<(), CompileError> {
4410 self.memory_op(
4411 target,
4412 memarg,
4413 true,
4414 2,
4415 need_check,
4416 imported_memories,
4417 offset,
4418 heap_access_oob,
4419 unaligned_atomic,
4420 |this, addr| {
4421 let mut temps = vec![];
4422 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
4423 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4424 })?;
4425 let dst =
4426 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4427 let org =
4428 this.location_to_reg(Size::S32, loc, &mut temps, ImmType::None, false, None)?;
4429 let reread = this.get_label();
4430
4431 this.emit_label(reread)?;
4432 this.assembler
4433 .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?;
4434 this.assembler.emit_stlxrh(
4435 Size::S32,
4436 Location::GPR(tmp),
4437 org,
4438 Location::GPR(addr),
4439 )?;
4440 this.assembler
4441 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
4442 this.assembler.emit_dmb()?;
4443
4444 if dst != ret {
4445 this.move_location(Size::S32, ret, dst)?;
4446 }
4447 for r in temps {
4448 this.release_gpr(r);
4449 }
4450 this.release_gpr(tmp);
4451 Ok(())
4452 },
4453 )
4454 }
4455 fn i32_atomic_cmpxchg(
4457 &mut self,
4458 new: Location,
4459 cmp: Location,
4460 target: Location,
4461 memarg: &MemArg,
4462 ret: Location,
4463 need_check: bool,
4464 imported_memories: bool,
4465 offset: i32,
4466 heap_access_oob: Label,
4467 unaligned_atomic: Label,
4468 ) -> Result<(), CompileError> {
4469 self.memory_op(
4470 target,
4471 memarg,
4472 true,
4473 4,
4474 need_check,
4475 imported_memories,
4476 offset,
4477 heap_access_oob,
4478 unaligned_atomic,
4479 |this, addr| {
4480 let mut temps = vec![];
4481 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
4482 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4483 })?;
4484 let dst =
4485 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4486 let org =
4487 this.location_to_reg(Size::S32, new, &mut temps, ImmType::None, false, None)?;
4488 let reread = this.get_label();
4489 let nosame = this.get_label();
4490
4491 this.emit_label(reread)?;
4492 this.assembler
4493 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
4494 this.emit_relaxed_cmp(Size::S32, dst, cmp)?;
4495 this.assembler.emit_bcond_label(Condition::Ne, nosame)?;
4496 this.assembler.emit_stlxr(
4497 Size::S32,
4498 Location::GPR(tmp),
4499 org,
4500 Location::GPR(addr),
4501 )?;
4502 this.assembler
4503 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
4504 this.assembler.emit_dmb()?;
4505
4506 this.emit_label(nosame)?;
4507 if dst != ret {
4508 this.move_location(Size::S32, ret, dst)?;
4509 }
4510 for r in temps {
4511 this.release_gpr(r);
4512 }
4513 this.release_gpr(tmp);
4514 Ok(())
4515 },
4516 )
4517 }
4518 fn i32_atomic_cmpxchg_8u(
4520 &mut self,
4521 new: Location,
4522 cmp: Location,
4523 target: Location,
4524 memarg: &MemArg,
4525 ret: Location,
4526 need_check: bool,
4527 imported_memories: bool,
4528 offset: i32,
4529 heap_access_oob: Label,
4530 unaligned_atomic: Label,
4531 ) -> Result<(), CompileError> {
4532 self.memory_op(
4533 target,
4534 memarg,
4535 true,
4536 1,
4537 need_check,
4538 imported_memories,
4539 offset,
4540 heap_access_oob,
4541 unaligned_atomic,
4542 |this, addr| {
4543 let mut temps = vec![];
4544 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
4545 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4546 })?;
4547 let dst =
4548 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4549 let org =
4550 this.location_to_reg(Size::S32, new, &mut temps, ImmType::None, false, None)?;
4551 let reread = this.get_label();
4552 let nosame = this.get_label();
4553
4554 this.emit_label(reread)?;
4555 this.assembler
4556 .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?;
4557 this.emit_relaxed_cmp(Size::S32, dst, cmp)?;
4558 this.assembler.emit_bcond_label(Condition::Ne, nosame)?;
4559 this.assembler.emit_stlxrb(
4560 Size::S32,
4561 Location::GPR(tmp),
4562 org,
4563 Location::GPR(addr),
4564 )?;
4565 this.assembler
4566 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
4567 this.assembler.emit_dmb()?;
4568
4569 this.emit_label(nosame)?;
4570 if dst != ret {
4571 this.move_location(Size::S32, ret, dst)?;
4572 }
4573 for r in temps {
4574 this.release_gpr(r);
4575 }
4576 this.release_gpr(tmp);
4577 Ok(())
4578 },
4579 )
4580 }
4581 fn i32_atomic_cmpxchg_16u(
4583 &mut self,
4584 new: Location,
4585 cmp: Location,
4586 target: Location,
4587 memarg: &MemArg,
4588 ret: Location,
4589 need_check: bool,
4590 imported_memories: bool,
4591 offset: i32,
4592 heap_access_oob: Label,
4593 unaligned_atomic: Label,
4594 ) -> Result<(), CompileError> {
4595 self.memory_op(
4596 target,
4597 memarg,
4598 true,
4599 2,
4600 need_check,
4601 imported_memories,
4602 offset,
4603 heap_access_oob,
4604 unaligned_atomic,
4605 |this, addr| {
4606 let mut temps = vec![];
4607 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
4608 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4609 })?;
4610 let dst =
4611 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4612 let org =
4613 this.location_to_reg(Size::S32, new, &mut temps, ImmType::None, false, None)?;
4614 let reread = this.get_label();
4615 let nosame = this.get_label();
4616
4617 this.emit_label(reread)?;
4618 this.assembler
4619 .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?;
4620 this.emit_relaxed_cmp(Size::S32, dst, cmp)?;
4621 this.assembler.emit_bcond_label(Condition::Ne, nosame)?;
4622 this.assembler.emit_stlxrh(
4623 Size::S32,
4624 Location::GPR(tmp),
4625 org,
4626 Location::GPR(addr),
4627 )?;
4628 this.assembler
4629 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
4630 this.assembler.emit_dmb()?;
4631
4632 this.emit_label(nosame)?;
4633 if dst != ret {
4634 this.move_location(Size::S32, ret, dst)?;
4635 }
4636 for r in temps {
4637 this.release_gpr(r);
4638 }
4639 this.release_gpr(tmp);
4640 Ok(())
4641 },
4642 )
4643 }
4644
4645 fn emit_call_with_reloc(
4646 &mut self,
4647 _calling_convention: CallingConvention,
4648 reloc_target: RelocationTarget,
4649 ) -> Result<Vec<Relocation>, CompileError> {
4650 let mut relocations = vec![];
4651 let next = self.get_label();
4652 let reloc_at = self.assembler.get_offset().0;
4653 self.emit_label(next)?; self.assembler.emit_call_label(next)?;
4655 relocations.push(Relocation {
4656 kind: RelocationKind::Arm64Call,
4657 reloc_target,
4658 offset: reloc_at as u32,
4659 addend: 0,
4660 });
4661 Ok(relocations)
4662 }
4663
4664 fn emit_binop_add64(
4665 &mut self,
4666 loc_a: Location,
4667 loc_b: Location,
4668 ret: Location,
4669 ) -> Result<(), CompileError> {
4670 self.emit_relaxed_binop3(
4671 Assembler::emit_add,
4672 Size::S64,
4673 loc_a,
4674 loc_b,
4675 ret,
4676 ImmType::Bits12,
4677 )
4678 }
4679 fn emit_binop_sub64(
4680 &mut self,
4681 loc_a: Location,
4682 loc_b: Location,
4683 ret: Location,
4684 ) -> Result<(), CompileError> {
4685 self.emit_relaxed_binop3(
4686 Assembler::emit_sub,
4687 Size::S64,
4688 loc_a,
4689 loc_b,
4690 ret,
4691 ImmType::Bits12,
4692 )
4693 }
4694 fn emit_binop_mul64(
4695 &mut self,
4696 loc_a: Location,
4697 loc_b: Location,
4698 ret: Location,
4699 ) -> Result<(), CompileError> {
4700 self.emit_relaxed_binop3(
4701 Assembler::emit_mul,
4702 Size::S64,
4703 loc_a,
4704 loc_b,
4705 ret,
4706 ImmType::None,
4707 )
4708 }
4709 fn emit_binop_udiv64(
4710 &mut self,
4711 loc_a: Location,
4712 loc_b: Location,
4713 ret: Location,
4714 integer_division_by_zero: Label,
4715 ) -> Result<usize, CompileError> {
4716 let mut temps = vec![];
4717 let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4718 let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4719 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4720
4721 self.assembler
4722 .emit_cbz_label(Size::S64, src2, integer_division_by_zero)?;
4723 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
4724 self.assembler.emit_udiv(Size::S64, src1, src2, dest)?;
4725 if ret != dest {
4726 self.move_location(Size::S64, dest, ret)?;
4727 }
4728 for r in temps {
4729 self.release_gpr(r);
4730 }
4731 Ok(offset)
4732 }
4733 fn emit_binop_sdiv64(
4734 &mut self,
4735 loc_a: Location,
4736 loc_b: Location,
4737 ret: Location,
4738 integer_division_by_zero: Label,
4739 integer_overflow: Label,
4740 ) -> Result<usize, CompileError> {
4741 let mut temps = vec![];
4742 let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4743 let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4744 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4745
4746 self.assembler
4747 .emit_cbz_label_far(Size::S64, src2, integer_division_by_zero)?;
4748 let label_nooverflow = self.assembler.get_label();
4749 let tmp = self.location_to_reg(
4750 Size::S64,
4751 Location::Imm64(0x8000000000000000),
4752 &mut temps,
4753 ImmType::None,
4754 true,
4755 None,
4756 )?;
4757 self.assembler.emit_cmp(Size::S64, tmp, src1)?;
4758 self.assembler
4759 .emit_bcond_label(Condition::Ne, label_nooverflow)?;
4760 self.assembler.emit_movn(Size::S64, tmp, 0)?;
4761 self.assembler.emit_cmp(Size::S64, tmp, src2)?;
4762 self.assembler
4763 .emit_bcond_label_far(Condition::Eq, integer_overflow)?;
4764 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
4765 self.assembler.emit_label(label_nooverflow)?;
4766 self.assembler.emit_sdiv(Size::S64, src1, src2, dest)?;
4767 if ret != dest {
4768 self.move_location(Size::S64, dest, ret)?;
4769 }
4770 for r in temps {
4771 self.release_gpr(r);
4772 }
4773 Ok(offset)
4774 }
4775 fn emit_binop_urem64(
4776 &mut self,
4777 loc_a: Location,
4778 loc_b: Location,
4779 ret: Location,
4780 integer_division_by_zero: Label,
4781 ) -> Result<usize, CompileError> {
4782 let mut temps = vec![];
4783 let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4784 let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4785 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4786 let dest = if dest == src1 || dest == src2 {
4787 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
4788 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4789 })?;
4790 temps.push(tmp);
4791 self.assembler
4792 .emit_mov(Size::S32, dest, Location::GPR(tmp))?;
4793 Location::GPR(tmp)
4794 } else {
4795 dest
4796 };
4797 self.assembler
4798 .emit_cbz_label_far(Size::S64, src2, integer_division_by_zero)?;
4799 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
4800 self.assembler.emit_udiv(Size::S64, src1, src2, dest)?;
4801 self.assembler
4803 .emit_msub(Size::S64, dest, src2, src1, dest)?;
4804 if ret != dest {
4805 self.move_location(Size::S64, dest, ret)?;
4806 }
4807 for r in temps {
4808 self.release_gpr(r);
4809 }
4810 Ok(offset)
4811 }
4812 fn emit_binop_srem64(
4813 &mut self,
4814 loc_a: Location,
4815 loc_b: Location,
4816 ret: Location,
4817 integer_division_by_zero: Label,
4818 ) -> Result<usize, CompileError> {
4819 let mut temps = vec![];
4820 let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4821 let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4822 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4823 let dest = if dest == src1 || dest == src2 {
4824 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
4825 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4826 })?;
4827 temps.push(tmp);
4828 self.assembler
4829 .emit_mov(Size::S64, dest, Location::GPR(tmp))?;
4830 Location::GPR(tmp)
4831 } else {
4832 dest
4833 };
4834 self.assembler
4835 .emit_cbz_label_far(Size::S64, src2, integer_division_by_zero)?;
4836 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
4837 self.assembler.emit_sdiv(Size::S64, src1, src2, dest)?;
4838 self.assembler
4840 .emit_msub(Size::S64, dest, src2, src1, dest)?;
4841 if ret != dest {
4842 self.move_location(Size::S64, dest, ret)?;
4843 }
4844 for r in temps {
4845 self.release_gpr(r);
4846 }
4847 Ok(offset)
4848 }
4849 fn emit_binop_and64(
4850 &mut self,
4851 loc_a: Location,
4852 loc_b: Location,
4853 ret: Location,
4854 ) -> Result<(), CompileError> {
4855 self.emit_relaxed_binop3(
4856 Assembler::emit_and,
4857 Size::S64,
4858 loc_a,
4859 loc_b,
4860 ret,
4861 ImmType::Logical64,
4862 )
4863 }
4864 fn emit_binop_or64(
4865 &mut self,
4866 loc_a: Location,
4867 loc_b: Location,
4868 ret: Location,
4869 ) -> Result<(), CompileError> {
4870 self.emit_relaxed_binop3(
4871 Assembler::emit_or,
4872 Size::S64,
4873 loc_a,
4874 loc_b,
4875 ret,
4876 ImmType::Logical64,
4877 )
4878 }
4879 fn emit_binop_xor64(
4880 &mut self,
4881 loc_a: Location,
4882 loc_b: Location,
4883 ret: Location,
4884 ) -> Result<(), CompileError> {
4885 self.emit_relaxed_binop3(
4886 Assembler::emit_eor,
4887 Size::S64,
4888 loc_a,
4889 loc_b,
4890 ret,
4891 ImmType::Logical64,
4892 )
4893 }
4894 fn i64_cmp_ge_s(
4895 &mut self,
4896 loc_a: Location,
4897 loc_b: Location,
4898 ret: Location,
4899 ) -> Result<(), CompileError> {
4900 self.emit_cmpop_i64_dynamic_b(Condition::Ge, loc_a, loc_b, ret)
4901 }
4902 fn i64_cmp_gt_s(
4903 &mut self,
4904 loc_a: Location,
4905 loc_b: Location,
4906 ret: Location,
4907 ) -> Result<(), CompileError> {
4908 self.emit_cmpop_i64_dynamic_b(Condition::Gt, loc_a, loc_b, ret)
4909 }
4910 fn i64_cmp_le_s(
4911 &mut self,
4912 loc_a: Location,
4913 loc_b: Location,
4914 ret: Location,
4915 ) -> Result<(), CompileError> {
4916 self.emit_cmpop_i64_dynamic_b(Condition::Le, loc_a, loc_b, ret)
4917 }
4918 fn i64_cmp_lt_s(
4919 &mut self,
4920 loc_a: Location,
4921 loc_b: Location,
4922 ret: Location,
4923 ) -> Result<(), CompileError> {
4924 self.emit_cmpop_i64_dynamic_b(Condition::Lt, loc_a, loc_b, ret)
4925 }
4926 fn i64_cmp_ge_u(
4927 &mut self,
4928 loc_a: Location,
4929 loc_b: Location,
4930 ret: Location,
4931 ) -> Result<(), CompileError> {
4932 self.emit_cmpop_i64_dynamic_b(Condition::Cs, loc_a, loc_b, ret)
4933 }
4934 fn i64_cmp_gt_u(
4935 &mut self,
4936 loc_a: Location,
4937 loc_b: Location,
4938 ret: Location,
4939 ) -> Result<(), CompileError> {
4940 self.emit_cmpop_i64_dynamic_b(Condition::Hi, loc_a, loc_b, ret)
4941 }
4942 fn i64_cmp_le_u(
4943 &mut self,
4944 loc_a: Location,
4945 loc_b: Location,
4946 ret: Location,
4947 ) -> Result<(), CompileError> {
4948 self.emit_cmpop_i64_dynamic_b(Condition::Ls, loc_a, loc_b, ret)
4949 }
4950 fn i64_cmp_lt_u(
4951 &mut self,
4952 loc_a: Location,
4953 loc_b: Location,
4954 ret: Location,
4955 ) -> Result<(), CompileError> {
4956 self.emit_cmpop_i64_dynamic_b(Condition::Cc, loc_a, loc_b, ret)
4957 }
4958 fn i64_cmp_ne(
4959 &mut self,
4960 loc_a: Location,
4961 loc_b: Location,
4962 ret: Location,
4963 ) -> Result<(), CompileError> {
4964 self.emit_cmpop_i64_dynamic_b(Condition::Ne, loc_a, loc_b, ret)
4965 }
4966 fn i64_cmp_eq(
4967 &mut self,
4968 loc_a: Location,
4969 loc_b: Location,
4970 ret: Location,
4971 ) -> Result<(), CompileError> {
4972 self.emit_cmpop_i64_dynamic_b(Condition::Eq, loc_a, loc_b, ret)
4973 }
4974 fn i64_clz(&mut self, src: Location, dst: Location) -> Result<(), CompileError> {
4975 self.emit_relaxed_binop(Assembler::emit_clz, Size::S64, src, dst, true)
4976 }
4977 fn i64_ctz(&mut self, src: Location, dst: Location) -> Result<(), CompileError> {
4978 let mut temps = vec![];
4979 let src = self.location_to_reg(Size::S64, src, &mut temps, ImmType::None, true, None)?;
4980 let dest = self.location_to_reg(Size::S64, dst, &mut temps, ImmType::None, false, None)?;
4981 self.assembler.emit_rbit(Size::S64, src, dest)?;
4982 self.assembler.emit_clz(Size::S64, dest, dest)?;
4983 if dst != dest {
4984 self.move_location(Size::S64, dest, dst)?;
4985 }
4986 for r in temps {
4987 self.release_gpr(r);
4988 }
4989 Ok(())
4990 }
4991 fn i64_popcnt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
4992 if self.has_neon {
4993 let mut temps = vec![];
4994
4995 let src_gpr =
4996 self.location_to_reg(Size::S64, loc, &mut temps, ImmType::None, true, None)?;
4997 let dst_gpr =
4998 self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4999
5000 let mut neon_temps = vec![];
5001 let neon_temp = self.acquire_temp_simd().ok_or_else(|| {
5002 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5003 })?;
5004 neon_temps.push(neon_temp);
5005
5006 self.assembler
5007 .emit_fmov(Size::S64, src_gpr, Size::S64, Location::SIMD(neon_temp))?;
5008 self.assembler.emit_cnt(neon_temp, neon_temp)?;
5009 self.assembler.emit_addv(neon_temp, neon_temp)?;
5010 self.assembler
5011 .emit_fmov(Size::S64, Location::SIMD(neon_temp), Size::S64, dst_gpr)?;
5012
5013 if ret != dst_gpr {
5014 self.move_location(Size::S64, dst_gpr, ret)?;
5015 }
5016
5017 for r in temps {
5018 self.release_gpr(r);
5019 }
5020
5021 for r in neon_temps {
5022 self.release_simd(r);
5023 }
5024 } else {
5025 let mut temps = vec![];
5026 let src =
5027 self.location_to_reg(Size::S64, loc, &mut temps, ImmType::None, true, None)?;
5028 let dest =
5029 self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
5030 let src = if src == loc {
5031 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
5032 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5033 })?;
5034 temps.push(tmp);
5035 self.assembler
5036 .emit_mov(Size::S64, src, Location::GPR(tmp))?;
5037 Location::GPR(tmp)
5038 } else {
5039 src
5040 };
5041 let tmp = {
5042 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
5043 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5044 })?;
5045 temps.push(tmp);
5046 Location::GPR(tmp)
5047 };
5048 let label_loop = self.assembler.get_label();
5049 let label_exit = self.assembler.get_label();
5050 self.assembler
5051 .emit_mov(Size::S32, Location::GPR(GPR::XzrSp), dest)?; self.assembler.emit_cbz_label(Size::S64, src, label_exit)?; self.assembler.emit_label(label_loop)?;
5054 self.assembler
5055 .emit_add(Size::S32, dest, Location::Imm8(1), dest)?; self.assembler.emit_clz(Size::S64, src, tmp)?; self.assembler.emit_lsl(Size::S64, src, tmp, src)?; self.assembler
5059 .emit_lsl(Size::S64, src, Location::Imm8(1), src)?; self.assembler.emit_cbnz_label(Size::S64, src, label_loop)?; self.assembler.emit_label(label_exit)?;
5062 if ret != dest {
5063 self.move_location(Size::S64, dest, ret)?;
5064 }
5065 for r in temps {
5066 self.release_gpr(r);
5067 }
5068 }
5069
5070 Ok(())
5071 }
5072 fn i64_shl(
5073 &mut self,
5074 loc_a: Location,
5075 loc_b: Location,
5076 ret: Location,
5077 ) -> Result<(), CompileError> {
5078 self.emit_relaxed_binop3(
5079 Assembler::emit_lsl,
5080 Size::S64,
5081 loc_a,
5082 loc_b,
5083 ret,
5084 ImmType::Shift64No0,
5085 )
5086 }
5087 fn i64_shr(
5088 &mut self,
5089 loc_a: Location,
5090 loc_b: Location,
5091 ret: Location,
5092 ) -> Result<(), CompileError> {
5093 self.emit_relaxed_binop3(
5094 Assembler::emit_lsr,
5095 Size::S64,
5096 loc_a,
5097 loc_b,
5098 ret,
5099 ImmType::Shift64No0,
5100 )
5101 }
5102 fn i64_sar(
5103 &mut self,
5104 loc_a: Location,
5105 loc_b: Location,
5106 ret: Location,
5107 ) -> Result<(), CompileError> {
5108 self.emit_relaxed_binop3(
5109 Assembler::emit_asr,
5110 Size::S64,
5111 loc_a,
5112 loc_b,
5113 ret,
5114 ImmType::Shift64No0,
5115 )
5116 }
5117 fn i64_rol(
5118 &mut self,
5119 loc_a: Location,
5120 loc_b: Location,
5121 ret: Location,
5122 ) -> Result<(), CompileError> {
5123 let mut temps = vec![];
5125 let src2 = match loc_b {
5126 Location::Imm8(imm) => Location::Imm8(64 - (imm & 63)),
5127 Location::Imm32(imm) => Location::Imm8(64 - (imm & 63) as u8),
5128 Location::Imm64(imm) => Location::Imm8(64 - (imm & 63) as u8),
5129 _ => {
5130 let tmp1 = self.location_to_reg(
5131 Size::S64,
5132 Location::Imm32(64),
5133 &mut temps,
5134 ImmType::None,
5135 true,
5136 None,
5137 )?;
5138 let tmp2 =
5139 self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
5140 self.assembler.emit_sub(Size::S64, tmp1, tmp2, tmp1)?;
5141 tmp1
5142 }
5143 };
5144 self.emit_relaxed_binop3(
5145 Assembler::emit_ror,
5146 Size::S64,
5147 loc_a,
5148 src2,
5149 ret,
5150 ImmType::Shift64No0,
5151 )?;
5152 for r in temps {
5153 self.release_gpr(r);
5154 }
5155 Ok(())
5156 }
5157 fn i64_ror(
5158 &mut self,
5159 loc_a: Location,
5160 loc_b: Location,
5161 ret: Location,
5162 ) -> Result<(), CompileError> {
5163 self.emit_relaxed_binop3(
5164 Assembler::emit_ror,
5165 Size::S64,
5166 loc_a,
5167 loc_b,
5168 ret,
5169 ImmType::Shift64No0,
5170 )
5171 }
5172 fn i64_load(
5173 &mut self,
5174 addr: Location,
5175 memarg: &MemArg,
5176 ret: Location,
5177 need_check: bool,
5178 imported_memories: bool,
5179 offset: i32,
5180 heap_access_oob: Label,
5181 unaligned_atomic: Label,
5182 ) -> Result<(), CompileError> {
5183 self.memory_op(
5184 addr,
5185 memarg,
5186 false,
5187 8,
5188 need_check,
5189 imported_memories,
5190 offset,
5191 heap_access_oob,
5192 unaligned_atomic,
5193 |this, addr| this.emit_relaxed_ldr64(Size::S64, ret, Location::Memory(addr, 0)),
5194 )
5195 }
5196 fn i64_load_8u(
5197 &mut self,
5198 addr: Location,
5199 memarg: &MemArg,
5200 ret: Location,
5201 need_check: bool,
5202 imported_memories: bool,
5203 offset: i32,
5204 heap_access_oob: Label,
5205 unaligned_atomic: Label,
5206 ) -> Result<(), CompileError> {
5207 self.memory_op(
5208 addr,
5209 memarg,
5210 false,
5211 1,
5212 need_check,
5213 imported_memories,
5214 offset,
5215 heap_access_oob,
5216 unaligned_atomic,
5217 |this, addr| this.emit_relaxed_ldr8(Size::S64, ret, Location::Memory(addr, 0)),
5218 )
5219 }
5220 fn i64_load_8s(
5221 &mut self,
5222 addr: Location,
5223 memarg: &MemArg,
5224 ret: Location,
5225 need_check: bool,
5226 imported_memories: bool,
5227 offset: i32,
5228 heap_access_oob: Label,
5229 unaligned_atomic: Label,
5230 ) -> Result<(), CompileError> {
5231 self.memory_op(
5232 addr,
5233 memarg,
5234 false,
5235 1,
5236 need_check,
5237 imported_memories,
5238 offset,
5239 heap_access_oob,
5240 unaligned_atomic,
5241 |this, addr| this.emit_relaxed_ldr8s(Size::S64, ret, Location::Memory(addr, 0)),
5242 )
5243 }
5244 fn i64_load_16u(
5245 &mut self,
5246 addr: Location,
5247 memarg: &MemArg,
5248 ret: Location,
5249 need_check: bool,
5250 imported_memories: bool,
5251 offset: i32,
5252 heap_access_oob: Label,
5253 unaligned_atomic: Label,
5254 ) -> Result<(), CompileError> {
5255 self.memory_op(
5256 addr,
5257 memarg,
5258 false,
5259 2,
5260 need_check,
5261 imported_memories,
5262 offset,
5263 heap_access_oob,
5264 unaligned_atomic,
5265 |this, addr| this.emit_relaxed_ldr16(Size::S64, ret, Location::Memory(addr, 0)),
5266 )
5267 }
5268 fn i64_load_16s(
5269 &mut self,
5270 addr: Location,
5271 memarg: &MemArg,
5272 ret: Location,
5273 need_check: bool,
5274 imported_memories: bool,
5275 offset: i32,
5276 heap_access_oob: Label,
5277 unaligned_atomic: Label,
5278 ) -> Result<(), CompileError> {
5279 self.memory_op(
5280 addr,
5281 memarg,
5282 false,
5283 2,
5284 need_check,
5285 imported_memories,
5286 offset,
5287 heap_access_oob,
5288 unaligned_atomic,
5289 |this, addr| this.emit_relaxed_ldr16s(Size::S64, ret, Location::Memory(addr, 0)),
5290 )
5291 }
5292 fn i64_load_32u(
5293 &mut self,
5294 addr: Location,
5295 memarg: &MemArg,
5296 ret: Location,
5297 need_check: bool,
5298 imported_memories: bool,
5299 offset: i32,
5300 heap_access_oob: Label,
5301 unaligned_atomic: Label,
5302 ) -> Result<(), CompileError> {
5303 self.memory_op(
5304 addr,
5305 memarg,
5306 false,
5307 4,
5308 need_check,
5309 imported_memories,
5310 offset,
5311 heap_access_oob,
5312 unaligned_atomic,
5313 |this, addr| this.emit_relaxed_ldr32(Size::S64, ret, Location::Memory(addr, 0)),
5314 )
5315 }
5316 fn i64_load_32s(
5317 &mut self,
5318 addr: Location,
5319 memarg: &MemArg,
5320 ret: Location,
5321 need_check: bool,
5322 imported_memories: bool,
5323 offset: i32,
5324 heap_access_oob: Label,
5325 unaligned_atomic: Label,
5326 ) -> Result<(), CompileError> {
5327 self.memory_op(
5328 addr,
5329 memarg,
5330 false,
5331 4,
5332 need_check,
5333 imported_memories,
5334 offset,
5335 heap_access_oob,
5336 unaligned_atomic,
5337 |this, addr| this.emit_relaxed_ldr32s(Size::S64, ret, Location::Memory(addr, 0)),
5338 )
5339 }
5340 fn i64_atomic_load(
5341 &mut self,
5342 addr: Location,
5343 memarg: &MemArg,
5344 ret: Location,
5345 need_check: bool,
5346 imported_memories: bool,
5347 offset: i32,
5348 heap_access_oob: Label,
5349 unaligned_atomic: Label,
5350 ) -> Result<(), CompileError> {
5351 self.memory_op(
5352 addr,
5353 memarg,
5354 true,
5355 8,
5356 need_check,
5357 imported_memories,
5358 offset,
5359 heap_access_oob,
5360 unaligned_atomic,
5361 |this, addr| this.emit_relaxed_ldr64(Size::S64, ret, Location::Memory(addr, 0)),
5362 )
5363 }
5364 fn i64_atomic_load_8u(
5365 &mut self,
5366 addr: Location,
5367 memarg: &MemArg,
5368 ret: Location,
5369 need_check: bool,
5370 imported_memories: bool,
5371 offset: i32,
5372 heap_access_oob: Label,
5373 unaligned_atomic: Label,
5374 ) -> Result<(), CompileError> {
5375 self.memory_op(
5376 addr,
5377 memarg,
5378 true,
5379 1,
5380 need_check,
5381 imported_memories,
5382 offset,
5383 heap_access_oob,
5384 unaligned_atomic,
5385 |this, addr| this.emit_relaxed_ldr8(Size::S64, ret, Location::Memory(addr, 0)),
5386 )
5387 }
5388 fn i64_atomic_load_16u(
5389 &mut self,
5390 addr: Location,
5391 memarg: &MemArg,
5392 ret: Location,
5393 need_check: bool,
5394 imported_memories: bool,
5395 offset: i32,
5396 heap_access_oob: Label,
5397 unaligned_atomic: Label,
5398 ) -> Result<(), CompileError> {
5399 self.memory_op(
5400 addr,
5401 memarg,
5402 true,
5403 2,
5404 need_check,
5405 imported_memories,
5406 offset,
5407 heap_access_oob,
5408 unaligned_atomic,
5409 |this, addr| this.emit_relaxed_ldr16(Size::S64, ret, Location::Memory(addr, 0)),
5410 )
5411 }
5412 fn i64_atomic_load_32u(
5413 &mut self,
5414 addr: Location,
5415 memarg: &MemArg,
5416 ret: Location,
5417 need_check: bool,
5418 imported_memories: bool,
5419 offset: i32,
5420 heap_access_oob: Label,
5421 unaligned_atomic: Label,
5422 ) -> Result<(), CompileError> {
5423 self.memory_op(
5424 addr,
5425 memarg,
5426 true,
5427 4,
5428 need_check,
5429 imported_memories,
5430 offset,
5431 heap_access_oob,
5432 unaligned_atomic,
5433 |this, addr| this.emit_relaxed_ldr32(Size::S64, ret, Location::Memory(addr, 0)),
5434 )
5435 }
5436 fn i64_save(
5437 &mut self,
5438 target_value: Location,
5439 memarg: &MemArg,
5440 target_addr: Location,
5441 need_check: bool,
5442 imported_memories: bool,
5443 offset: i32,
5444 heap_access_oob: Label,
5445 unaligned_atomic: Label,
5446 ) -> Result<(), CompileError> {
5447 self.memory_op(
5448 target_addr,
5449 memarg,
5450 false,
5451 8,
5452 need_check,
5453 imported_memories,
5454 offset,
5455 heap_access_oob,
5456 unaligned_atomic,
5457 |this, addr| this.emit_relaxed_str64(target_value, Location::Memory(addr, 0)),
5458 )
5459 }
5460 fn i64_save_8(
5461 &mut self,
5462 target_value: Location,
5463 memarg: &MemArg,
5464 target_addr: Location,
5465 need_check: bool,
5466 imported_memories: bool,
5467 offset: i32,
5468 heap_access_oob: Label,
5469 unaligned_atomic: Label,
5470 ) -> Result<(), CompileError> {
5471 self.memory_op(
5472 target_addr,
5473 memarg,
5474 false,
5475 1,
5476 need_check,
5477 imported_memories,
5478 offset,
5479 heap_access_oob,
5480 unaligned_atomic,
5481 |this, addr| this.emit_relaxed_str8(target_value, Location::Memory(addr, 0)),
5482 )
5483 }
5484 fn i64_save_16(
5485 &mut self,
5486 target_value: Location,
5487 memarg: &MemArg,
5488 target_addr: Location,
5489 need_check: bool,
5490 imported_memories: bool,
5491 offset: i32,
5492 heap_access_oob: Label,
5493 unaligned_atomic: Label,
5494 ) -> Result<(), CompileError> {
5495 self.memory_op(
5496 target_addr,
5497 memarg,
5498 false,
5499 2,
5500 need_check,
5501 imported_memories,
5502 offset,
5503 heap_access_oob,
5504 unaligned_atomic,
5505 |this, addr| this.emit_relaxed_str16(target_value, Location::Memory(addr, 0)),
5506 )
5507 }
5508 fn i64_save_32(
5509 &mut self,
5510 target_value: Location,
5511 memarg: &MemArg,
5512 target_addr: Location,
5513 need_check: bool,
5514 imported_memories: bool,
5515 offset: i32,
5516 heap_access_oob: Label,
5517 unaligned_atomic: Label,
5518 ) -> Result<(), CompileError> {
5519 self.memory_op(
5520 target_addr,
5521 memarg,
5522 false,
5523 4,
5524 need_check,
5525 imported_memories,
5526 offset,
5527 heap_access_oob,
5528 unaligned_atomic,
5529 |this, addr| this.emit_relaxed_str32(target_value, Location::Memory(addr, 0)),
5530 )
5531 }
5532 fn i64_atomic_save(
5533 &mut self,
5534 target_value: Location,
5535 memarg: &MemArg,
5536 target_addr: Location,
5537 need_check: bool,
5538 imported_memories: bool,
5539 offset: i32,
5540 heap_access_oob: Label,
5541 unaligned_atomic: Label,
5542 ) -> Result<(), CompileError> {
5543 self.memory_op(
5544 target_addr,
5545 memarg,
5546 true,
5547 8,
5548 need_check,
5549 imported_memories,
5550 offset,
5551 heap_access_oob,
5552 unaligned_atomic,
5553 |this, addr| this.emit_relaxed_str64(target_value, Location::Memory(addr, 0)),
5554 )?;
5555 self.assembler.emit_dmb()
5556 }
5557 fn i64_atomic_save_8(
5558 &mut self,
5559 target_value: Location,
5560 memarg: &MemArg,
5561 target_addr: Location,
5562 need_check: bool,
5563 imported_memories: bool,
5564 offset: i32,
5565 heap_access_oob: Label,
5566 unaligned_atomic: Label,
5567 ) -> Result<(), CompileError> {
5568 self.memory_op(
5569 target_addr,
5570 memarg,
5571 true,
5572 1,
5573 need_check,
5574 imported_memories,
5575 offset,
5576 heap_access_oob,
5577 unaligned_atomic,
5578 |this, addr| this.emit_relaxed_str8(target_value, Location::Memory(addr, 0)),
5579 )?;
5580 self.assembler.emit_dmb()
5581 }
5582 fn i64_atomic_save_16(
5583 &mut self,
5584 target_value: Location,
5585 memarg: &MemArg,
5586 target_addr: Location,
5587 need_check: bool,
5588 imported_memories: bool,
5589 offset: i32,
5590 heap_access_oob: Label,
5591 unaligned_atomic: Label,
5592 ) -> Result<(), CompileError> {
5593 self.memory_op(
5594 target_addr,
5595 memarg,
5596 true,
5597 2,
5598 need_check,
5599 imported_memories,
5600 offset,
5601 heap_access_oob,
5602 unaligned_atomic,
5603 |this, addr| this.emit_relaxed_str16(target_value, Location::Memory(addr, 0)),
5604 )?;
5605 self.assembler.emit_dmb()
5606 }
5607 fn i64_atomic_save_32(
5608 &mut self,
5609 target_value: Location,
5610 memarg: &MemArg,
5611 target_addr: Location,
5612 need_check: bool,
5613 imported_memories: bool,
5614 offset: i32,
5615 heap_access_oob: Label,
5616 unaligned_atomic: Label,
5617 ) -> Result<(), CompileError> {
5618 self.memory_op(
5619 target_addr,
5620 memarg,
5621 true,
5622 4,
5623 need_check,
5624 imported_memories,
5625 offset,
5626 heap_access_oob,
5627 unaligned_atomic,
5628 |this, addr| this.emit_relaxed_str32(target_value, Location::Memory(addr, 0)),
5629 )?;
5630 self.assembler.emit_dmb()
5631 }
5632 fn i64_atomic_add(
5634 &mut self,
5635 loc: Location,
5636 target: Location,
5637 memarg: &MemArg,
5638 ret: Location,
5639 need_check: bool,
5640 imported_memories: bool,
5641 offset: i32,
5642 heap_access_oob: Label,
5643 unaligned_atomic: Label,
5644 ) -> Result<(), CompileError> {
5645 self.memory_op(
5646 target,
5647 memarg,
5648 true,
5649 8,
5650 need_check,
5651 imported_memories,
5652 offset,
5653 heap_access_oob,
5654 unaligned_atomic,
5655 |this, addr| {
5656 let mut temps = vec![];
5657 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
5658 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5659 })?;
5660 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
5661 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5662 })?;
5663 let dst =
5664 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
5665 let reread = this.get_label();
5666
5667 this.emit_label(reread)?;
5668 this.assembler
5669 .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?;
5670 this.emit_binop_add64(dst, loc, Location::GPR(tmp1))?;
5671 this.assembler.emit_stlxr(
5672 Size::S64,
5673 Location::GPR(tmp2),
5674 Location::GPR(tmp1),
5675 Location::GPR(addr),
5676 )?;
5677 this.assembler
5678 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
5679 this.assembler.emit_dmb()?;
5680
5681 if dst != ret {
5682 this.move_location(Size::S64, ret, dst)?;
5683 }
5684 for r in temps {
5685 this.release_gpr(r);
5686 }
5687 this.release_gpr(tmp1);
5688 this.release_gpr(tmp2);
5689 Ok(())
5690 },
5691 )
5692 }
5693 fn i64_atomic_add_8u(
5695 &mut self,
5696 loc: Location,
5697 target: Location,
5698 memarg: &MemArg,
5699 ret: Location,
5700 need_check: bool,
5701 imported_memories: bool,
5702 offset: i32,
5703 heap_access_oob: Label,
5704 unaligned_atomic: Label,
5705 ) -> Result<(), CompileError> {
5706 self.memory_op(
5707 target,
5708 memarg,
5709 true,
5710 1,
5711 need_check,
5712 imported_memories,
5713 offset,
5714 heap_access_oob,
5715 unaligned_atomic,
5716 |this, addr| {
5717 let mut temps = vec![];
5718 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
5719 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5720 })?;
5721 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
5722 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5723 })?;
5724 let dst =
5725 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
5726 let reread = this.get_label();
5727
5728 this.emit_label(reread)?;
5729 this.assembler
5730 .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?;
5731 this.emit_binop_add64(dst, loc, Location::GPR(tmp1))?;
5732 this.assembler.emit_stlxrb(
5733 Size::S64,
5734 Location::GPR(tmp2),
5735 Location::GPR(tmp1),
5736 Location::GPR(addr),
5737 )?;
5738 this.assembler
5739 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
5740 this.assembler.emit_dmb()?;
5741
5742 if dst != ret {
5743 this.move_location(Size::S64, ret, dst)?;
5744 }
5745 for r in temps {
5746 this.release_gpr(r);
5747 }
5748 this.release_gpr(tmp1);
5749 this.release_gpr(tmp2);
5750 Ok(())
5751 },
5752 )
5753 }
5754 fn i64_atomic_add_16u(
5756 &mut self,
5757 loc: Location,
5758 target: Location,
5759 memarg: &MemArg,
5760 ret: Location,
5761 need_check: bool,
5762 imported_memories: bool,
5763 offset: i32,
5764 heap_access_oob: Label,
5765 unaligned_atomic: Label,
5766 ) -> Result<(), CompileError> {
5767 self.memory_op(
5768 target,
5769 memarg,
5770 true,
5771 2,
5772 need_check,
5773 imported_memories,
5774 offset,
5775 heap_access_oob,
5776 unaligned_atomic,
5777 |this, addr| {
5778 let mut temps = vec![];
5779 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
5780 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5781 })?;
5782 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
5783 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5784 })?;
5785 let dst =
5786 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
5787 let reread = this.get_label();
5788
5789 this.emit_label(reread)?;
5790 this.assembler
5791 .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?;
5792 this.emit_binop_add64(dst, loc, Location::GPR(tmp1))?;
5793 this.assembler.emit_stlxrh(
5794 Size::S64,
5795 Location::GPR(tmp2),
5796 Location::GPR(tmp1),
5797 Location::GPR(addr),
5798 )?;
5799 this.assembler
5800 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
5801 this.assembler.emit_dmb()?;
5802
5803 if dst != ret {
5804 this.move_location(Size::S64, ret, dst)?;
5805 }
5806 for r in temps {
5807 this.release_gpr(r);
5808 }
5809 this.release_gpr(tmp1);
5810 this.release_gpr(tmp2);
5811 Ok(())
5812 },
5813 )
5814 }
5815 fn i64_atomic_add_32u(
5817 &mut self,
5818 loc: Location,
5819 target: Location,
5820 memarg: &MemArg,
5821 ret: Location,
5822 need_check: bool,
5823 imported_memories: bool,
5824 offset: i32,
5825 heap_access_oob: Label,
5826 unaligned_atomic: Label,
5827 ) -> Result<(), CompileError> {
5828 self.memory_op(
5829 target,
5830 memarg,
5831 true,
5832 4,
5833 need_check,
5834 imported_memories,
5835 offset,
5836 heap_access_oob,
5837 unaligned_atomic,
5838 |this, addr| {
5839 let mut temps = vec![];
5840 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
5841 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5842 })?;
5843 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
5844 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5845 })?;
5846 let dst =
5847 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
5848 let reread = this.get_label();
5849
5850 this.emit_label(reread)?;
5851 this.assembler
5852 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
5853 this.emit_binop_add64(dst, loc, Location::GPR(tmp1))?;
5854 this.assembler.emit_stlxr(
5855 Size::S32,
5856 Location::GPR(tmp2),
5857 Location::GPR(tmp1),
5858 Location::GPR(addr),
5859 )?;
5860 this.assembler
5861 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
5862 this.assembler.emit_dmb()?;
5863
5864 if dst != ret {
5865 this.move_location(Size::S64, ret, dst)?;
5866 }
5867 for r in temps {
5868 this.release_gpr(r);
5869 }
5870 this.release_gpr(tmp1);
5871 this.release_gpr(tmp2);
5872 Ok(())
5873 },
5874 )
5875 }
5876 fn i64_atomic_sub(
5878 &mut self,
5879 loc: Location,
5880 target: Location,
5881 memarg: &MemArg,
5882 ret: Location,
5883 need_check: bool,
5884 imported_memories: bool,
5885 offset: i32,
5886 heap_access_oob: Label,
5887 unaligned_atomic: Label,
5888 ) -> Result<(), CompileError> {
5889 self.memory_op(
5890 target,
5891 memarg,
5892 true,
5893 8,
5894 need_check,
5895 imported_memories,
5896 offset,
5897 heap_access_oob,
5898 unaligned_atomic,
5899 |this, addr| {
5900 let mut temps = vec![];
5901 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
5902 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5903 })?;
5904 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
5905 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5906 })?;
5907 let dst =
5908 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
5909 let reread = this.get_label();
5910
5911 this.emit_label(reread)?;
5912 this.assembler
5913 .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?;
5914 this.emit_binop_sub64(dst, loc, Location::GPR(tmp1))?;
5915 this.assembler.emit_stlxr(
5916 Size::S64,
5917 Location::GPR(tmp2),
5918 Location::GPR(tmp1),
5919 Location::GPR(addr),
5920 )?;
5921 this.assembler
5922 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
5923 this.assembler.emit_dmb()?;
5924
5925 if dst != ret {
5926 this.move_location(Size::S64, ret, dst)?;
5927 }
5928 for r in temps {
5929 this.release_gpr(r);
5930 }
5931 this.release_gpr(tmp1);
5932 this.release_gpr(tmp2);
5933 Ok(())
5934 },
5935 )
5936 }
5937 fn i64_atomic_sub_8u(
5939 &mut self,
5940 loc: Location,
5941 target: Location,
5942 memarg: &MemArg,
5943 ret: Location,
5944 need_check: bool,
5945 imported_memories: bool,
5946 offset: i32,
5947 heap_access_oob: Label,
5948 unaligned_atomic: Label,
5949 ) -> Result<(), CompileError> {
5950 self.memory_op(
5951 target,
5952 memarg,
5953 true,
5954 1,
5955 need_check,
5956 imported_memories,
5957 offset,
5958 heap_access_oob,
5959 unaligned_atomic,
5960 |this, addr| {
5961 let mut temps = vec![];
5962 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
5963 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5964 })?;
5965 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
5966 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5967 })?;
5968 let dst =
5969 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
5970 let reread = this.get_label();
5971
5972 this.emit_label(reread)?;
5973 this.assembler
5974 .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?;
5975 this.emit_binop_sub64(dst, loc, Location::GPR(tmp1))?;
5976 this.assembler.emit_stlxrb(
5977 Size::S64,
5978 Location::GPR(tmp2),
5979 Location::GPR(tmp1),
5980 Location::GPR(addr),
5981 )?;
5982 this.assembler
5983 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
5984 this.assembler.emit_dmb()?;
5985
5986 if dst != ret {
5987 this.move_location(Size::S64, ret, dst)?;
5988 }
5989 for r in temps {
5990 this.release_gpr(r);
5991 }
5992 this.release_gpr(tmp1);
5993 this.release_gpr(tmp2);
5994 Ok(())
5995 },
5996 )
5997 }
5998 fn i64_atomic_sub_16u(
6000 &mut self,
6001 loc: Location,
6002 target: Location,
6003 memarg: &MemArg,
6004 ret: Location,
6005 need_check: bool,
6006 imported_memories: bool,
6007 offset: i32,
6008 heap_access_oob: Label,
6009 unaligned_atomic: Label,
6010 ) -> Result<(), CompileError> {
6011 self.memory_op(
6012 target,
6013 memarg,
6014 true,
6015 2,
6016 need_check,
6017 imported_memories,
6018 offset,
6019 heap_access_oob,
6020 unaligned_atomic,
6021 |this, addr| {
6022 let mut temps = vec![];
6023 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6024 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6025 })?;
6026 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6027 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6028 })?;
6029 let dst =
6030 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6031 let reread = this.get_label();
6032
6033 this.emit_label(reread)?;
6034 this.assembler
6035 .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?;
6036 this.emit_binop_sub64(dst, loc, Location::GPR(tmp1))?;
6037 this.assembler.emit_stlxrh(
6038 Size::S64,
6039 Location::GPR(tmp2),
6040 Location::GPR(tmp1),
6041 Location::GPR(addr),
6042 )?;
6043 this.assembler
6044 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6045 this.assembler.emit_dmb()?;
6046
6047 if dst != ret {
6048 this.move_location(Size::S64, ret, dst)?;
6049 }
6050 for r in temps {
6051 this.release_gpr(r);
6052 }
6053 this.release_gpr(tmp1);
6054 this.release_gpr(tmp2);
6055 Ok(())
6056 },
6057 )
6058 }
6059 fn i64_atomic_sub_32u(
6061 &mut self,
6062 loc: Location,
6063 target: Location,
6064 memarg: &MemArg,
6065 ret: Location,
6066 need_check: bool,
6067 imported_memories: bool,
6068 offset: i32,
6069 heap_access_oob: Label,
6070 unaligned_atomic: Label,
6071 ) -> Result<(), CompileError> {
6072 self.memory_op(
6073 target,
6074 memarg,
6075 true,
6076 4,
6077 need_check,
6078 imported_memories,
6079 offset,
6080 heap_access_oob,
6081 unaligned_atomic,
6082 |this, addr| {
6083 let mut temps = vec![];
6084 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6085 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6086 })?;
6087 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6088 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6089 })?;
6090 let dst =
6091 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6092 let reread = this.get_label();
6093
6094 this.emit_label(reread)?;
6095 this.assembler
6096 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
6097 this.emit_binop_sub64(dst, loc, Location::GPR(tmp1))?;
6098 this.assembler.emit_stlxr(
6099 Size::S32,
6100 Location::GPR(tmp2),
6101 Location::GPR(tmp1),
6102 Location::GPR(addr),
6103 )?;
6104 this.assembler
6105 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6106 this.assembler.emit_dmb()?;
6107
6108 if dst != ret {
6109 this.move_location(Size::S64, ret, dst)?;
6110 }
6111 for r in temps {
6112 this.release_gpr(r);
6113 }
6114 this.release_gpr(tmp1);
6115 this.release_gpr(tmp2);
6116 Ok(())
6117 },
6118 )
6119 }
6120 fn i64_atomic_and(
6122 &mut self,
6123 loc: Location,
6124 target: Location,
6125 memarg: &MemArg,
6126 ret: Location,
6127 need_check: bool,
6128 imported_memories: bool,
6129 offset: i32,
6130 heap_access_oob: Label,
6131 unaligned_atomic: Label,
6132 ) -> Result<(), CompileError> {
6133 self.memory_op(
6134 target,
6135 memarg,
6136 true,
6137 8,
6138 need_check,
6139 imported_memories,
6140 offset,
6141 heap_access_oob,
6142 unaligned_atomic,
6143 |this, addr| {
6144 let mut temps = vec![];
6145 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6146 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6147 })?;
6148 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6149 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6150 })?;
6151 let dst =
6152 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6153 let reread = this.get_label();
6154
6155 this.emit_label(reread)?;
6156 this.assembler
6157 .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?;
6158 this.emit_binop_and64(dst, loc, Location::GPR(tmp1))?;
6159 this.assembler.emit_stlxr(
6160 Size::S64,
6161 Location::GPR(tmp2),
6162 Location::GPR(tmp1),
6163 Location::GPR(addr),
6164 )?;
6165 this.assembler
6166 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6167 this.assembler.emit_dmb()?;
6168
6169 if dst != ret {
6170 this.move_location(Size::S64, ret, dst)?;
6171 }
6172 for r in temps {
6173 this.release_gpr(r);
6174 }
6175 this.release_gpr(tmp1);
6176 this.release_gpr(tmp2);
6177 Ok(())
6178 },
6179 )
6180 }
6181 fn i64_atomic_and_8u(
6183 &mut self,
6184 loc: Location,
6185 target: Location,
6186 memarg: &MemArg,
6187 ret: Location,
6188 need_check: bool,
6189 imported_memories: bool,
6190 offset: i32,
6191 heap_access_oob: Label,
6192 unaligned_atomic: Label,
6193 ) -> Result<(), CompileError> {
6194 self.memory_op(
6195 target,
6196 memarg,
6197 true,
6198 1,
6199 need_check,
6200 imported_memories,
6201 offset,
6202 heap_access_oob,
6203 unaligned_atomic,
6204 |this, addr| {
6205 let mut temps = vec![];
6206 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6207 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6208 })?;
6209 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6210 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6211 })?;
6212 let dst =
6213 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6214 let reread = this.get_label();
6215
6216 this.emit_label(reread)?;
6217 this.assembler
6218 .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?;
6219 this.emit_binop_and64(dst, loc, Location::GPR(tmp1))?;
6220 this.assembler.emit_stlxrb(
6221 Size::S64,
6222 Location::GPR(tmp2),
6223 Location::GPR(tmp1),
6224 Location::GPR(addr),
6225 )?;
6226 this.assembler
6227 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6228 this.assembler.emit_dmb()?;
6229
6230 if dst != ret {
6231 this.move_location(Size::S64, ret, dst)?;
6232 }
6233 for r in temps {
6234 this.release_gpr(r);
6235 }
6236 this.release_gpr(tmp1);
6237 this.release_gpr(tmp2);
6238 Ok(())
6239 },
6240 )
6241 }
6242 fn i64_atomic_and_16u(
6244 &mut self,
6245 loc: Location,
6246 target: Location,
6247 memarg: &MemArg,
6248 ret: Location,
6249 need_check: bool,
6250 imported_memories: bool,
6251 offset: i32,
6252 heap_access_oob: Label,
6253 unaligned_atomic: Label,
6254 ) -> Result<(), CompileError> {
6255 self.memory_op(
6256 target,
6257 memarg,
6258 true,
6259 2,
6260 need_check,
6261 imported_memories,
6262 offset,
6263 heap_access_oob,
6264 unaligned_atomic,
6265 |this, addr| {
6266 let mut temps = vec![];
6267 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6268 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6269 })?;
6270 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6271 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6272 })?;
6273 let dst =
6274 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6275 let reread = this.get_label();
6276
6277 this.emit_label(reread)?;
6278 this.assembler
6279 .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?;
6280 this.emit_binop_and64(dst, loc, Location::GPR(tmp1))?;
6281 this.assembler.emit_stlxrh(
6282 Size::S64,
6283 Location::GPR(tmp2),
6284 Location::GPR(tmp1),
6285 Location::GPR(addr),
6286 )?;
6287 this.assembler
6288 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6289 this.assembler.emit_dmb()?;
6290
6291 if dst != ret {
6292 this.move_location(Size::S64, ret, dst)?;
6293 }
6294 for r in temps {
6295 this.release_gpr(r);
6296 }
6297 this.release_gpr(tmp1);
6298 this.release_gpr(tmp2);
6299 Ok(())
6300 },
6301 )
6302 }
6303 fn i64_atomic_and_32u(
6305 &mut self,
6306 loc: Location,
6307 target: Location,
6308 memarg: &MemArg,
6309 ret: Location,
6310 need_check: bool,
6311 imported_memories: bool,
6312 offset: i32,
6313 heap_access_oob: Label,
6314 unaligned_atomic: Label,
6315 ) -> Result<(), CompileError> {
6316 self.memory_op(
6317 target,
6318 memarg,
6319 true,
6320 4,
6321 need_check,
6322 imported_memories,
6323 offset,
6324 heap_access_oob,
6325 unaligned_atomic,
6326 |this, addr| {
6327 let mut temps = vec![];
6328 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6329 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6330 })?;
6331 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6332 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6333 })?;
6334 let dst =
6335 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6336 let reread = this.get_label();
6337
6338 this.emit_label(reread)?;
6339 this.assembler
6340 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
6341 this.emit_binop_and64(dst, loc, Location::GPR(tmp1))?;
6342 this.assembler.emit_stlxr(
6343 Size::S32,
6344 Location::GPR(tmp2),
6345 Location::GPR(tmp1),
6346 Location::GPR(addr),
6347 )?;
6348 this.assembler
6349 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6350 this.assembler.emit_dmb()?;
6351
6352 if dst != ret {
6353 this.move_location(Size::S64, ret, dst)?;
6354 }
6355 for r in temps {
6356 this.release_gpr(r);
6357 }
6358 this.release_gpr(tmp1);
6359 this.release_gpr(tmp2);
6360 Ok(())
6361 },
6362 )
6363 }
6364 fn i64_atomic_or(
6366 &mut self,
6367 loc: Location,
6368 target: Location,
6369 memarg: &MemArg,
6370 ret: Location,
6371 need_check: bool,
6372 imported_memories: bool,
6373 offset: i32,
6374 heap_access_oob: Label,
6375 unaligned_atomic: Label,
6376 ) -> Result<(), CompileError> {
6377 self.memory_op(
6378 target,
6379 memarg,
6380 true,
6381 8,
6382 need_check,
6383 imported_memories,
6384 offset,
6385 heap_access_oob,
6386 unaligned_atomic,
6387 |this, addr| {
6388 let mut temps = vec![];
6389 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6390 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6391 })?;
6392 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6393 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6394 })?;
6395 let dst =
6396 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6397 let reread = this.get_label();
6398
6399 this.emit_label(reread)?;
6400 this.assembler
6401 .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?;
6402 this.emit_binop_or64(dst, loc, Location::GPR(tmp1))?;
6403 this.assembler.emit_stlxr(
6404 Size::S64,
6405 Location::GPR(tmp2),
6406 Location::GPR(tmp1),
6407 Location::GPR(addr),
6408 )?;
6409 this.assembler
6410 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6411 this.assembler.emit_dmb()?;
6412
6413 if dst != ret {
6414 this.move_location(Size::S64, ret, dst)?;
6415 }
6416 for r in temps {
6417 this.release_gpr(r);
6418 }
6419 this.release_gpr(tmp1);
6420 this.release_gpr(tmp2);
6421 Ok(())
6422 },
6423 )
6424 }
6425 fn i64_atomic_or_8u(
6427 &mut self,
6428 loc: Location,
6429 target: Location,
6430 memarg: &MemArg,
6431 ret: Location,
6432 need_check: bool,
6433 imported_memories: bool,
6434 offset: i32,
6435 heap_access_oob: Label,
6436 unaligned_atomic: Label,
6437 ) -> Result<(), CompileError> {
6438 self.memory_op(
6439 target,
6440 memarg,
6441 true,
6442 1,
6443 need_check,
6444 imported_memories,
6445 offset,
6446 heap_access_oob,
6447 unaligned_atomic,
6448 |this, addr| {
6449 let mut temps = vec![];
6450 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6451 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6452 })?;
6453 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6454 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6455 })?;
6456 let dst =
6457 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6458 let reread = this.get_label();
6459
6460 this.emit_label(reread)?;
6461 this.assembler
6462 .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?;
6463 this.emit_binop_or64(dst, loc, Location::GPR(tmp1))?;
6464 this.assembler.emit_stlxrb(
6465 Size::S64,
6466 Location::GPR(tmp2),
6467 Location::GPR(tmp1),
6468 Location::GPR(addr),
6469 )?;
6470 this.assembler
6471 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6472 this.assembler.emit_dmb()?;
6473
6474 if dst != ret {
6475 this.move_location(Size::S64, ret, dst)?;
6476 }
6477 for r in temps {
6478 this.release_gpr(r);
6479 }
6480 this.release_gpr(tmp1);
6481 this.release_gpr(tmp2);
6482 Ok(())
6483 },
6484 )
6485 }
6486 fn i64_atomic_or_16u(
6488 &mut self,
6489 loc: Location,
6490 target: Location,
6491 memarg: &MemArg,
6492 ret: Location,
6493 need_check: bool,
6494 imported_memories: bool,
6495 offset: i32,
6496 heap_access_oob: Label,
6497 unaligned_atomic: Label,
6498 ) -> Result<(), CompileError> {
6499 self.memory_op(
6500 target,
6501 memarg,
6502 true,
6503 2,
6504 need_check,
6505 imported_memories,
6506 offset,
6507 heap_access_oob,
6508 unaligned_atomic,
6509 |this, addr| {
6510 let mut temps = vec![];
6511 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6512 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6513 })?;
6514 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6515 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6516 })?;
6517 let dst =
6518 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6519 let reread = this.get_label();
6520
6521 this.emit_label(reread)?;
6522 this.assembler
6523 .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?;
6524 this.emit_binop_or64(dst, loc, Location::GPR(tmp1))?;
6525 this.assembler.emit_stlxrh(
6526 Size::S64,
6527 Location::GPR(tmp2),
6528 Location::GPR(tmp1),
6529 Location::GPR(addr),
6530 )?;
6531 this.assembler
6532 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6533 this.assembler.emit_dmb()?;
6534
6535 if dst != ret {
6536 this.move_location(Size::S64, ret, dst)?;
6537 }
6538 for r in temps {
6539 this.release_gpr(r);
6540 }
6541 this.release_gpr(tmp1);
6542 this.release_gpr(tmp2);
6543 Ok(())
6544 },
6545 )
6546 }
6547 fn i64_atomic_or_32u(
6549 &mut self,
6550 loc: Location,
6551 target: Location,
6552 memarg: &MemArg,
6553 ret: Location,
6554 need_check: bool,
6555 imported_memories: bool,
6556 offset: i32,
6557 heap_access_oob: Label,
6558 unaligned_atomic: Label,
6559 ) -> Result<(), CompileError> {
6560 self.memory_op(
6561 target,
6562 memarg,
6563 true,
6564 4,
6565 need_check,
6566 imported_memories,
6567 offset,
6568 heap_access_oob,
6569 unaligned_atomic,
6570 |this, addr| {
6571 let mut temps = vec![];
6572 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6573 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6574 })?;
6575 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6576 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6577 })?;
6578 let dst =
6579 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6580 let reread = this.get_label();
6581
6582 this.emit_label(reread)?;
6583 this.assembler
6584 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
6585 this.emit_binop_or64(dst, loc, Location::GPR(tmp1))?;
6586 this.assembler.emit_stlxr(
6587 Size::S32,
6588 Location::GPR(tmp2),
6589 Location::GPR(tmp1),
6590 Location::GPR(addr),
6591 )?;
6592 this.assembler
6593 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6594 this.assembler.emit_dmb()?;
6595
6596 if dst != ret {
6597 this.move_location(Size::S64, ret, dst)?;
6598 }
6599 for r in temps {
6600 this.release_gpr(r);
6601 }
6602 this.release_gpr(tmp1);
6603 this.release_gpr(tmp2);
6604 Ok(())
6605 },
6606 )
6607 }
6608 fn i64_atomic_xor(
6610 &mut self,
6611 loc: Location,
6612 target: Location,
6613 memarg: &MemArg,
6614 ret: Location,
6615 need_check: bool,
6616 imported_memories: bool,
6617 offset: i32,
6618 heap_access_oob: Label,
6619 unaligned_atomic: Label,
6620 ) -> Result<(), CompileError> {
6621 self.memory_op(
6622 target,
6623 memarg,
6624 true,
6625 8,
6626 need_check,
6627 imported_memories,
6628 offset,
6629 heap_access_oob,
6630 unaligned_atomic,
6631 |this, addr| {
6632 let mut temps = vec![];
6633 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6634 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6635 })?;
6636 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6637 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6638 })?;
6639 let dst =
6640 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6641 let reread = this.get_label();
6642
6643 this.emit_label(reread)?;
6644 this.assembler
6645 .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?;
6646 this.emit_binop_xor64(dst, loc, Location::GPR(tmp1))?;
6647 this.assembler.emit_stlxr(
6648 Size::S64,
6649 Location::GPR(tmp2),
6650 Location::GPR(tmp1),
6651 Location::GPR(addr),
6652 )?;
6653 this.assembler
6654 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6655 this.assembler.emit_dmb()?;
6656
6657 if dst != ret {
6658 this.move_location(Size::S64, ret, dst)?;
6659 }
6660 for r in temps {
6661 this.release_gpr(r);
6662 }
6663 this.release_gpr(tmp1);
6664 this.release_gpr(tmp2);
6665 Ok(())
6666 },
6667 )
6668 }
6669 fn i64_atomic_xor_8u(
6671 &mut self,
6672 loc: Location,
6673 target: Location,
6674 memarg: &MemArg,
6675 ret: Location,
6676 need_check: bool,
6677 imported_memories: bool,
6678 offset: i32,
6679 heap_access_oob: Label,
6680 unaligned_atomic: Label,
6681 ) -> Result<(), CompileError> {
6682 self.memory_op(
6683 target,
6684 memarg,
6685 true,
6686 1,
6687 need_check,
6688 imported_memories,
6689 offset,
6690 heap_access_oob,
6691 unaligned_atomic,
6692 |this, addr| {
6693 let mut temps = vec![];
6694 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6695 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6696 })?;
6697 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6698 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6699 })?;
6700 let dst =
6701 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6702 let reread = this.get_label();
6703
6704 this.emit_label(reread)?;
6705 this.assembler
6706 .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?;
6707 this.emit_binop_xor64(dst, loc, Location::GPR(tmp1))?;
6708 this.assembler.emit_stlxrb(
6709 Size::S64,
6710 Location::GPR(tmp2),
6711 Location::GPR(tmp1),
6712 Location::GPR(addr),
6713 )?;
6714 this.assembler
6715 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6716 this.assembler.emit_dmb()?;
6717
6718 if dst != ret {
6719 this.move_location(Size::S64, ret, dst)?;
6720 }
6721 for r in temps {
6722 this.release_gpr(r);
6723 }
6724 this.release_gpr(tmp1);
6725 this.release_gpr(tmp2);
6726 Ok(())
6727 },
6728 )
6729 }
6730 fn i64_atomic_xor_16u(
6732 &mut self,
6733 loc: Location,
6734 target: Location,
6735 memarg: &MemArg,
6736 ret: Location,
6737 need_check: bool,
6738 imported_memories: bool,
6739 offset: i32,
6740 heap_access_oob: Label,
6741 unaligned_atomic: Label,
6742 ) -> Result<(), CompileError> {
6743 self.memory_op(
6744 target,
6745 memarg,
6746 true,
6747 2,
6748 need_check,
6749 imported_memories,
6750 offset,
6751 heap_access_oob,
6752 unaligned_atomic,
6753 |this, addr| {
6754 let mut temps = vec![];
6755 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6756 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6757 })?;
6758 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6759 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6760 })?;
6761 let dst =
6762 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6763 let reread = this.get_label();
6764
6765 this.emit_label(reread)?;
6766 this.assembler
6767 .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?;
6768 this.emit_binop_xor64(dst, loc, Location::GPR(tmp1))?;
6769 this.assembler.emit_stlxrh(
6770 Size::S64,
6771 Location::GPR(tmp2),
6772 Location::GPR(tmp1),
6773 Location::GPR(addr),
6774 )?;
6775 this.assembler
6776 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6777 this.assembler.emit_dmb()?;
6778
6779 if dst != ret {
6780 this.move_location(Size::S64, ret, dst)?;
6781 }
6782 for r in temps {
6783 this.release_gpr(r);
6784 }
6785 this.release_gpr(tmp1);
6786 this.release_gpr(tmp2);
6787 Ok(())
6788 },
6789 )
6790 }
6791 fn i64_atomic_xor_32u(
6793 &mut self,
6794 loc: Location,
6795 target: Location,
6796 memarg: &MemArg,
6797 ret: Location,
6798 need_check: bool,
6799 imported_memories: bool,
6800 offset: i32,
6801 heap_access_oob: Label,
6802 unaligned_atomic: Label,
6803 ) -> Result<(), CompileError> {
6804 self.memory_op(
6805 target,
6806 memarg,
6807 true,
6808 4,
6809 need_check,
6810 imported_memories,
6811 offset,
6812 heap_access_oob,
6813 unaligned_atomic,
6814 |this, addr| {
6815 let mut temps = vec![];
6816 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6817 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6818 })?;
6819 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6820 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6821 })?;
6822 let dst =
6823 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6824 let reread = this.get_label();
6825
6826 this.emit_label(reread)?;
6827 this.assembler
6828 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
6829 this.emit_binop_xor64(dst, loc, Location::GPR(tmp1))?;
6830 this.assembler.emit_stlxr(
6831 Size::S32,
6832 Location::GPR(tmp2),
6833 Location::GPR(tmp1),
6834 Location::GPR(addr),
6835 )?;
6836 this.assembler
6837 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6838 this.assembler.emit_dmb()?;
6839
6840 if dst != ret {
6841 this.move_location(Size::S64, ret, dst)?;
6842 }
6843 for r in temps {
6844 this.release_gpr(r);
6845 }
6846 this.release_gpr(tmp1);
6847 this.release_gpr(tmp2);
6848 Ok(())
6849 },
6850 )
6851 }
6852 fn i64_atomic_xchg(
6854 &mut self,
6855 loc: Location,
6856 target: Location,
6857 memarg: &MemArg,
6858 ret: Location,
6859 need_check: bool,
6860 imported_memories: bool,
6861 offset: i32,
6862 heap_access_oob: Label,
6863 unaligned_atomic: Label,
6864 ) -> Result<(), CompileError> {
6865 self.memory_op(
6866 target,
6867 memarg,
6868 true,
6869 8,
6870 need_check,
6871 imported_memories,
6872 offset,
6873 heap_access_oob,
6874 unaligned_atomic,
6875 |this, addr| {
6876 let mut temps = vec![];
6877 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
6878 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6879 })?;
6880 let dst =
6881 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6882 let org =
6883 this.location_to_reg(Size::S64, loc, &mut temps, ImmType::None, false, None)?;
6884 let reread = this.get_label();
6885
6886 this.emit_label(reread)?;
6887 this.assembler
6888 .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?;
6889 this.assembler.emit_stlxr(
6890 Size::S64,
6891 Location::GPR(tmp),
6892 org,
6893 Location::GPR(addr),
6894 )?;
6895 this.assembler
6896 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
6897 this.assembler.emit_dmb()?;
6898
6899 if dst != ret {
6900 this.move_location(Size::S64, ret, dst)?;
6901 }
6902 for r in temps {
6903 this.release_gpr(r);
6904 }
6905 this.release_gpr(tmp);
6906 Ok(())
6907 },
6908 )
6909 }
6910 fn i64_atomic_xchg_8u(
6912 &mut self,
6913 loc: Location,
6914 target: Location,
6915 memarg: &MemArg,
6916 ret: Location,
6917 need_check: bool,
6918 imported_memories: bool,
6919 offset: i32,
6920 heap_access_oob: Label,
6921 unaligned_atomic: Label,
6922 ) -> Result<(), CompileError> {
6923 self.memory_op(
6924 target,
6925 memarg,
6926 true,
6927 1,
6928 need_check,
6929 imported_memories,
6930 offset,
6931 heap_access_oob,
6932 unaligned_atomic,
6933 |this, addr| {
6934 let mut temps = vec![];
6935 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
6936 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6937 })?;
6938 let dst =
6939 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6940 let org =
6941 this.location_to_reg(Size::S64, loc, &mut temps, ImmType::None, false, None)?;
6942 let reread = this.get_label();
6943
6944 this.emit_label(reread)?;
6945 this.assembler
6946 .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?;
6947 this.assembler.emit_stlxrb(
6948 Size::S64,
6949 Location::GPR(tmp),
6950 org,
6951 Location::GPR(addr),
6952 )?;
6953 this.assembler
6954 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
6955 this.assembler.emit_dmb()?;
6956
6957 if dst != ret {
6958 this.move_location(Size::S64, ret, dst)?;
6959 }
6960 for r in temps {
6961 this.release_gpr(r);
6962 }
6963 this.release_gpr(tmp);
6964 Ok(())
6965 },
6966 )
6967 }
6968 fn i64_atomic_xchg_16u(
6970 &mut self,
6971 loc: Location,
6972 target: Location,
6973 memarg: &MemArg,
6974 ret: Location,
6975 need_check: bool,
6976 imported_memories: bool,
6977 offset: i32,
6978 heap_access_oob: Label,
6979 unaligned_atomic: Label,
6980 ) -> Result<(), CompileError> {
6981 self.memory_op(
6982 target,
6983 memarg,
6984 true,
6985 2,
6986 need_check,
6987 imported_memories,
6988 offset,
6989 heap_access_oob,
6990 unaligned_atomic,
6991 |this, addr| {
6992 let mut temps = vec![];
6993 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
6994 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6995 })?;
6996 let dst =
6997 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6998 let org =
6999 this.location_to_reg(Size::S64, loc, &mut temps, ImmType::None, false, None)?;
7000 let reread = this.get_label();
7001
7002 this.emit_label(reread)?;
7003 this.assembler
7004 .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?;
7005 this.assembler.emit_stlxrh(
7006 Size::S64,
7007 Location::GPR(tmp),
7008 org,
7009 Location::GPR(addr),
7010 )?;
7011 this.assembler
7012 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7013 this.assembler.emit_dmb()?;
7014
7015 if dst != ret {
7016 this.move_location(Size::S64, ret, dst)?;
7017 }
7018 for r in temps {
7019 this.release_gpr(r);
7020 }
7021 this.release_gpr(tmp);
7022 Ok(())
7023 },
7024 )
7025 }
7026 fn i64_atomic_xchg_32u(
7028 &mut self,
7029 loc: Location,
7030 target: Location,
7031 memarg: &MemArg,
7032 ret: Location,
7033 need_check: bool,
7034 imported_memories: bool,
7035 offset: i32,
7036 heap_access_oob: Label,
7037 unaligned_atomic: Label,
7038 ) -> Result<(), CompileError> {
7039 self.memory_op(
7040 target,
7041 memarg,
7042 true,
7043 4,
7044 need_check,
7045 imported_memories,
7046 offset,
7047 heap_access_oob,
7048 unaligned_atomic,
7049 |this, addr| {
7050 let mut temps = vec![];
7051 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
7052 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7053 })?;
7054 let dst =
7055 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7056 let org =
7057 this.location_to_reg(Size::S64, loc, &mut temps, ImmType::None, false, None)?;
7058 let reread = this.get_label();
7059
7060 this.emit_label(reread)?;
7061 this.assembler
7062 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
7063 this.assembler.emit_stlxr(
7064 Size::S32,
7065 Location::GPR(tmp),
7066 org,
7067 Location::GPR(addr),
7068 )?;
7069 this.assembler
7070 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7071 this.assembler.emit_dmb()?;
7072
7073 if dst != ret {
7074 this.move_location(Size::S64, ret, dst)?;
7075 }
7076 for r in temps {
7077 this.release_gpr(r);
7078 }
7079 this.release_gpr(tmp);
7080 Ok(())
7081 },
7082 )
7083 }
7084 fn i64_atomic_cmpxchg(
7086 &mut self,
7087 new: Location,
7088 cmp: Location,
7089 target: Location,
7090 memarg: &MemArg,
7091 ret: Location,
7092 need_check: bool,
7093 imported_memories: bool,
7094 offset: i32,
7095 heap_access_oob: Label,
7096 unaligned_atomic: Label,
7097 ) -> Result<(), CompileError> {
7098 self.memory_op(
7099 target,
7100 memarg,
7101 true,
7102 8,
7103 need_check,
7104 imported_memories,
7105 offset,
7106 heap_access_oob,
7107 unaligned_atomic,
7108 |this, addr| {
7109 let mut temps = vec![];
7110 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
7111 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7112 })?;
7113 let dst =
7114 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7115 let org =
7116 this.location_to_reg(Size::S64, new, &mut temps, ImmType::None, false, None)?;
7117 let reread = this.get_label();
7118 let nosame = this.get_label();
7119
7120 this.emit_label(reread)?;
7121 this.assembler
7122 .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?;
7123 this.emit_relaxed_cmp(Size::S64, dst, cmp)?;
7124 this.assembler.emit_bcond_label(Condition::Ne, nosame)?;
7125 this.assembler.emit_stlxr(
7126 Size::S64,
7127 Location::GPR(tmp),
7128 org,
7129 Location::GPR(addr),
7130 )?;
7131 this.assembler
7132 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7133 this.assembler.emit_dmb()?;
7134
7135 this.emit_label(nosame)?;
7136 if dst != ret {
7137 this.move_location(Size::S64, ret, dst)?;
7138 }
7139 for r in temps {
7140 this.release_gpr(r);
7141 }
7142 this.release_gpr(tmp);
7143 Ok(())
7144 },
7145 )
7146 }
7147 fn i64_atomic_cmpxchg_8u(
7149 &mut self,
7150 new: Location,
7151 cmp: Location,
7152 target: Location,
7153 memarg: &MemArg,
7154 ret: Location,
7155 need_check: bool,
7156 imported_memories: bool,
7157 offset: i32,
7158 heap_access_oob: Label,
7159 unaligned_atomic: Label,
7160 ) -> Result<(), CompileError> {
7161 self.memory_op(
7162 target,
7163 memarg,
7164 true,
7165 1,
7166 need_check,
7167 imported_memories,
7168 offset,
7169 heap_access_oob,
7170 unaligned_atomic,
7171 |this, addr| {
7172 let mut temps = vec![];
7173 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
7174 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7175 })?;
7176 let dst =
7177 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7178 let org =
7179 this.location_to_reg(Size::S64, new, &mut temps, ImmType::None, false, None)?;
7180 let reread = this.get_label();
7181 let nosame = this.get_label();
7182
7183 this.emit_label(reread)?;
7184 this.assembler
7185 .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?;
7186 this.emit_relaxed_cmp(Size::S64, dst, cmp)?;
7187 this.assembler.emit_bcond_label(Condition::Ne, nosame)?;
7188 this.assembler.emit_stlxrb(
7189 Size::S64,
7190 Location::GPR(tmp),
7191 org,
7192 Location::GPR(addr),
7193 )?;
7194 this.assembler
7195 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7196 this.assembler.emit_dmb()?;
7197
7198 this.emit_label(nosame)?;
7199 if dst != ret {
7200 this.move_location(Size::S64, ret, dst)?;
7201 }
7202 for r in temps {
7203 this.release_gpr(r);
7204 }
7205 this.release_gpr(tmp);
7206 Ok(())
7207 },
7208 )
7209 }
7210 fn i64_atomic_cmpxchg_16u(
7212 &mut self,
7213 new: Location,
7214 cmp: Location,
7215 target: Location,
7216 memarg: &MemArg,
7217 ret: Location,
7218 need_check: bool,
7219 imported_memories: bool,
7220 offset: i32,
7221 heap_access_oob: Label,
7222 unaligned_atomic: Label,
7223 ) -> Result<(), CompileError> {
7224 self.memory_op(
7225 target,
7226 memarg,
7227 true,
7228 2,
7229 need_check,
7230 imported_memories,
7231 offset,
7232 heap_access_oob,
7233 unaligned_atomic,
7234 |this, addr| {
7235 let mut temps = vec![];
7236 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
7237 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7238 })?;
7239 let dst =
7240 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7241 let org =
7242 this.location_to_reg(Size::S64, new, &mut temps, ImmType::None, false, None)?;
7243 let reread = this.get_label();
7244 let nosame = this.get_label();
7245
7246 this.emit_label(reread)?;
7247 this.assembler
7248 .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?;
7249 this.emit_relaxed_cmp(Size::S64, dst, cmp)?;
7250 this.assembler.emit_bcond_label(Condition::Ne, nosame)?;
7251 this.assembler.emit_stlxrh(
7252 Size::S64,
7253 Location::GPR(tmp),
7254 org,
7255 Location::GPR(addr),
7256 )?;
7257 this.assembler
7258 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7259 this.assembler.emit_dmb()?;
7260
7261 this.emit_label(nosame)?;
7262 if dst != ret {
7263 this.move_location(Size::S64, ret, dst)?;
7264 }
7265 for r in temps {
7266 this.release_gpr(r);
7267 }
7268 this.release_gpr(tmp);
7269 Ok(())
7270 },
7271 )
7272 }
7273 fn i64_atomic_cmpxchg_32u(
7275 &mut self,
7276 new: Location,
7277 cmp: Location,
7278 target: Location,
7279 memarg: &MemArg,
7280 ret: Location,
7281 need_check: bool,
7282 imported_memories: bool,
7283 offset: i32,
7284 heap_access_oob: Label,
7285 unaligned_atomic: Label,
7286 ) -> Result<(), CompileError> {
7287 self.memory_op(
7288 target,
7289 memarg,
7290 true,
7291 4,
7292 need_check,
7293 imported_memories,
7294 offset,
7295 heap_access_oob,
7296 unaligned_atomic,
7297 |this, addr| {
7298 let mut temps = vec![];
7299 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
7300 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7301 })?;
7302 let dst =
7303 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7304 let org =
7305 this.location_to_reg(Size::S64, new, &mut temps, ImmType::None, false, None)?;
7306 let reread = this.get_label();
7307 let nosame = this.get_label();
7308
7309 this.emit_label(reread)?;
7310 this.assembler
7311 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
7312 this.emit_relaxed_cmp(Size::S64, dst, cmp)?;
7313 this.assembler.emit_bcond_label(Condition::Ne, nosame)?;
7314 this.assembler.emit_stlxr(
7315 Size::S32,
7316 Location::GPR(tmp),
7317 org,
7318 Location::GPR(addr),
7319 )?;
7320 this.assembler
7321 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7322 this.assembler.emit_dmb()?;
7323
7324 this.emit_label(nosame)?;
7325 if dst != ret {
7326 this.move_location(Size::S64, ret, dst)?;
7327 }
7328 for r in temps {
7329 this.release_gpr(r);
7330 }
7331 this.release_gpr(tmp);
7332 Ok(())
7333 },
7334 )
7335 }
7336
7337 fn f32_load(
7338 &mut self,
7339 addr: Location,
7340 memarg: &MemArg,
7341 ret: Location,
7342 need_check: bool,
7343 imported_memories: bool,
7344 offset: i32,
7345 heap_access_oob: Label,
7346 unaligned_atomic: Label,
7347 ) -> Result<(), CompileError> {
7348 self.memory_op(
7349 addr,
7350 memarg,
7351 false,
7352 4,
7353 need_check,
7354 imported_memories,
7355 offset,
7356 heap_access_oob,
7357 unaligned_atomic,
7358 |this, addr| this.emit_relaxed_ldr32(Size::S32, ret, Location::Memory(addr, 0)),
7359 )
7360 }
7361 fn f32_save(
7362 &mut self,
7363 target_value: Location,
7364 memarg: &MemArg,
7365 target_addr: Location,
7366 canonicalize: bool,
7367 need_check: bool,
7368 imported_memories: bool,
7369 offset: i32,
7370 heap_access_oob: Label,
7371 unaligned_atomic: Label,
7372 ) -> Result<(), CompileError> {
7373 self.memory_op(
7374 target_addr,
7375 memarg,
7376 false,
7377 4,
7378 need_check,
7379 imported_memories,
7380 offset,
7381 heap_access_oob,
7382 unaligned_atomic,
7383 |this, addr| {
7384 if !canonicalize {
7385 this.emit_relaxed_str32(target_value, Location::Memory(addr, 0))
7386 } else {
7387 this.canonicalize_nan(Size::S32, target_value, Location::Memory(addr, 0))
7388 }
7389 },
7390 )
7391 }
7392 fn f64_load(
7393 &mut self,
7394 addr: Location,
7395 memarg: &MemArg,
7396 ret: Location,
7397 need_check: bool,
7398 imported_memories: bool,
7399 offset: i32,
7400 heap_access_oob: Label,
7401 unaligned_atomic: Label,
7402 ) -> Result<(), CompileError> {
7403 self.memory_op(
7404 addr,
7405 memarg,
7406 false,
7407 8,
7408 need_check,
7409 imported_memories,
7410 offset,
7411 heap_access_oob,
7412 unaligned_atomic,
7413 |this, addr| this.emit_relaxed_ldr64(Size::S64, ret, Location::Memory(addr, 0)),
7414 )
7415 }
7416 fn f64_save(
7417 &mut self,
7418 target_value: Location,
7419 memarg: &MemArg,
7420 target_addr: Location,
7421 canonicalize: bool,
7422 need_check: bool,
7423 imported_memories: bool,
7424 offset: i32,
7425 heap_access_oob: Label,
7426 unaligned_atomic: Label,
7427 ) -> Result<(), CompileError> {
7428 self.memory_op(
7429 target_addr,
7430 memarg,
7431 false,
7432 8,
7433 need_check,
7434 imported_memories,
7435 offset,
7436 heap_access_oob,
7437 unaligned_atomic,
7438 |this, addr| {
7439 if !canonicalize {
7440 this.emit_relaxed_str64(target_value, Location::Memory(addr, 0))
7441 } else {
7442 this.canonicalize_nan(Size::S64, target_value, Location::Memory(addr, 0))
7443 }
7444 },
7445 )
7446 }
7447
7448 fn convert_f64_i64(
7449 &mut self,
7450 loc: Location,
7451 signed: bool,
7452 ret: Location,
7453 ) -> Result<(), CompileError> {
7454 let mut gprs = vec![];
7455 let mut neons = vec![];
7456 let src = self.location_to_reg(Size::S64, loc, &mut gprs, ImmType::NoneXzr, true, None)?;
7457 let dest = self.location_to_neon(Size::S64, ret, &mut neons, ImmType::None, false)?;
7458 if signed {
7459 self.assembler.emit_scvtf(Size::S64, src, Size::S64, dest)?;
7460 } else {
7461 self.assembler.emit_ucvtf(Size::S64, src, Size::S64, dest)?;
7462 }
7463 if ret != dest {
7464 self.move_location(Size::S64, dest, ret)?;
7465 }
7466 for r in gprs {
7467 self.release_gpr(r);
7468 }
7469 for r in neons {
7470 self.release_simd(r);
7471 }
7472 Ok(())
7473 }
7474 fn convert_f64_i32(
7475 &mut self,
7476 loc: Location,
7477 signed: bool,
7478 ret: Location,
7479 ) -> Result<(), CompileError> {
7480 let mut gprs = vec![];
7481 let mut neons = vec![];
7482 let src = self.location_to_reg(Size::S32, loc, &mut gprs, ImmType::NoneXzr, true, None)?;
7483 let dest = self.location_to_neon(Size::S64, ret, &mut neons, ImmType::None, false)?;
7484 if signed {
7485 self.assembler.emit_scvtf(Size::S32, src, Size::S64, dest)?;
7486 } else {
7487 self.assembler.emit_ucvtf(Size::S32, src, Size::S64, dest)?;
7488 }
7489 if ret != dest {
7490 self.move_location(Size::S64, dest, ret)?;
7491 }
7492 for r in gprs {
7493 self.release_gpr(r);
7494 }
7495 for r in neons {
7496 self.release_simd(r);
7497 }
7498 Ok(())
7499 }
7500 fn convert_f32_i64(
7501 &mut self,
7502 loc: Location,
7503 signed: bool,
7504 ret: Location,
7505 ) -> Result<(), CompileError> {
7506 let mut gprs = vec![];
7507 let mut neons = vec![];
7508 let src = self.location_to_reg(Size::S64, loc, &mut gprs, ImmType::NoneXzr, true, None)?;
7509 let dest = self.location_to_neon(Size::S32, ret, &mut neons, ImmType::None, false)?;
7510 if signed {
7511 self.assembler.emit_scvtf(Size::S64, src, Size::S32, dest)?;
7512 } else {
7513 self.assembler.emit_ucvtf(Size::S64, src, Size::S32, dest)?;
7514 }
7515 if ret != dest {
7516 self.move_location(Size::S32, dest, ret)?;
7517 }
7518 for r in gprs {
7519 self.release_gpr(r);
7520 }
7521 for r in neons {
7522 self.release_simd(r);
7523 }
7524 Ok(())
7525 }
7526 fn convert_f32_i32(
7527 &mut self,
7528 loc: Location,
7529 signed: bool,
7530 ret: Location,
7531 ) -> Result<(), CompileError> {
7532 let mut gprs = vec![];
7533 let mut neons = vec![];
7534 let src = self.location_to_reg(Size::S32, loc, &mut gprs, ImmType::NoneXzr, true, None)?;
7535 let dest = self.location_to_neon(Size::S32, ret, &mut neons, ImmType::None, false)?;
7536 if signed {
7537 self.assembler.emit_scvtf(Size::S32, src, Size::S32, dest)?;
7538 } else {
7539 self.assembler.emit_ucvtf(Size::S32, src, Size::S32, dest)?;
7540 }
7541 if ret != dest {
7542 self.move_location(Size::S32, dest, ret)?;
7543 }
7544 for r in gprs {
7545 self.release_gpr(r);
7546 }
7547 for r in neons {
7548 self.release_simd(r);
7549 }
7550 Ok(())
7551 }
7552 fn convert_i64_f64(
7553 &mut self,
7554 loc: Location,
7555 ret: Location,
7556 signed: bool,
7557 sat: bool,
7558 ) -> Result<(), CompileError> {
7559 let mut gprs = vec![];
7560 let mut neons = vec![];
7561 let src = self.location_to_neon(Size::S64, loc, &mut neons, ImmType::None, true)?;
7562 let dest = self.location_to_reg(Size::S64, ret, &mut gprs, ImmType::None, false, None)?;
7563 let old_fpcr = if !sat {
7564 self.reset_exception_fpsr()?;
7565 self.set_trap_enabled(&mut gprs)?
7566 } else {
7567 GPR::XzrSp
7568 };
7569 if signed {
7570 self.assembler
7571 .emit_fcvtzs(Size::S64, src, Size::S64, dest)?;
7572 } else {
7573 self.assembler
7574 .emit_fcvtzu(Size::S64, src, Size::S64, dest)?;
7575 }
7576 if !sat {
7577 self.trap_float_convertion_errors(old_fpcr, Size::S64, src, &mut gprs)?;
7578 }
7579 if ret != dest {
7580 self.move_location(Size::S64, dest, ret)?;
7581 }
7582 for r in gprs {
7583 self.release_gpr(r);
7584 }
7585 for r in neons {
7586 self.release_simd(r);
7587 }
7588 Ok(())
7589 }
7590 fn convert_i32_f64(
7591 &mut self,
7592 loc: Location,
7593 ret: Location,
7594 signed: bool,
7595 sat: bool,
7596 ) -> Result<(), CompileError> {
7597 let mut gprs = vec![];
7598 let mut neons = vec![];
7599 let src = self.location_to_neon(Size::S64, loc, &mut neons, ImmType::None, true)?;
7600 let dest = self.location_to_reg(Size::S32, ret, &mut gprs, ImmType::None, false, None)?;
7601 let old_fpcr = if !sat {
7602 self.reset_exception_fpsr()?;
7603 self.set_trap_enabled(&mut gprs)?
7604 } else {
7605 GPR::XzrSp
7606 };
7607 if signed {
7608 self.assembler
7609 .emit_fcvtzs(Size::S64, src, Size::S32, dest)?;
7610 } else {
7611 self.assembler
7612 .emit_fcvtzu(Size::S64, src, Size::S32, dest)?;
7613 }
7614 if !sat {
7615 self.trap_float_convertion_errors(old_fpcr, Size::S64, src, &mut gprs)?;
7616 }
7617 if ret != dest {
7618 self.move_location(Size::S32, dest, ret)?;
7619 }
7620 for r in gprs {
7621 self.release_gpr(r);
7622 }
7623 for r in neons {
7624 self.release_simd(r);
7625 }
7626 Ok(())
7627 }
7628 fn convert_i64_f32(
7629 &mut self,
7630 loc: Location,
7631 ret: Location,
7632 signed: bool,
7633 sat: bool,
7634 ) -> Result<(), CompileError> {
7635 let mut gprs = vec![];
7636 let mut neons = vec![];
7637 let src = self.location_to_neon(Size::S32, loc, &mut neons, ImmType::None, true)?;
7638 let dest = self.location_to_reg(Size::S64, ret, &mut gprs, ImmType::None, false, None)?;
7639 let old_fpcr = if !sat {
7640 self.reset_exception_fpsr()?;
7641 self.set_trap_enabled(&mut gprs)?
7642 } else {
7643 GPR::XzrSp
7644 };
7645 if signed {
7646 self.assembler
7647 .emit_fcvtzs(Size::S32, src, Size::S64, dest)?;
7648 } else {
7649 self.assembler
7650 .emit_fcvtzu(Size::S32, src, Size::S64, dest)?;
7651 }
7652 if !sat {
7653 self.trap_float_convertion_errors(old_fpcr, Size::S32, src, &mut gprs)?;
7654 }
7655 if ret != dest {
7656 self.move_location(Size::S64, dest, ret)?;
7657 }
7658 for r in gprs {
7659 self.release_gpr(r);
7660 }
7661 for r in neons {
7662 self.release_simd(r);
7663 }
7664 Ok(())
7665 }
7666 fn convert_i32_f32(
7667 &mut self,
7668 loc: Location,
7669 ret: Location,
7670 signed: bool,
7671 sat: bool,
7672 ) -> Result<(), CompileError> {
7673 let mut gprs = vec![];
7674 let mut neons = vec![];
7675 let src = self.location_to_neon(Size::S32, loc, &mut neons, ImmType::None, true)?;
7676 let dest = self.location_to_reg(Size::S32, ret, &mut gprs, ImmType::None, false, None)?;
7677 let old_fpcr = if !sat {
7678 self.reset_exception_fpsr()?;
7679 self.set_trap_enabled(&mut gprs)?
7680 } else {
7681 GPR::XzrSp
7682 };
7683 if signed {
7684 self.assembler
7685 .emit_fcvtzs(Size::S32, src, Size::S32, dest)?;
7686 } else {
7687 self.assembler
7688 .emit_fcvtzu(Size::S32, src, Size::S32, dest)?;
7689 }
7690 if !sat {
7691 self.trap_float_convertion_errors(old_fpcr, Size::S32, src, &mut gprs)?;
7692 }
7693 if ret != dest {
7694 self.move_location(Size::S32, dest, ret)?;
7695 }
7696 for r in gprs {
7697 self.release_gpr(r);
7698 }
7699 for r in neons {
7700 self.release_simd(r);
7701 }
7702 Ok(())
7703 }
7704 fn convert_f64_f32(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7705 self.emit_relaxed_binop_neon(Assembler::emit_fcvt, Size::S32, loc, ret, true)
7706 }
7707 fn convert_f32_f64(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7708 self.emit_relaxed_binop_neon(Assembler::emit_fcvt, Size::S64, loc, ret, true)
7709 }
7710 fn f64_neg(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7711 self.emit_relaxed_binop_neon(Assembler::emit_fneg, Size::S64, loc, ret, true)
7712 }
7713 fn f64_abs(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7714 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
7715 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7716 })?;
7717
7718 self.move_location(Size::S64, loc, Location::GPR(tmp))?;
7719 self.assembler.emit_and(
7720 Size::S64,
7721 Location::GPR(tmp),
7722 Location::Imm64(0x7fffffffffffffffu64),
7723 Location::GPR(tmp),
7724 )?;
7725 self.move_location(Size::S64, Location::GPR(tmp), ret)?;
7726
7727 self.release_gpr(tmp);
7728 Ok(())
7729 }
7730 fn emit_i64_copysign(&mut self, tmp1: GPR, tmp2: GPR) -> Result<(), CompileError> {
7731 self.assembler.emit_and(
7732 Size::S64,
7733 Location::GPR(tmp1),
7734 Location::Imm64(0x7fffffffffffffffu64),
7735 Location::GPR(tmp1),
7736 )?;
7737
7738 self.assembler.emit_and(
7739 Size::S64,
7740 Location::GPR(tmp2),
7741 Location::Imm64(0x8000000000000000u64),
7742 Location::GPR(tmp2),
7743 )?;
7744
7745 self.assembler.emit_or(
7746 Size::S64,
7747 Location::GPR(tmp1),
7748 Location::GPR(tmp2),
7749 Location::GPR(tmp1),
7750 )
7751 }
7752 fn f64_sqrt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7753 self.emit_relaxed_binop_neon(Assembler::emit_fsqrt, Size::S64, loc, ret, true)
7754 }
7755 fn f64_trunc(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7756 self.emit_relaxed_binop_neon(Assembler::emit_frintz, Size::S64, loc, ret, true)
7757 }
7758 fn f64_ceil(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7759 self.emit_relaxed_binop_neon(Assembler::emit_frintp, Size::S64, loc, ret, true)
7760 }
7761 fn f64_floor(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7762 self.emit_relaxed_binop_neon(Assembler::emit_frintm, Size::S64, loc, ret, true)
7763 }
7764 fn f64_nearest(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7765 self.emit_relaxed_binop_neon(Assembler::emit_frintn, Size::S64, loc, ret, true)
7766 }
7767 fn f64_cmp_ge(
7768 &mut self,
7769 loc_a: Location,
7770 loc_b: Location,
7771 ret: Location,
7772 ) -> Result<(), CompileError> {
7773 let mut temps = vec![];
7774 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7775 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S64, loc_b, loc_a, false)?;
7776 self.assembler.emit_cset(Size::S32, dest, Condition::Ls)?;
7777 if ret != dest {
7778 self.move_location(Size::S32, dest, ret)?;
7779 }
7780 for r in temps {
7781 self.release_gpr(r);
7782 }
7783 Ok(())
7784 }
7785 fn f64_cmp_gt(
7786 &mut self,
7787 loc_a: Location,
7788 loc_b: Location,
7789 ret: Location,
7790 ) -> Result<(), CompileError> {
7791 let mut temps = vec![];
7792 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7793 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S64, loc_b, loc_a, false)?;
7794 self.assembler.emit_cset(Size::S32, dest, Condition::Cc)?;
7795 if ret != dest {
7796 self.move_location(Size::S32, dest, ret)?;
7797 }
7798 for r in temps {
7799 self.release_gpr(r);
7800 }
7801 Ok(())
7802 }
7803 fn f64_cmp_le(
7804 &mut self,
7805 loc_a: Location,
7806 loc_b: Location,
7807 ret: Location,
7808 ) -> Result<(), CompileError> {
7809 let mut temps = vec![];
7810 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7811 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S64, loc_a, loc_b, false)?;
7812 self.assembler.emit_cset(Size::S32, dest, Condition::Ls)?;
7813 if ret != dest {
7814 self.move_location(Size::S32, dest, ret)?;
7815 }
7816 for r in temps {
7817 self.release_gpr(r);
7818 }
7819 Ok(())
7820 }
7821 fn f64_cmp_lt(
7822 &mut self,
7823 loc_a: Location,
7824 loc_b: Location,
7825 ret: Location,
7826 ) -> Result<(), CompileError> {
7827 let mut temps = vec![];
7828 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7829 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S64, loc_a, loc_b, false)?;
7830 self.assembler.emit_cset(Size::S32, dest, Condition::Cc)?;
7831 if ret != dest {
7832 self.move_location(Size::S32, dest, ret)?;
7833 }
7834 for r in temps {
7835 self.release_gpr(r);
7836 }
7837 Ok(())
7838 }
7839 fn f64_cmp_ne(
7840 &mut self,
7841 loc_a: Location,
7842 loc_b: Location,
7843 ret: Location,
7844 ) -> Result<(), CompileError> {
7845 let mut temps = vec![];
7846 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7847 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S64, loc_a, loc_b, false)?;
7848 self.assembler.emit_cset(Size::S32, dest, Condition::Ne)?;
7849 if ret != dest {
7850 self.move_location(Size::S32, dest, ret)?;
7851 }
7852 for r in temps {
7853 self.release_gpr(r);
7854 }
7855 Ok(())
7856 }
7857 fn f64_cmp_eq(
7858 &mut self,
7859 loc_a: Location,
7860 loc_b: Location,
7861 ret: Location,
7862 ) -> Result<(), CompileError> {
7863 let mut temps = vec![];
7864 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7865 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S64, loc_a, loc_b, false)?;
7866 self.assembler.emit_cset(Size::S32, dest, Condition::Eq)?;
7867 if ret != dest {
7868 self.move_location(Size::S32, dest, ret)?;
7869 }
7870 for r in temps {
7871 self.release_gpr(r);
7872 }
7873 Ok(())
7874 }
7875 fn f64_min(
7876 &mut self,
7877 loc_a: Location,
7878 loc_b: Location,
7879 ret: Location,
7880 ) -> Result<(), CompileError> {
7881 let mut temps = vec![];
7882 let old_fpcr = self.set_default_nan(&mut temps)?;
7883 self.emit_relaxed_binop3_neon(
7884 Assembler::emit_fmin,
7885 Size::S64,
7886 loc_a,
7887 loc_b,
7888 ret,
7889 ImmType::None,
7890 )?;
7891 self.restore_fpcr(old_fpcr)?;
7892 for r in temps {
7893 self.release_gpr(r);
7894 }
7895 Ok(())
7896 }
7897 fn f64_max(
7898 &mut self,
7899 loc_a: Location,
7900 loc_b: Location,
7901 ret: Location,
7902 ) -> Result<(), CompileError> {
7903 let mut temps = vec![];
7904 let old_fpcr = self.set_default_nan(&mut temps)?;
7905 self.emit_relaxed_binop3_neon(
7906 Assembler::emit_fmax,
7907 Size::S64,
7908 loc_a,
7909 loc_b,
7910 ret,
7911 ImmType::None,
7912 )?;
7913 self.restore_fpcr(old_fpcr)?;
7914 for r in temps {
7915 self.release_gpr(r);
7916 }
7917 Ok(())
7918 }
7919 fn f64_add(
7920 &mut self,
7921 loc_a: Location,
7922 loc_b: Location,
7923 ret: Location,
7924 ) -> Result<(), CompileError> {
7925 self.emit_relaxed_binop3_neon(
7926 Assembler::emit_fadd,
7927 Size::S64,
7928 loc_a,
7929 loc_b,
7930 ret,
7931 ImmType::None,
7932 )
7933 }
7934 fn f64_sub(
7935 &mut self,
7936 loc_a: Location,
7937 loc_b: Location,
7938 ret: Location,
7939 ) -> Result<(), CompileError> {
7940 self.emit_relaxed_binop3_neon(
7941 Assembler::emit_fsub,
7942 Size::S64,
7943 loc_a,
7944 loc_b,
7945 ret,
7946 ImmType::None,
7947 )
7948 }
7949 fn f64_mul(
7950 &mut self,
7951 loc_a: Location,
7952 loc_b: Location,
7953 ret: Location,
7954 ) -> Result<(), CompileError> {
7955 self.emit_relaxed_binop3_neon(
7956 Assembler::emit_fmul,
7957 Size::S64,
7958 loc_a,
7959 loc_b,
7960 ret,
7961 ImmType::None,
7962 )
7963 }
7964 fn f64_div(
7965 &mut self,
7966 loc_a: Location,
7967 loc_b: Location,
7968 ret: Location,
7969 ) -> Result<(), CompileError> {
7970 self.emit_relaxed_binop3_neon(
7971 Assembler::emit_fdiv,
7972 Size::S64,
7973 loc_a,
7974 loc_b,
7975 ret,
7976 ImmType::None,
7977 )
7978 }
7979 fn f32_neg(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7980 self.emit_relaxed_binop_neon(Assembler::emit_fneg, Size::S32, loc, ret, true)
7981 }
7982 fn f32_abs(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7983 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
7984 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7985 })?;
7986 self.move_location(Size::S32, loc, Location::GPR(tmp))?;
7987 self.assembler.emit_and(
7988 Size::S32,
7989 Location::GPR(tmp),
7990 Location::Imm32(0x7fffffffu32),
7991 Location::GPR(tmp),
7992 )?;
7993 self.move_location(Size::S32, Location::GPR(tmp), ret)?;
7994 self.release_gpr(tmp);
7995 Ok(())
7996 }
7997 fn emit_i32_copysign(&mut self, tmp1: GPR, tmp2: GPR) -> Result<(), CompileError> {
7998 self.assembler.emit_and(
7999 Size::S32,
8000 Location::GPR(tmp1),
8001 Location::Imm32(0x7fffffffu32),
8002 Location::GPR(tmp1),
8003 )?;
8004 self.assembler.emit_and(
8005 Size::S32,
8006 Location::GPR(tmp2),
8007 Location::Imm32(0x80000000u32),
8008 Location::GPR(tmp2),
8009 )?;
8010 self.assembler.emit_or(
8011 Size::S32,
8012 Location::GPR(tmp1),
8013 Location::GPR(tmp2),
8014 Location::GPR(tmp1),
8015 )
8016 }
8017 fn f32_sqrt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
8018 self.emit_relaxed_binop_neon(Assembler::emit_fsqrt, Size::S32, loc, ret, true)
8019 }
8020 fn f32_trunc(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
8021 self.emit_relaxed_binop_neon(Assembler::emit_frintz, Size::S32, loc, ret, true)
8022 }
8023 fn f32_ceil(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
8024 self.emit_relaxed_binop_neon(Assembler::emit_frintp, Size::S32, loc, ret, true)
8025 }
8026 fn f32_floor(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
8027 self.emit_relaxed_binop_neon(Assembler::emit_frintm, Size::S32, loc, ret, true)
8028 }
8029 fn f32_nearest(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
8030 self.emit_relaxed_binop_neon(Assembler::emit_frintn, Size::S32, loc, ret, true)
8031 }
8032 fn f32_cmp_ge(
8033 &mut self,
8034 loc_a: Location,
8035 loc_b: Location,
8036 ret: Location,
8037 ) -> Result<(), CompileError> {
8038 let mut temps = vec![];
8039 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
8040 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S32, loc_b, loc_a, false)?;
8041 self.assembler.emit_cset(Size::S32, dest, Condition::Ls)?;
8042 if ret != dest {
8043 self.move_location(Size::S32, dest, ret)?;
8044 }
8045 for r in temps {
8046 self.release_gpr(r);
8047 }
8048 Ok(())
8049 }
8050 fn f32_cmp_gt(
8051 &mut self,
8052 loc_a: Location,
8053 loc_b: Location,
8054 ret: Location,
8055 ) -> Result<(), CompileError> {
8056 let mut temps = vec![];
8057 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
8058 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S32, loc_b, loc_a, false)?;
8059 self.assembler.emit_cset(Size::S32, dest, Condition::Cc)?;
8060 if ret != dest {
8061 self.move_location(Size::S32, dest, ret)?;
8062 }
8063 for r in temps {
8064 self.release_gpr(r);
8065 }
8066 Ok(())
8067 }
8068 fn f32_cmp_le(
8069 &mut self,
8070 loc_a: Location,
8071 loc_b: Location,
8072 ret: Location,
8073 ) -> Result<(), CompileError> {
8074 let mut temps = vec![];
8075 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
8076 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S32, loc_a, loc_b, false)?;
8077 self.assembler.emit_cset(Size::S32, dest, Condition::Ls)?;
8078 if ret != dest {
8079 self.move_location(Size::S32, dest, ret)?;
8080 }
8081 for r in temps {
8082 self.release_gpr(r);
8083 }
8084 Ok(())
8085 }
8086 fn f32_cmp_lt(
8087 &mut self,
8088 loc_a: Location,
8089 loc_b: Location,
8090 ret: Location,
8091 ) -> Result<(), CompileError> {
8092 let mut temps = vec![];
8093 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
8094 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S32, loc_a, loc_b, false)?;
8095 self.assembler.emit_cset(Size::S32, dest, Condition::Cc)?;
8096 if ret != dest {
8097 self.move_location(Size::S32, dest, ret)?;
8098 }
8099 for r in temps {
8100 self.release_gpr(r);
8101 }
8102 Ok(())
8103 }
8104 fn f32_cmp_ne(
8105 &mut self,
8106 loc_a: Location,
8107 loc_b: Location,
8108 ret: Location,
8109 ) -> Result<(), CompileError> {
8110 let mut temps = vec![];
8111 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
8112 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S32, loc_a, loc_b, false)?;
8113 self.assembler.emit_cset(Size::S32, dest, Condition::Ne)?;
8114 if ret != dest {
8115 self.move_location(Size::S32, dest, ret)?;
8116 }
8117 for r in temps {
8118 self.release_gpr(r);
8119 }
8120 Ok(())
8121 }
8122 fn f32_cmp_eq(
8123 &mut self,
8124 loc_a: Location,
8125 loc_b: Location,
8126 ret: Location,
8127 ) -> Result<(), CompileError> {
8128 let mut temps = vec![];
8129 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
8130 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S32, loc_a, loc_b, false)?;
8131 self.assembler.emit_cset(Size::S32, dest, Condition::Eq)?;
8132 if ret != dest {
8133 self.move_location(Size::S32, dest, ret)?;
8134 }
8135 for r in temps {
8136 self.release_gpr(r);
8137 }
8138 Ok(())
8139 }
8140 fn f32_min(
8141 &mut self,
8142 loc_a: Location,
8143 loc_b: Location,
8144 ret: Location,
8145 ) -> Result<(), CompileError> {
8146 let mut temps = vec![];
8147 let old_fpcr = self.set_default_nan(&mut temps)?;
8148 self.emit_relaxed_binop3_neon(
8149 Assembler::emit_fmin,
8150 Size::S32,
8151 loc_a,
8152 loc_b,
8153 ret,
8154 ImmType::None,
8155 )?;
8156 self.restore_fpcr(old_fpcr)?;
8157 for r in temps {
8158 self.release_gpr(r);
8159 }
8160 Ok(())
8161 }
8162 fn f32_max(
8163 &mut self,
8164 loc_a: Location,
8165 loc_b: Location,
8166 ret: Location,
8167 ) -> Result<(), CompileError> {
8168 let mut temps = vec![];
8169 let old_fpcr = self.set_default_nan(&mut temps)?;
8170 self.emit_relaxed_binop3_neon(
8171 Assembler::emit_fmax,
8172 Size::S32,
8173 loc_a,
8174 loc_b,
8175 ret,
8176 ImmType::None,
8177 )?;
8178 self.restore_fpcr(old_fpcr)?;
8179 for r in temps {
8180 self.release_gpr(r);
8181 }
8182 Ok(())
8183 }
8184 fn f32_add(
8185 &mut self,
8186 loc_a: Location,
8187 loc_b: Location,
8188 ret: Location,
8189 ) -> Result<(), CompileError> {
8190 self.emit_relaxed_binop3_neon(
8191 Assembler::emit_fadd,
8192 Size::S32,
8193 loc_a,
8194 loc_b,
8195 ret,
8196 ImmType::None,
8197 )
8198 }
8199 fn f32_sub(
8200 &mut self,
8201 loc_a: Location,
8202 loc_b: Location,
8203 ret: Location,
8204 ) -> Result<(), CompileError> {
8205 self.emit_relaxed_binop3_neon(
8206 Assembler::emit_fsub,
8207 Size::S32,
8208 loc_a,
8209 loc_b,
8210 ret,
8211 ImmType::None,
8212 )
8213 }
8214 fn f32_mul(
8215 &mut self,
8216 loc_a: Location,
8217 loc_b: Location,
8218 ret: Location,
8219 ) -> Result<(), CompileError> {
8220 self.emit_relaxed_binop3_neon(
8221 Assembler::emit_fmul,
8222 Size::S32,
8223 loc_a,
8224 loc_b,
8225 ret,
8226 ImmType::None,
8227 )
8228 }
8229 fn f32_div(
8230 &mut self,
8231 loc_a: Location,
8232 loc_b: Location,
8233 ret: Location,
8234 ) -> Result<(), CompileError> {
8235 self.emit_relaxed_binop3_neon(
8236 Assembler::emit_fdiv,
8237 Size::S32,
8238 loc_a,
8239 loc_b,
8240 ret,
8241 ImmType::None,
8242 )
8243 }
8244
8245 fn gen_std_trampoline(
8246 &self,
8247 sig: &FunctionType,
8248 calling_convention: CallingConvention,
8249 ) -> Result<FunctionBody, CompileError> {
8250 gen_std_trampoline_arm64(sig, calling_convention)
8251 }
8252 fn gen_std_dynamic_import_trampoline(
8254 &self,
8255 vmoffsets: &VMOffsets,
8256 sig: &FunctionType,
8257 calling_convention: CallingConvention,
8258 ) -> Result<FunctionBody, CompileError> {
8259 gen_std_dynamic_import_trampoline_arm64(vmoffsets, sig, calling_convention)
8260 }
8261 fn gen_import_call_trampoline(
8263 &self,
8264 vmoffsets: &VMOffsets,
8265 index: FunctionIndex,
8266 sig: &FunctionType,
8267 calling_convention: CallingConvention,
8268 ) -> Result<CustomSection, CompileError> {
8269 gen_import_call_trampoline_arm64(vmoffsets, index, sig, calling_convention)
8270 }
8271 #[cfg(feature = "unwind")]
8272 fn gen_dwarf_unwind_info(&mut self, code_len: usize) -> Option<UnwindInstructions> {
8273 let mut instructions = vec![];
8274 for &(instruction_offset, ref inst) in &self.unwind_ops {
8275 let instruction_offset = instruction_offset as u32;
8276 match *inst {
8277 UnwindOps::PushFP { up_to_sp } => {
8278 instructions.push((
8279 instruction_offset,
8280 CallFrameInstruction::CfaOffset(up_to_sp as i32),
8281 ));
8282 instructions.push((
8283 instruction_offset,
8284 CallFrameInstruction::Offset(AArch64::X29, -(up_to_sp as i32)),
8285 ));
8286 }
8287 UnwindOps::Push2Regs {
8288 reg1,
8289 reg2,
8290 up_to_sp,
8291 } => {
8292 instructions.push((
8293 instruction_offset,
8294 CallFrameInstruction::CfaOffset(up_to_sp as i32),
8295 ));
8296 instructions.push((
8297 instruction_offset,
8298 CallFrameInstruction::Offset(reg2.dwarf_index(), -(up_to_sp as i32) + 8),
8299 ));
8300 instructions.push((
8301 instruction_offset,
8302 CallFrameInstruction::Offset(reg1.dwarf_index(), -(up_to_sp as i32)),
8303 ));
8304 }
8305 UnwindOps::DefineNewFrame => {
8306 instructions.push((
8307 instruction_offset,
8308 CallFrameInstruction::CfaRegister(AArch64::X29),
8309 ));
8310 }
8311 UnwindOps::SaveRegister { reg, bp_neg_offset } => instructions.push((
8312 instruction_offset,
8313 CallFrameInstruction::Offset(reg.dwarf_index(), -bp_neg_offset),
8314 )),
8315 }
8316 }
8317 Some(UnwindInstructions {
8318 instructions,
8319 len: code_len as u32,
8320 })
8321 }
8322 #[cfg(not(feature = "unwind"))]
8323 fn gen_dwarf_unwind_info(&mut self, _code_len: usize) -> Option<UnwindInstructions> {
8324 None
8325 }
8326
8327 fn gen_windows_unwind_info(&mut self, _code_len: usize) -> Option<Vec<u8>> {
8328 None
8329 }
8330}
8331
8332#[cfg(test)]
8333mod test {
8334 use super::*;
8335
8336 fn test_move_location(machine: &mut MachineARM64, size: Size) -> Result<(), CompileError> {
8337 machine.move_location(size, Location::GPR(GPR::X1), Location::GPR(GPR::X2))?;
8338 machine.move_location(size, Location::GPR(GPR::X1), Location::Memory(GPR::X2, 10))?;
8339 machine.move_location(size, Location::GPR(GPR::X1), Location::Memory(GPR::X2, -10))?;
8340 machine.move_location(
8341 size,
8342 Location::GPR(GPR::X1),
8343 Location::Memory(GPR::X2, 1024),
8344 )?;
8345 machine.move_location(
8346 size,
8347 Location::GPR(GPR::X1),
8348 Location::Memory(GPR::X2, -1024),
8349 )?;
8350 machine.move_location(size, Location::Memory(GPR::X2, 10), Location::GPR(GPR::X1))?;
8351 machine.move_location(size, Location::Memory(GPR::X2, -10), Location::GPR(GPR::X1))?;
8352 machine.move_location(
8353 size,
8354 Location::Memory(GPR::X2, 1024),
8355 Location::GPR(GPR::X1),
8356 )?;
8357 machine.move_location(
8358 size,
8359 Location::Memory(GPR::X2, -1024),
8360 Location::GPR(GPR::X1),
8361 )?;
8362 machine.move_location(size, Location::GPR(GPR::X1), Location::SIMD(NEON::V0))?;
8363 machine.move_location(size, Location::SIMD(NEON::V0), Location::GPR(GPR::X1))?;
8364 machine.move_location(
8365 size,
8366 Location::SIMD(NEON::V0),
8367 Location::Memory(GPR::X2, 10),
8368 )?;
8369 machine.move_location(
8370 size,
8371 Location::SIMD(NEON::V0),
8372 Location::Memory(GPR::X2, -10),
8373 )?;
8374 machine.move_location(
8375 size,
8376 Location::SIMD(NEON::V0),
8377 Location::Memory(GPR::X2, 1024),
8378 )?;
8379 machine.move_location(
8380 size,
8381 Location::SIMD(NEON::V0),
8382 Location::Memory(GPR::X2, -1024),
8383 )?;
8384 machine.move_location(
8385 size,
8386 Location::Memory(GPR::X2, 10),
8387 Location::SIMD(NEON::V0),
8388 )?;
8389 machine.move_location(
8390 size,
8391 Location::Memory(GPR::X2, -10),
8392 Location::SIMD(NEON::V0),
8393 )?;
8394 machine.move_location(
8395 size,
8396 Location::Memory(GPR::X2, 1024),
8397 Location::SIMD(NEON::V0),
8398 )?;
8399 machine.move_location(
8400 size,
8401 Location::Memory(GPR::X2, -1024),
8402 Location::SIMD(NEON::V0),
8403 )?;
8404
8405 Ok(())
8406 }
8407
8408 fn test_move_location_extended(
8409 machine: &mut MachineARM64,
8410 signed: bool,
8411 sized: Size,
8412 ) -> Result<(), CompileError> {
8413 machine.move_location_extend(
8414 sized,
8415 signed,
8416 Location::GPR(GPR::X0),
8417 Size::S64,
8418 Location::GPR(GPR::X1),
8419 )?;
8420 machine.move_location_extend(
8421 sized,
8422 signed,
8423 Location::GPR(GPR::X0),
8424 Size::S64,
8425 Location::Memory(GPR::X1, 10),
8426 )?;
8427 machine.move_location_extend(
8428 sized,
8429 signed,
8430 Location::GPR(GPR::X0),
8431 Size::S64,
8432 Location::Memory(GPR::X1, 16),
8433 )?;
8434 machine.move_location_extend(
8435 sized,
8436 signed,
8437 Location::GPR(GPR::X0),
8438 Size::S64,
8439 Location::Memory(GPR::X1, -16),
8440 )?;
8441 machine.move_location_extend(
8442 sized,
8443 signed,
8444 Location::GPR(GPR::X0),
8445 Size::S64,
8446 Location::Memory(GPR::X1, 1024),
8447 )?;
8448 machine.move_location_extend(
8449 sized,
8450 signed,
8451 Location::GPR(GPR::X0),
8452 Size::S64,
8453 Location::Memory(GPR::X1, -1024),
8454 )?;
8455 machine.move_location_extend(
8456 sized,
8457 signed,
8458 Location::Memory(GPR::X0, 10),
8459 Size::S64,
8460 Location::GPR(GPR::X1),
8461 )?;
8462
8463 Ok(())
8464 }
8465
8466 fn test_binop_op(
8467 machine: &mut MachineARM64,
8468 op: fn(&mut MachineARM64, Location, Location, Location) -> Result<(), CompileError>,
8469 ) -> Result<(), CompileError> {
8470 op(
8471 machine,
8472 Location::GPR(GPR::X2),
8473 Location::GPR(GPR::X2),
8474 Location::GPR(GPR::X0),
8475 )?;
8476 op(
8477 machine,
8478 Location::GPR(GPR::X2),
8479 Location::Imm32(10),
8480 Location::GPR(GPR::X0),
8481 )?;
8482 op(
8483 machine,
8484 Location::GPR(GPR::X0),
8485 Location::GPR(GPR::X0),
8486 Location::GPR(GPR::X0),
8487 )?;
8488 op(
8489 machine,
8490 Location::Imm32(10),
8491 Location::GPR(GPR::X2),
8492 Location::GPR(GPR::X0),
8493 )?;
8494 op(
8495 machine,
8496 Location::GPR(GPR::X0),
8497 Location::GPR(GPR::X2),
8498 Location::Memory(GPR::X0, 10),
8499 )?;
8500 op(
8501 machine,
8502 Location::GPR(GPR::X0),
8503 Location::Memory(GPR::X2, 16),
8504 Location::Memory(GPR::X0, 10),
8505 )?;
8506 op(
8507 machine,
8508 Location::Memory(GPR::X0, 0),
8509 Location::Memory(GPR::X2, 16),
8510 Location::Memory(GPR::X0, 10),
8511 )?;
8512
8513 Ok(())
8514 }
8515
8516 fn test_float_binop_op(
8517 machine: &mut MachineARM64,
8518 op: fn(&mut MachineARM64, Location, Location, Location) -> Result<(), CompileError>,
8519 ) -> Result<(), CompileError> {
8520 op(
8521 machine,
8522 Location::SIMD(NEON::V3),
8523 Location::SIMD(NEON::V2),
8524 Location::SIMD(NEON::V0),
8525 )?;
8526 op(
8527 machine,
8528 Location::SIMD(NEON::V0),
8529 Location::SIMD(NEON::V2),
8530 Location::SIMD(NEON::V0),
8531 )?;
8532 op(
8533 machine,
8534 Location::SIMD(NEON::V0),
8535 Location::SIMD(NEON::V0),
8536 Location::SIMD(NEON::V0),
8537 )?;
8538 op(
8539 machine,
8540 Location::Memory(GPR::X0, 0),
8541 Location::SIMD(NEON::V2),
8542 Location::SIMD(NEON::V0),
8543 )?;
8544 op(
8545 machine,
8546 Location::Memory(GPR::X0, 0),
8547 Location::Memory(GPR::X1, 10),
8548 Location::SIMD(NEON::V0),
8549 )?;
8550 op(
8551 machine,
8552 Location::Memory(GPR::X0, 0),
8553 Location::Memory(GPR::X1, 16),
8554 Location::Memory(GPR::X2, 32),
8555 )?;
8556 op(
8557 machine,
8558 Location::SIMD(NEON::V0),
8559 Location::Memory(GPR::X1, 16),
8560 Location::Memory(GPR::X2, 32),
8561 )?;
8562 op(
8563 machine,
8564 Location::SIMD(NEON::V0),
8565 Location::SIMD(NEON::V1),
8566 Location::Memory(GPR::X2, 32),
8567 )?;
8568
8569 Ok(())
8570 }
8571
8572 fn test_float_cmp_op(
8573 machine: &mut MachineARM64,
8574 op: fn(&mut MachineARM64, Location, Location, Location) -> Result<(), CompileError>,
8575 ) -> Result<(), CompileError> {
8576 op(
8577 machine,
8578 Location::SIMD(NEON::V3),
8579 Location::SIMD(NEON::V2),
8580 Location::GPR(GPR::X0),
8581 )?;
8582 op(
8583 machine,
8584 Location::SIMD(NEON::V0),
8585 Location::SIMD(NEON::V0),
8586 Location::GPR(GPR::X0),
8587 )?;
8588 op(
8589 machine,
8590 Location::Memory(GPR::X1, 0),
8591 Location::SIMD(NEON::V2),
8592 Location::GPR(GPR::X0),
8593 )?;
8594 op(
8595 machine,
8596 Location::Memory(GPR::X1, 0),
8597 Location::Memory(GPR::X2, 10),
8598 Location::GPR(GPR::X0),
8599 )?;
8600 op(
8601 machine,
8602 Location::Memory(GPR::X1, 0),
8603 Location::Memory(GPR::X2, 16),
8604 Location::Memory(GPR::X0, 32),
8605 )?;
8606 op(
8607 machine,
8608 Location::SIMD(NEON::V0),
8609 Location::Memory(GPR::X2, 16),
8610 Location::Memory(GPR::X0, 32),
8611 )?;
8612 op(
8613 machine,
8614 Location::SIMD(NEON::V0),
8615 Location::SIMD(NEON::V1),
8616 Location::Memory(GPR::X0, 32),
8617 )?;
8618
8619 Ok(())
8620 }
8621
8622 #[test]
8623 fn tests_arm64() -> Result<(), CompileError> {
8624 let mut machine = MachineARM64::new(None);
8625
8626 test_move_location(&mut machine, Size::S32)?;
8627 test_move_location(&mut machine, Size::S64)?;
8628 test_move_location_extended(&mut machine, false, Size::S8)?;
8629 test_move_location_extended(&mut machine, false, Size::S16)?;
8630 test_move_location_extended(&mut machine, false, Size::S32)?;
8631 test_move_location_extended(&mut machine, true, Size::S8)?;
8632 test_move_location_extended(&mut machine, true, Size::S16)?;
8633 test_move_location_extended(&mut machine, true, Size::S32)?;
8634 test_binop_op(&mut machine, MachineARM64::emit_binop_add32)?;
8635 test_binop_op(&mut machine, MachineARM64::emit_binop_add64)?;
8636 test_binop_op(&mut machine, MachineARM64::emit_binop_sub32)?;
8637 test_binop_op(&mut machine, MachineARM64::emit_binop_sub64)?;
8638 test_binop_op(&mut machine, MachineARM64::emit_binop_and32)?;
8639 test_binop_op(&mut machine, MachineARM64::emit_binop_and64)?;
8640 test_binop_op(&mut machine, MachineARM64::emit_binop_xor32)?;
8641 test_binop_op(&mut machine, MachineARM64::emit_binop_xor64)?;
8642 test_binop_op(&mut machine, MachineARM64::emit_binop_or32)?;
8643 test_binop_op(&mut machine, MachineARM64::emit_binop_or64)?;
8644 test_binop_op(&mut machine, MachineARM64::emit_binop_mul32)?;
8645 test_binop_op(&mut machine, MachineARM64::emit_binop_mul64)?;
8646 test_float_binop_op(&mut machine, MachineARM64::f32_add)?;
8647 test_float_binop_op(&mut machine, MachineARM64::f32_sub)?;
8648 test_float_binop_op(&mut machine, MachineARM64::f32_mul)?;
8649 test_float_binop_op(&mut machine, MachineARM64::f32_div)?;
8650 test_float_cmp_op(&mut machine, MachineARM64::f32_cmp_eq)?;
8651 test_float_cmp_op(&mut machine, MachineARM64::f32_cmp_lt)?;
8652 test_float_cmp_op(&mut machine, MachineARM64::f32_cmp_le)?;
8653
8654 Ok(())
8655 }
8656}