1use std::collections::HashMap;
2
3use dynasmrt::{VecAssembler, aarch64::Aarch64Relocation};
4use fixedbitset::FixedBitSet;
5#[cfg(feature = "unwind")]
6use gimli::{AArch64, write::CallFrameInstruction};
7
8use wasmer_compiler::{
9 types::{
10 address_map::InstructionAddressMap,
11 function::FunctionBody,
12 relocation::{Relocation, RelocationKind, RelocationTarget},
13 section::CustomSection,
14 },
15 wasmparser::MemArg,
16};
17use wasmer_types::{
18 CompilationProgressCallback, CompileError, FunctionIndex, FunctionType, SourceLoc, TrapCode,
19 TrapInformation, VMOffsets,
20 target::{CallingConvention, CpuFeature, Target},
21};
22
23use crate::{
24 arm64_decl::{GPR, NEON},
25 codegen_error,
26 common_decl::*,
27 emitter_arm64::*,
28 location::{Location as AbstractLocation, Reg},
29 machine::*,
30 unwind::{UnwindInstructions, UnwindOps, UnwindRegister},
31};
32
33type Assembler = VecAssembler<Aarch64Relocation>;
34type Location = AbstractLocation<GPR, NEON>;
35
36pub struct MachineARM64 {
37 assembler: Assembler,
38 used_gprs: FixedBitSet,
39 used_simd: FixedBitSet,
40 trap_table: TrapTable,
41 instructions_address_map: Vec<InstructionAddressMap>,
44 src_loc: u32,
46 pushed: bool,
48 unwind_ops: Vec<(usize, UnwindOps<GPR, NEON>)>,
50 has_neon: bool,
52}
53
54pub(crate) const ARM64_RETURN_VALUE_REGISTERS: [GPR; 8] = [
57 GPR::X0,
58 GPR::X1,
59 GPR::X2,
60 GPR::X3,
61 GPR::X4,
62 GPR::X5,
63 GPR::X6,
64 GPR::X7,
65];
66
67#[allow(dead_code)]
68#[derive(PartialEq)]
69enum ImmType {
70 None,
71 NoneXzr,
72 Bits8,
73 Bits12,
74 Shift32,
75 Shift32No0,
76 Shift64,
77 Shift64No0,
78 Logical32,
79 Logical64,
80 UnscaledOffset,
81 OffsetByte,
82 OffsetHWord,
83 OffsetWord,
84 OffsetDWord,
85}
86
87const SCRATCH_REG: GPR = GPR::X17;
88
89#[allow(dead_code)]
90impl MachineARM64 {
91 pub fn new(target: Option<Target>) -> Self {
92 let has_neon = match target {
96 Some(ref target) => target.cpu_features().contains(CpuFeature::NEON),
97 None => false,
98 };
99
100 MachineARM64 {
101 assembler: Assembler::new(0),
102 used_gprs: FixedBitSet::with_capacity(32),
103 used_simd: FixedBitSet::with_capacity(32),
104 trap_table: TrapTable::default(),
105 instructions_address_map: vec![],
106 src_loc: 0,
107 pushed: false,
108 unwind_ops: vec![],
109 has_neon,
110 }
111 }
112 fn compatible_imm(&self, imm: i64, ty: ImmType) -> bool {
113 match ty {
114 ImmType::None => false,
115 ImmType::NoneXzr => false,
116 ImmType::Bits8 => (0..256).contains(&imm),
117 ImmType::Bits12 => (0..0x1000).contains(&imm),
118 ImmType::Shift32 => (0..32).contains(&imm),
119 ImmType::Shift32No0 => (1..32).contains(&imm),
120 ImmType::Shift64 => (0..64).contains(&imm),
121 ImmType::Shift64No0 => (1..64).contains(&imm),
122 ImmType::Logical32 => encode_logical_immediate_32bit(imm as u32).is_some(),
123 ImmType::Logical64 => encode_logical_immediate_64bit(imm as u64).is_some(),
124 ImmType::UnscaledOffset => (imm > -256) && (imm < 256),
125 ImmType::OffsetByte => (0..0x1000).contains(&imm),
126 ImmType::OffsetHWord => (imm & 1 == 0) && (0..0x2000).contains(&imm),
127 ImmType::OffsetWord => (imm & 3 == 0) && (0..0x4000).contains(&imm),
128 ImmType::OffsetDWord => (imm & 7 == 0) && (0..0x8000).contains(&imm),
129 }
130 }
131
132 fn location_to_reg(
133 &mut self,
134 sz: Size,
135 src: Location,
136 temps: &mut Vec<GPR>,
137 allow_imm: ImmType,
138 read_val: bool,
139 wanted: Option<GPR>,
140 ) -> Result<Location, CompileError> {
141 match src {
142 Location::GPR(_) | Location::SIMD(_) => Ok(src),
143 Location::Imm8(val) => {
144 if allow_imm == ImmType::NoneXzr && val == 0 {
145 Ok(Location::GPR(GPR::XzrSp))
146 } else if self.compatible_imm(val as i64, allow_imm) {
147 Ok(src)
148 } else {
149 let tmp = if let Some(wanted) = wanted {
150 wanted
151 } else {
152 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
153 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
154 })?;
155 temps.push(tmp);
156 tmp
157 };
158 self.assembler
159 .emit_mov_imm(Location::GPR(tmp), val as u64)?;
160 Ok(Location::GPR(tmp))
161 }
162 }
163 Location::Imm32(val) => {
164 if allow_imm == ImmType::NoneXzr && val == 0 {
165 Ok(Location::GPR(GPR::XzrSp))
166 } else if self.compatible_imm(val as i64, allow_imm) {
167 Ok(src)
168 } else {
169 let tmp = if let Some(wanted) = wanted {
170 wanted
171 } else {
172 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
173 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
174 })?;
175 temps.push(tmp);
176 tmp
177 };
178 self.assembler
179 .emit_mov_imm(Location::GPR(tmp), (val as i64) as u64)?;
180 Ok(Location::GPR(tmp))
181 }
182 }
183 Location::Imm64(val) => {
184 if allow_imm == ImmType::NoneXzr && val == 0 {
185 Ok(Location::GPR(GPR::XzrSp))
186 } else if self.compatible_imm(val as i64, allow_imm) {
187 Ok(src)
188 } else {
189 let tmp = if let Some(wanted) = wanted {
190 wanted
191 } else {
192 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
193 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
194 })?;
195 temps.push(tmp);
196 tmp
197 };
198 self.assembler
199 .emit_mov_imm(Location::GPR(tmp), val as u64)?;
200 Ok(Location::GPR(tmp))
201 }
202 }
203 Location::Memory(reg, val) => {
204 let tmp = if let Some(wanted) = wanted {
205 wanted
206 } else {
207 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
208 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
209 })?;
210 temps.push(tmp);
211 tmp
212 };
213 if read_val {
214 let offsize = match sz {
215 Size::S8 => ImmType::OffsetByte,
216 Size::S16 => ImmType::OffsetHWord,
217 Size::S32 => ImmType::OffsetWord,
218 Size::S64 => ImmType::OffsetDWord,
219 };
220 if sz == Size::S8 {
221 if self.compatible_imm(val as i64, offsize) {
222 self.assembler.emit_ldrb(
223 sz,
224 Location::GPR(tmp),
225 Location::Memory(reg, val as _),
226 )?;
227 } else {
228 if reg == tmp {
229 codegen_error!("singlepass reg==tmp unreachable");
230 }
231 self.assembler
232 .emit_mov_imm(Location::GPR(tmp), (val as i64) as u64)?;
233 self.assembler.emit_ldrb(
234 sz,
235 Location::GPR(tmp),
236 Location::Memory2(reg, tmp, Multiplier::One, 0),
237 )?;
238 }
239 } else if sz == Size::S16 {
240 if self.compatible_imm(val as i64, offsize) {
241 self.assembler.emit_ldrh(
242 sz,
243 Location::GPR(tmp),
244 Location::Memory(reg, val as _),
245 )?;
246 } else {
247 if reg == tmp {
248 codegen_error!("singlepass reg==tmp unreachable");
249 }
250 self.assembler
251 .emit_mov_imm(Location::GPR(tmp), (val as i64) as u64)?;
252 self.assembler.emit_ldrh(
253 sz,
254 Location::GPR(tmp),
255 Location::Memory2(reg, tmp, Multiplier::One, 0),
256 )?;
257 }
258 } else if self.compatible_imm(val as i64, offsize) {
259 self.assembler.emit_ldr(
260 sz,
261 Location::GPR(tmp),
262 Location::Memory(reg, val as _),
263 )?;
264 } else if self.compatible_imm(val as i64, ImmType::UnscaledOffset) {
265 self.assembler.emit_ldur(sz, Location::GPR(tmp), reg, val)?;
266 } else {
267 if reg == tmp {
268 codegen_error!("singlepass reg == tmp unreachable");
269 }
270 self.assembler
271 .emit_mov_imm(Location::GPR(tmp), (val as i64) as u64)?;
272 self.assembler.emit_ldr(
273 sz,
274 Location::GPR(tmp),
275 Location::Memory2(reg, tmp, Multiplier::One, 0),
276 )?;
277 }
278 }
279 Ok(Location::GPR(tmp))
280 }
281 _ => codegen_error!("singlepass can't emit location_to_reg {:?} {:?}", sz, src),
282 }
283 }
284 fn location_to_neon(
285 &mut self,
286 sz: Size,
287 src: Location,
288 temps: &mut Vec<NEON>,
289 allow_imm: ImmType,
290 read_val: bool,
291 ) -> Result<Location, CompileError> {
292 match src {
293 Location::SIMD(_) => Ok(src),
294 Location::GPR(_) => {
295 let tmp = self.acquire_temp_simd().ok_or_else(|| {
296 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
297 })?;
298 temps.push(tmp);
299 if read_val {
300 self.assembler.emit_mov(sz, src, Location::SIMD(tmp))?;
301 }
302 Ok(Location::SIMD(tmp))
303 }
304 Location::Imm8(val) => {
305 if self.compatible_imm(val as i64, allow_imm) {
306 Ok(src)
307 } else {
308 let gpr = self.acquire_temp_gpr().ok_or_else(|| {
309 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
310 })?;
311 let tmp = self.acquire_temp_simd().ok_or_else(|| {
312 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
313 })?;
314 temps.push(tmp);
315 self.assembler
316 .emit_mov_imm(Location::GPR(gpr), val as u64)?;
317 self.assembler
318 .emit_mov(sz, Location::GPR(gpr), Location::SIMD(tmp))?;
319 self.release_gpr(gpr);
320 Ok(Location::SIMD(tmp))
321 }
322 }
323 Location::Imm32(val) => {
324 if self.compatible_imm(val as i64, allow_imm) {
325 Ok(src)
326 } else {
327 let gpr = self.acquire_temp_gpr().ok_or_else(|| {
328 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
329 })?;
330 let tmp = self.acquire_temp_simd().ok_or_else(|| {
331 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
332 })?;
333 temps.push(tmp);
334 self.assembler
335 .emit_mov_imm(Location::GPR(gpr), (val as i64) as u64)?;
336 self.assembler
337 .emit_mov(sz, Location::GPR(gpr), Location::SIMD(tmp))?;
338 self.release_gpr(gpr);
339 Ok(Location::SIMD(tmp))
340 }
341 }
342 Location::Imm64(val) => {
343 if self.compatible_imm(val as i64, allow_imm) {
344 Ok(src)
345 } else {
346 let gpr = self.acquire_temp_gpr().ok_or_else(|| {
347 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
348 })?;
349 let tmp = self.acquire_temp_simd().ok_or_else(|| {
350 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
351 })?;
352 temps.push(tmp);
353 self.assembler
354 .emit_mov_imm(Location::GPR(gpr), val as u64)?;
355 self.assembler
356 .emit_mov(sz, Location::GPR(gpr), Location::SIMD(tmp))?;
357 self.release_gpr(gpr);
358 Ok(Location::SIMD(tmp))
359 }
360 }
361 Location::Memory(reg, val) => {
362 let tmp = self.acquire_temp_simd().ok_or_else(|| {
363 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
364 })?;
365 temps.push(tmp);
366 if read_val {
367 let offsize = if sz == Size::S32 {
368 ImmType::OffsetWord
369 } else {
370 ImmType::OffsetDWord
371 };
372 if self.compatible_imm(val as i64, offsize) {
373 self.assembler.emit_ldr(
374 sz,
375 Location::SIMD(tmp),
376 Location::Memory(reg, val as _),
377 )?;
378 } else if self.compatible_imm(val as i64, ImmType::UnscaledOffset) {
379 self.assembler
380 .emit_ldur(sz, Location::SIMD(tmp), reg, val)?;
381 } else {
382 let gpr = self.acquire_temp_gpr().ok_or_else(|| {
383 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
384 })?;
385 self.assembler
386 .emit_mov_imm(Location::GPR(gpr), (val as i64) as u64)?;
387 self.assembler.emit_ldr(
388 sz,
389 Location::SIMD(tmp),
390 Location::Memory2(reg, gpr, Multiplier::One, 0),
391 )?;
392 self.release_gpr(gpr);
393 }
394 }
395 Ok(Location::SIMD(tmp))
396 }
397 _ => codegen_error!("singlepass can't emit location_to_neon {:?} {:?}", sz, src),
398 }
399 }
400
401 fn emit_relaxed_binop(
402 &mut self,
403 op: fn(&mut Assembler, Size, Location, Location) -> Result<(), CompileError>,
404 sz: Size,
405 src: Location,
406 dst: Location,
407 putback: bool,
408 ) -> Result<(), CompileError> {
409 let mut temps = vec![];
410 let src_imm = if putback {
411 ImmType::None
412 } else {
413 ImmType::Bits12
414 };
415 let src = self.location_to_reg(sz, src, &mut temps, src_imm, true, None)?;
416 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, !putback, None)?;
417 op(&mut self.assembler, sz, src, dest)?;
418 if dst != dest && putback {
419 self.move_location(sz, dest, dst)?;
420 }
421 for r in temps {
422 self.release_gpr(r);
423 }
424 Ok(())
425 }
426 fn emit_relaxed_binop_neon(
427 &mut self,
428 op: fn(&mut Assembler, Size, Location, Location) -> Result<(), CompileError>,
429 sz: Size,
430 src: Location,
431 dst: Location,
432 putback: bool,
433 ) -> Result<(), CompileError> {
434 let mut temps = vec![];
435 let src = self.location_to_neon(sz, src, &mut temps, ImmType::None, true)?;
436 let dest = self.location_to_neon(sz, dst, &mut temps, ImmType::None, !putback)?;
437 op(&mut self.assembler, sz, src, dest)?;
438 if dst != dest && putback {
439 self.move_location(sz, dest, dst)?;
440 }
441 for r in temps {
442 self.release_simd(r);
443 }
444 Ok(())
445 }
446 fn emit_relaxed_fcvt(
447 &mut self,
448 src_size: Size,
449 dst_size: Size,
450 src: Location,
451 dst: Location,
452 ) -> Result<(), CompileError> {
453 let mut temps = vec![];
454 let src = self.location_to_neon(src_size, src, &mut temps, ImmType::None, true)?;
455 let dest = self.location_to_neon(dst_size, dst, &mut temps, ImmType::None, false)?;
456 self.assembler.emit_fcvt(src_size, src, dest)?;
457 if dst != dest {
458 self.move_location(dst_size, dest, dst)?;
459 }
460 for r in temps {
461 self.release_simd(r);
462 }
463 Ok(())
464 }
465 fn emit_relaxed_binop3(
466 &mut self,
467 op: fn(&mut Assembler, Size, Location, Location, Location) -> Result<(), CompileError>,
468 sz: Size,
469 src1: Location,
470 src2: Location,
471 dst: Location,
472 allow_imm: ImmType,
473 ) -> Result<(), CompileError> {
474 let mut temps = vec![];
475 let src1 = self.location_to_reg(sz, src1, &mut temps, ImmType::None, true, None)?;
476 let src2 = self.location_to_reg(sz, src2, &mut temps, allow_imm, true, None)?;
477 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
478 op(&mut self.assembler, sz, src1, src2, dest)?;
479 if dst != dest {
480 self.move_location(sz, dest, dst)?;
481 }
482 for r in temps {
483 self.release_gpr(r);
484 }
485 Ok(())
486 }
487 fn emit_relaxed_binop3_neon(
488 &mut self,
489 op: fn(&mut Assembler, Size, Location, Location, Location) -> Result<(), CompileError>,
490 sz: Size,
491 src1: Location,
492 src2: Location,
493 dst: Location,
494 allow_imm: ImmType,
495 ) -> Result<(), CompileError> {
496 let mut temps = vec![];
497 let src1 = self.location_to_neon(sz, src1, &mut temps, ImmType::None, true)?;
498 let src2 = self.location_to_neon(sz, src2, &mut temps, allow_imm, true)?;
499 let dest = self.location_to_neon(sz, dst, &mut temps, ImmType::None, false)?;
500 op(&mut self.assembler, sz, src1, src2, dest)?;
501 if dst != dest {
502 self.move_location(sz, dest, dst)?;
503 }
504 for r in temps {
505 self.release_simd(r);
506 }
507 Ok(())
508 }
509 fn emit_relaxed_ldr64(
510 &mut self,
511 sz: Size,
512 dst: Location,
513 src: Location,
514 ) -> Result<(), CompileError> {
515 let mut temps = vec![];
516 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
517 match src {
518 Location::Memory(addr, offset) => {
519 if self.compatible_imm(offset as i64, ImmType::OffsetDWord) {
520 self.assembler.emit_ldr(Size::S64, dest, src)?;
521 } else if self.compatible_imm(offset as i64, ImmType::UnscaledOffset) {
522 self.assembler.emit_ldur(Size::S64, dest, addr, offset)?;
523 } else {
524 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
525 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
526 })?;
527 self.assembler
528 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
529 self.assembler.emit_ldr(
530 Size::S64,
531 dest,
532 Location::Memory2(addr, tmp, Multiplier::One, 0),
533 )?;
534 temps.push(tmp);
535 }
536 }
537 _ => codegen_error!("singlepass emit_relaxed_ldr64 unreachable"),
538 }
539 if dst != dest {
540 self.move_location(sz, dest, dst)?;
541 }
542 for r in temps {
543 self.release_gpr(r);
544 }
545 Ok(())
546 }
547 fn emit_relaxed_ldr32(
548 &mut self,
549 sz: Size,
550 dst: Location,
551 src: Location,
552 ) -> Result<(), CompileError> {
553 let mut temps = vec![];
554 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
555 match src {
556 Location::Memory(addr, offset) => {
557 if self.compatible_imm(offset as i64, ImmType::OffsetWord) {
558 self.assembler.emit_ldr(Size::S32, dest, src)?;
559 } else if self.compatible_imm(offset as i64, ImmType::UnscaledOffset) {
560 self.assembler.emit_ldur(Size::S32, dest, addr, offset)?;
561 } else {
562 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
563 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
564 })?;
565 self.assembler
566 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
567 self.assembler.emit_ldr(
568 Size::S32,
569 dest,
570 Location::Memory2(addr, tmp, Multiplier::One, 0),
571 )?;
572 temps.push(tmp);
573 }
574 }
575 _ => codegen_error!("singlepass emit_relaxed_ldr32 unreachable"),
576 }
577 if dst != dest {
578 self.move_location(sz, dest, dst)?;
579 }
580 for r in temps {
581 self.release_gpr(r);
582 }
583 Ok(())
584 }
585 fn emit_relaxed_ldr32s(
586 &mut self,
587 sz: Size,
588 dst: Location,
589 src: Location,
590 ) -> Result<(), CompileError> {
591 let mut temps = vec![];
592 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
593 match src {
594 Location::Memory(addr, offset) => {
595 if self.compatible_imm(offset as i64, ImmType::OffsetWord) {
596 self.assembler.emit_ldrsw(Size::S64, dest, src)?;
597 } else {
598 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
599 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
600 })?;
601 self.assembler
602 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
603 self.assembler.emit_ldrsw(
604 Size::S64,
605 dest,
606 Location::Memory2(addr, tmp, Multiplier::One, 0),
607 )?;
608 temps.push(tmp);
609 }
610 }
611 _ => codegen_error!("singplepass emit_relaxed_ldr32s unreachable"),
612 }
613 if dst != dest {
614 self.move_location(sz, dest, dst)?;
615 }
616 for r in temps {
617 self.release_gpr(r);
618 }
619 Ok(())
620 }
621 fn emit_relaxed_ldr16(
622 &mut self,
623 sz: Size,
624 dst: Location,
625 src: Location,
626 ) -> Result<(), CompileError> {
627 let mut temps = vec![];
628 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
629 match src {
630 Location::Memory(addr, offset) => {
631 if self.compatible_imm(offset as i64, ImmType::OffsetHWord) {
632 self.assembler.emit_ldrh(Size::S32, dest, src)?;
633 } else {
634 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
635 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
636 })?;
637 self.assembler
638 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
639 self.assembler.emit_ldrh(
640 Size::S32,
641 dest,
642 Location::Memory2(addr, tmp, Multiplier::One, 0),
643 )?;
644 temps.push(tmp);
645 }
646 }
647 _ => codegen_error!("singlpass emit_relaxed_ldr16 unreachable"),
648 }
649 if dst != dest {
650 self.move_location(sz, dest, dst)?;
651 }
652 for r in temps {
653 self.release_gpr(r);
654 }
655 Ok(())
656 }
657 fn emit_relaxed_ldr16s(
658 &mut self,
659 sz: Size,
660 dst: Location,
661 src: Location,
662 ) -> Result<(), CompileError> {
663 let mut temps = vec![];
664 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
665 match src {
666 Location::Memory(addr, offset) => {
667 if self.compatible_imm(offset as i64, ImmType::OffsetHWord) {
668 self.assembler.emit_ldrsh(sz, dest, src)?;
669 } else {
670 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
671 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
672 })?;
673 self.assembler
674 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
675 self.assembler.emit_ldrsh(
676 sz,
677 dest,
678 Location::Memory2(addr, tmp, Multiplier::One, 0),
679 )?;
680 temps.push(tmp);
681 }
682 }
683 _ => codegen_error!("singlepass emit_relaxed_ldr16s unreachable"),
684 }
685 if dst != dest {
686 self.move_location(sz, dest, dst)?;
687 }
688 for r in temps {
689 self.release_gpr(r);
690 }
691 Ok(())
692 }
693 fn emit_relaxed_ldr8(
694 &mut self,
695 sz: Size,
696 dst: Location,
697 src: Location,
698 ) -> Result<(), CompileError> {
699 let mut temps = vec![];
700 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
701 match src {
702 Location::Memory(addr, offset) => {
703 if self.compatible_imm(offset as i64, ImmType::OffsetByte) {
704 self.assembler.emit_ldrb(Size::S32, dest, src)?;
705 } else {
706 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
707 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
708 })?;
709 self.assembler
710 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
711 self.assembler.emit_ldrb(
712 Size::S32,
713 dest,
714 Location::Memory2(addr, tmp, Multiplier::One, 0),
715 )?;
716 temps.push(tmp);
717 }
718 }
719 _ => codegen_error!("singplepass emit_relaxed_ldr8 unreachable"),
720 }
721 if dst != dest {
722 self.move_location(sz, dest, dst)?;
723 }
724 for r in temps {
725 self.release_gpr(r);
726 }
727 Ok(())
728 }
729 fn emit_relaxed_ldr8s(
730 &mut self,
731 sz: Size,
732 dst: Location,
733 src: Location,
734 ) -> Result<(), CompileError> {
735 let mut temps = vec![];
736 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
737 match src {
738 Location::Memory(addr, offset) => {
739 if self.compatible_imm(offset as i64, ImmType::OffsetByte) {
740 self.assembler.emit_ldrsb(sz, dest, src)?;
741 } else {
742 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
743 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
744 })?;
745 self.assembler
746 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
747 self.assembler.emit_ldrsb(
748 sz,
749 dest,
750 Location::Memory2(addr, tmp, Multiplier::One, 0),
751 )?;
752 temps.push(tmp);
753 }
754 }
755 _ => codegen_error!("singlepass emit_relaxed_ldr8s unreachable"),
756 }
757 if dst != dest {
758 self.move_location(sz, dest, dst)?;
759 }
760 for r in temps {
761 self.release_gpr(r);
762 }
763 Ok(())
764 }
765 fn emit_relaxed_str64(&mut self, dst: Location, src: Location) -> Result<(), CompileError> {
766 let mut temps = vec![];
767 let dst = self.location_to_reg(Size::S64, dst, &mut temps, ImmType::NoneXzr, true, None)?;
768 match src {
769 Location::Memory(addr, offset) => {
770 if self.compatible_imm(offset as i64, ImmType::OffsetDWord) {
771 self.assembler.emit_str(Size::S64, dst, src)?;
772 } else if self.compatible_imm(offset as i64, ImmType::UnscaledOffset) {
773 self.assembler.emit_stur(Size::S64, dst, addr, offset)?;
774 } else {
775 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
776 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
777 })?;
778 self.assembler
779 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
780 self.assembler.emit_str(
781 Size::S64,
782 dst,
783 Location::Memory2(addr, tmp, Multiplier::One, 0),
784 )?;
785 temps.push(tmp);
786 }
787 }
788 _ => codegen_error!("singlepass can't emit str64 {:?} {:?}", dst, src),
789 }
790 for r in temps {
791 self.release_gpr(r);
792 }
793 Ok(())
794 }
795 fn emit_relaxed_str32(&mut self, dst: Location, src: Location) -> Result<(), CompileError> {
796 let mut temps = vec![];
797 let dst = self.location_to_reg(Size::S64, dst, &mut temps, ImmType::NoneXzr, true, None)?;
798 match src {
799 Location::Memory(addr, offset) => {
800 if self.compatible_imm(offset as i64, ImmType::OffsetWord) {
801 self.assembler.emit_str(Size::S32, dst, src)?;
802 } else if self.compatible_imm(offset as i64, ImmType::UnscaledOffset) {
803 self.assembler.emit_stur(Size::S32, dst, addr, offset)?;
804 } else {
805 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
806 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
807 })?;
808 self.assembler
809 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
810 self.assembler.emit_str(
811 Size::S32,
812 dst,
813 Location::Memory2(addr, tmp, Multiplier::One, 0),
814 )?;
815 temps.push(tmp);
816 }
817 }
818 _ => codegen_error!("singplepass emit_relaxed_str32 unreachable"),
819 }
820 for r in temps {
821 self.release_gpr(r);
822 }
823 Ok(())
824 }
825 fn emit_relaxed_str16(&mut self, dst: Location, src: Location) -> Result<(), CompileError> {
826 let mut temps = vec![];
827 let dst = self.location_to_reg(Size::S64, dst, &mut temps, ImmType::NoneXzr, true, None)?;
828 match src {
829 Location::Memory(addr, offset) => {
830 if self.compatible_imm(offset as i64, ImmType::OffsetHWord) {
831 self.assembler.emit_strh(Size::S32, dst, src)?;
832 } else {
833 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
834 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
835 })?;
836 self.assembler
837 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
838 self.assembler.emit_strh(
839 Size::S32,
840 dst,
841 Location::Memory2(addr, tmp, Multiplier::One, 0),
842 )?;
843 temps.push(tmp);
844 }
845 }
846 _ => codegen_error!("singlepass emit_relaxed_str16 unreachable"),
847 }
848 for r in temps {
849 self.release_gpr(r);
850 }
851 Ok(())
852 }
853 fn emit_relaxed_str8(&mut self, dst: Location, src: Location) -> Result<(), CompileError> {
854 let mut temps = vec![];
855 let dst = self.location_to_reg(Size::S64, dst, &mut temps, ImmType::NoneXzr, true, None)?;
856 match src {
857 Location::Memory(addr, offset) => {
858 if self.compatible_imm(offset as i64, ImmType::OffsetByte) {
859 self.assembler
860 .emit_strb(Size::S32, dst, Location::Memory(addr, offset))?;
861 } else {
862 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
863 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
864 })?;
865 self.assembler
866 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
867 self.assembler.emit_strb(
868 Size::S32,
869 dst,
870 Location::Memory2(addr, tmp, Multiplier::One, 0),
871 )?;
872 temps.push(tmp);
873 }
874 }
875 _ => codegen_error!("singlepass emit_relaxed_str8 unreachable"),
876 }
877 for r in temps {
878 self.release_gpr(r);
879 }
880 Ok(())
881 }
882 fn emit_cmpop_i64_dynamic_b(
884 &mut self,
885 c: Condition,
886 loc_a: Location,
887 loc_b: Location,
888 ret: Location,
889 ) -> Result<(), CompileError> {
890 match ret {
891 Location::GPR(_) => {
892 self.emit_relaxed_cmp(Size::S64, loc_b, loc_a)?;
893 self.assembler.emit_cset(Size::S32, ret, c)?;
894 }
895 Location::Memory(_, _) => {
896 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
897 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
898 })?;
899 self.emit_relaxed_cmp(Size::S64, loc_b, loc_a)?;
900 self.assembler.emit_cset(Size::S32, Location::GPR(tmp), c)?;
901 self.move_location(Size::S32, Location::GPR(tmp), ret)?;
902 self.release_gpr(tmp);
903 }
904 _ => {
905 codegen_error!("singlepass emit_compop_i64_dynamic_b unreachable");
906 }
907 }
908 Ok(())
909 }
910 fn emit_cmpop_i32_dynamic_b(
912 &mut self,
913 c: Condition,
914 loc_a: Location,
915 loc_b: Location,
916 ret: Location,
917 ) -> Result<(), CompileError> {
918 match ret {
919 Location::GPR(_) => {
920 self.emit_relaxed_cmp(Size::S32, loc_b, loc_a)?;
921 self.assembler.emit_cset(Size::S32, ret, c)?;
922 }
923 Location::Memory(_, _) => {
924 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
925 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
926 })?;
927 self.emit_relaxed_cmp(Size::S32, loc_b, loc_a)?;
928 self.assembler.emit_cset(Size::S32, Location::GPR(tmp), c)?;
929 self.move_location(Size::S32, Location::GPR(tmp), ret)?;
930 self.release_gpr(tmp);
931 }
932 _ => {
933 codegen_error!("singlepass emit_cmpop_i32_dynamic_b unreachable");
934 }
935 }
936 Ok(())
937 }
938
939 #[allow(clippy::too_many_arguments)]
940 fn memory_op<F: FnOnce(&mut Self, GPR) -> Result<(), CompileError>>(
941 &mut self,
942 addr: Location,
943 memarg: &MemArg,
944 check_alignment: bool,
945 value_size: usize,
946 need_check: bool,
947 imported_memories: bool,
948 offset: i32,
949 heap_access_oob: Label,
950 unaligned_atomic: Label,
951 cb: F,
952 ) -> Result<(), CompileError> {
953 let tmp_addr = self.acquire_temp_gpr().ok_or_else(|| {
954 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
955 })?;
956
957 let (base_loc, bound_loc) = if imported_memories {
959 self.emit_relaxed_binop(
961 Assembler::emit_mov,
962 Size::S64,
963 Location::Memory(self.get_vmctx_reg(), offset),
964 Location::GPR(tmp_addr),
965 true,
966 )?;
967 (Location::Memory(tmp_addr, 0), Location::Memory(tmp_addr, 8))
968 } else {
969 (
970 Location::Memory(self.get_vmctx_reg(), offset),
971 Location::Memory(self.get_vmctx_reg(), offset + 8),
972 )
973 };
974
975 let tmp_base = self.acquire_temp_gpr().ok_or_else(|| {
976 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
977 })?;
978 let tmp_bound = self.acquire_temp_gpr().ok_or_else(|| {
979 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
980 })?;
981
982 self.emit_relaxed_ldr64(Size::S64, Location::GPR(tmp_base), base_loc)?;
984
985 if need_check {
987 self.emit_relaxed_ldr64(Size::S64, Location::GPR(tmp_bound), bound_loc)?;
988
989 self.assembler.emit_add(
993 Size::S64,
994 Location::GPR(tmp_bound),
995 Location::GPR(tmp_base),
996 Location::GPR(tmp_bound),
997 )?;
998 if self.compatible_imm(value_size as _, ImmType::Bits12) {
999 self.assembler.emit_sub(
1000 Size::S64,
1001 Location::GPR(tmp_bound),
1002 Location::Imm32(value_size as _),
1003 Location::GPR(tmp_bound),
1004 )?;
1005 } else {
1006 let tmp2 = self.acquire_temp_gpr().ok_or_else(|| {
1007 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1008 })?;
1009 self.assembler
1010 .emit_mov_imm(Location::GPR(tmp2), value_size as u64)?;
1011 self.assembler.emit_sub(
1012 Size::S64,
1013 Location::GPR(tmp_bound),
1014 Location::GPR(tmp2),
1015 Location::GPR(tmp_bound),
1016 )?;
1017 self.release_gpr(tmp2);
1018 }
1019 }
1020
1021 self.move_location(Size::S32, addr, Location::GPR(tmp_addr))?;
1025
1026 if memarg.offset != 0 {
1028 if self.compatible_imm(memarg.offset as _, ImmType::Bits12) {
1029 self.assembler.emit_adds(
1030 Size::S32,
1031 Location::Imm32(memarg.offset as u32),
1032 Location::GPR(tmp_addr),
1033 Location::GPR(tmp_addr),
1034 )?;
1035 } else {
1036 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
1037 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1038 })?;
1039 self.assembler
1040 .emit_mov_imm(Location::GPR(tmp), memarg.offset as _)?;
1041 self.assembler.emit_adds(
1042 Size::S32,
1043 Location::GPR(tmp_addr),
1044 Location::GPR(tmp),
1045 Location::GPR(tmp_addr),
1046 )?;
1047 self.release_gpr(tmp);
1048 }
1049
1050 self.assembler
1052 .emit_bcond_label_far(Condition::Cs, heap_access_oob)?;
1053 }
1054
1055 self.assembler.emit_add(
1057 Size::S64,
1058 Location::GPR(tmp_base),
1059 Location::GPR(tmp_addr),
1060 Location::GPR(tmp_addr),
1061 )?;
1062
1063 if need_check {
1064 self.assembler.emit_cmp(
1066 Size::S64,
1067 Location::GPR(tmp_bound),
1068 Location::GPR(tmp_addr),
1069 )?;
1070
1071 self.assembler
1073 .emit_bcond_label_far(Condition::Hi, heap_access_oob)?;
1074 }
1075
1076 self.release_gpr(tmp_bound);
1077 self.release_gpr(tmp_base);
1078
1079 let align = value_size as u32;
1080 if check_alignment && align != 1 {
1081 self.assembler.emit_tst(
1082 Size::S64,
1083 Location::Imm32(align - 1),
1084 Location::GPR(tmp_addr),
1085 )?;
1086 self.assembler
1087 .emit_bcond_label_far(Condition::Ne, unaligned_atomic)?;
1088 }
1089 let begin = self.assembler.get_offset().0;
1090 cb(self, tmp_addr)?;
1091 let end = self.assembler.get_offset().0;
1092 self.mark_address_range_with_trap_code(TrapCode::HeapAccessOutOfBounds, begin, end);
1093
1094 self.release_gpr(tmp_addr);
1095 Ok(())
1096 }
1097
1098 fn offset_is_ok(&self, size: Size, offset: i32) -> bool {
1099 if offset < 0 {
1100 return false;
1101 }
1102 let shift = size.bytes().trailing_zeros() as i32;
1103 if offset >= 0x1000 << shift {
1104 return false;
1105 }
1106 if (offset & ((1 << shift) - 1)) != 0 {
1107 return false;
1108 }
1109 true
1110 }
1111
1112 fn emit_push(&mut self, sz: Size, src: Location) -> Result<(), CompileError> {
1113 match (sz, src) {
1114 (Size::S64, Location::GPR(_)) | (Size::S64, Location::SIMD(_)) => {
1115 let offset = if self.pushed {
1116 0
1117 } else {
1118 self.assembler.emit_sub(
1119 Size::S64,
1120 Location::GPR(GPR::XzrSp),
1121 Location::Imm8(16),
1122 Location::GPR(GPR::XzrSp),
1123 )?;
1124 8
1125 };
1126 self.assembler
1127 .emit_stur(Size::S64, src, GPR::XzrSp, offset)?;
1128 self.pushed = !self.pushed;
1129 }
1130 (Size::S64, _) => {
1131 let mut temps = vec![];
1132 let src = self.location_to_reg(sz, src, &mut temps, ImmType::None, true, None)?;
1133 let offset = if self.pushed {
1134 0
1135 } else {
1136 self.assembler.emit_sub(
1137 Size::S64,
1138 Location::GPR(GPR::XzrSp),
1139 Location::Imm8(16),
1140 Location::GPR(GPR::XzrSp),
1141 )?;
1142 8
1143 };
1144 self.assembler
1145 .emit_stur(Size::S64, src, GPR::XzrSp, offset)?;
1146 self.pushed = !self.pushed;
1147 for r in temps {
1148 self.release_gpr(r);
1149 }
1150 }
1151 _ => codegen_error!("singlepass can't emit PUSH {:?} {:?}", sz, src),
1152 }
1153 Ok(())
1154 }
1155 fn emit_double_push(
1156 &mut self,
1157 sz: Size,
1158 src1: Location,
1159 src2: Location,
1160 ) -> Result<(), CompileError> {
1161 if !self.pushed {
1162 match (sz, src1, src2) {
1163 (Size::S64, Location::GPR(_), Location::GPR(_)) => {
1164 self.assembler
1165 .emit_stpdb(Size::S64, src1, src2, GPR::XzrSp, 16)?;
1166 }
1167 _ => {
1168 self.emit_push(sz, src1)?;
1169 self.emit_push(sz, src2)?;
1170 }
1171 }
1172 } else {
1173 self.emit_push(sz, src1)?;
1174 self.emit_push(sz, src2)?;
1175 }
1176 Ok(())
1177 }
1178 fn emit_pop(&mut self, sz: Size, dst: Location) -> Result<(), CompileError> {
1179 match (sz, dst) {
1180 (Size::S64, Location::GPR(_)) | (Size::S64, Location::SIMD(_)) => {
1181 let offset = if self.pushed { 8 } else { 0 };
1182 self.assembler
1183 .emit_ldur(Size::S64, dst, GPR::XzrSp, offset)?;
1184 if self.pushed {
1185 self.assembler.emit_add(
1186 Size::S64,
1187 Location::GPR(GPR::XzrSp),
1188 Location::Imm8(16),
1189 Location::GPR(GPR::XzrSp),
1190 )?;
1191 }
1192 self.pushed = !self.pushed;
1193 }
1194 _ => codegen_error!("singlepass can't emit POP {:?} {:?}", sz, dst),
1195 }
1196 Ok(())
1197 }
1198 fn emit_double_pop(
1199 &mut self,
1200 sz: Size,
1201 dst1: Location,
1202 dst2: Location,
1203 ) -> Result<(), CompileError> {
1204 if !self.pushed {
1205 match (sz, dst1, dst2) {
1206 (Size::S64, Location::GPR(_), Location::GPR(_)) => {
1207 self.assembler
1208 .emit_ldpia(Size::S64, dst1, dst2, GPR::XzrSp, 16)?;
1209 }
1210 _ => {
1211 self.emit_pop(sz, dst2)?;
1212 self.emit_pop(sz, dst1)?;
1213 }
1214 }
1215 } else {
1216 self.emit_pop(sz, dst2)?;
1217 self.emit_pop(sz, dst1)?;
1218 }
1219 Ok(())
1220 }
1221
1222 fn set_default_nan(&mut self, temps: &mut Vec<GPR>) -> Result<GPR, CompileError> {
1223 let old_fpcr = self.acquire_temp_gpr().ok_or_else(|| {
1225 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1226 })?;
1227 temps.push(old_fpcr);
1228 self.assembler.emit_read_fpcr(old_fpcr)?;
1229 let new_fpcr = self.acquire_temp_gpr().ok_or_else(|| {
1230 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1231 })?;
1232 temps.push(new_fpcr);
1233 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
1234 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1235 })?;
1236 temps.push(tmp);
1237 self.assembler
1238 .emit_mov(Size::S32, Location::Imm32(1), Location::GPR(tmp))?;
1239 self.assembler
1240 .emit_mov(Size::S64, Location::GPR(old_fpcr), Location::GPR(new_fpcr))?;
1241 self.assembler.emit_bfi(
1243 Size::S64,
1244 Location::GPR(tmp),
1245 25,
1246 1,
1247 Location::GPR(new_fpcr),
1248 )?;
1249 self.assembler.emit_write_fpcr(new_fpcr)?;
1250 Ok(old_fpcr)
1251 }
1252 fn set_trap_enabled(&mut self, temps: &mut Vec<GPR>) -> Result<GPR, CompileError> {
1253 let old_fpcr = self.acquire_temp_gpr().ok_or_else(|| {
1255 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1256 })?;
1257 temps.push(old_fpcr);
1258 self.assembler.emit_read_fpcr(old_fpcr)?;
1259 let new_fpcr = self.acquire_temp_gpr().ok_or_else(|| {
1260 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1261 })?;
1262 temps.push(new_fpcr);
1263 self.assembler
1264 .emit_mov(Size::S64, Location::GPR(old_fpcr), Location::GPR(new_fpcr))?;
1265 self.assembler
1267 .emit_bfc(Size::S64, 8, 1, Location::GPR(new_fpcr))?;
1268 self.assembler.emit_write_fpcr(new_fpcr)?;
1269 Ok(old_fpcr)
1270 }
1271 fn restore_fpcr(&mut self, old_fpcr: GPR) -> Result<(), CompileError> {
1272 self.assembler.emit_write_fpcr(old_fpcr)
1273 }
1274
1275 fn reset_exception_fpsr(&mut self) -> Result<(), CompileError> {
1276 let fpsr = self.acquire_temp_gpr().ok_or_else(|| {
1278 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1279 })?;
1280 self.assembler.emit_read_fpsr(fpsr)?;
1281 self.assembler
1283 .emit_bfc(Size::S64, 0, 1, Location::GPR(fpsr))?;
1284 self.assembler.emit_write_fpsr(fpsr)?;
1285 self.release_gpr(fpsr);
1286 Ok(())
1287 }
1288 fn read_fpsr(&mut self) -> Result<GPR, CompileError> {
1289 let fpsr = self.acquire_temp_gpr().ok_or_else(|| {
1290 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1291 })?;
1292 self.assembler.emit_read_fpsr(fpsr)?;
1293 Ok(fpsr)
1294 }
1295
1296 fn trap_float_conversion_errors(
1297 &mut self,
1298 old_fpcr: GPR,
1299 sz: Size,
1300 f: Location,
1301 temps: &mut Vec<GPR>,
1302 ) -> Result<(), CompileError> {
1303 let trap_badconv = self.assembler.get_label();
1304 let end = self.assembler.get_label();
1305
1306 let fpsr = self.read_fpsr()?;
1307 temps.push(fpsr);
1308 self.assembler
1310 .emit_tbz_label(Size::S32, Location::GPR(fpsr), 0, end)?;
1311 self.assembler
1313 .emit_bfc(Size::S64, 0, 4, Location::GPR(fpsr))?;
1314 self.restore_fpcr(old_fpcr)?;
1315 self.assembler.emit_fcmp(sz, f, f)?;
1316 self.assembler
1317 .emit_bcond_label(Condition::Vs, trap_badconv)?;
1318 self.emit_illegal_op_internal(TrapCode::IntegerOverflow)?;
1320
1321 self.emit_label(trap_badconv)?;
1322 self.emit_illegal_op_internal(TrapCode::BadConversionToInteger)?;
1323
1324 self.emit_label(end)?;
1325 self.restore_fpcr(old_fpcr)
1326 }
1327
1328 fn used_gprs_contains(&self, r: &GPR) -> bool {
1329 self.used_gprs.contains(r.into_index())
1330 }
1331 fn used_simd_contains(&self, r: &NEON) -> bool {
1332 self.used_simd.contains(r.into_index())
1333 }
1334 fn used_gprs_insert(&mut self, r: GPR) {
1335 self.used_gprs.insert(r.into_index());
1336 }
1337 fn used_simd_insert(&mut self, r: NEON) {
1338 self.used_simd.insert(r.into_index());
1339 }
1340 fn used_gprs_remove(&mut self, r: &GPR) -> bool {
1341 let ret = self.used_gprs_contains(r);
1342 self.used_gprs.set(r.into_index(), false);
1343 ret
1344 }
1345 fn used_simd_remove(&mut self, r: &NEON) -> bool {
1346 let ret = self.used_simd_contains(r);
1347 self.used_simd.set(r.into_index(), false);
1348 ret
1349 }
1350 fn emit_unwind_op(&mut self, op: UnwindOps<GPR, NEON>) {
1351 self.unwind_ops.push((self.get_offset().0, op));
1352 }
1353 fn emit_illegal_op_internal(&mut self, trap: TrapCode) -> Result<(), CompileError> {
1354 self.assembler.emit_udf(0xc0 | (trap as u8) as u16)
1355 }
1356}
1357
1358impl Machine for MachineARM64 {
1359 type GPR = GPR;
1360 type SIMD = NEON;
1361
1362 const STACK_ALIGNMENT: usize = 16;
1363
1364 fn assembler_get_offset(&self) -> Offset {
1365 self.assembler.get_offset()
1366 }
1367
1368 fn get_vmctx_reg(&self) -> GPR {
1369 GPR::X28
1370 }
1371
1372 fn get_used_gprs(&self) -> Vec<GPR> {
1373 GPR::iterator()
1374 .filter(|x| self.used_gprs.contains(x.into_index()))
1375 .cloned()
1376 .collect()
1377 }
1378
1379 fn get_used_simd(&self) -> Vec<NEON> {
1380 NEON::iterator()
1381 .filter(|x| self.used_simd.contains(x.into_index()))
1382 .cloned()
1383 .collect()
1384 }
1385
1386 fn pick_gpr(&self) -> Option<GPR> {
1387 use GPR::*;
1388 static REGS: &[GPR] = &[X9, X10, X11, X12, X13, X14, X15];
1389 for r in REGS {
1390 if !self.used_gprs_contains(r) {
1391 return Some(*r);
1392 }
1393 }
1394 None
1395 }
1396
1397 fn pick_temp_gpr(&self) -> Option<GPR> {
1398 use GPR::*;
1399 static REGS: &[GPR] = &[X8, X7, X6, X5, X4, X3, X2, X1];
1400 for r in REGS {
1401 if !self.used_gprs_contains(r) {
1402 return Some(*r);
1403 }
1404 }
1405 None
1406 }
1407
1408 fn acquire_temp_gpr(&mut self) -> Option<GPR> {
1409 let gpr = self.pick_temp_gpr();
1410 if let Some(x) = gpr {
1411 self.used_gprs_insert(x);
1412 }
1413 gpr
1414 }
1415
1416 fn release_gpr(&mut self, gpr: GPR) {
1417 assert!(self.used_gprs_remove(&gpr));
1418 }
1419
1420 fn reserve_unused_temp_gpr(&mut self, gpr: GPR) -> GPR {
1421 assert!(!self.used_gprs_contains(&gpr));
1422 self.used_gprs_insert(gpr);
1423 gpr
1424 }
1425
1426 fn reserve_gpr(&mut self, gpr: GPR) {
1427 self.used_gprs_insert(gpr);
1428 }
1429
1430 fn push_used_gpr(&mut self, used_gprs: &[GPR]) -> Result<usize, CompileError> {
1431 if used_gprs.len() % 2 == 1 {
1432 self.emit_push(Size::S64, Location::GPR(GPR::XzrSp))?;
1433 }
1434 for r in used_gprs.iter() {
1435 self.emit_push(Size::S64, Location::GPR(*r))?;
1436 }
1437 Ok(used_gprs.len().div_ceil(2) * 16)
1438 }
1439
1440 fn pop_used_gpr(&mut self, used_gprs: &[GPR]) -> Result<(), CompileError> {
1441 for r in used_gprs.iter().rev() {
1442 self.emit_pop(Size::S64, Location::GPR(*r))?;
1443 }
1444 if used_gprs.len() % 2 == 1 {
1445 self.emit_pop(Size::S64, Location::GPR(GPR::XzrSp))?;
1446 }
1447 Ok(())
1448 }
1449
1450 fn pick_simd(&self) -> Option<NEON> {
1451 use NEON::*;
1452 static REGS: &[NEON] = &[V8, V9, V10, V11, V12];
1453 for r in REGS {
1454 if !self.used_simd_contains(r) {
1455 return Some(*r);
1456 }
1457 }
1458 None
1459 }
1460
1461 fn pick_temp_simd(&self) -> Option<NEON> {
1462 use NEON::*;
1463 static REGS: &[NEON] = &[V0, V1, V2, V3, V4, V5, V6, V7];
1464 for r in REGS {
1465 if !self.used_simd_contains(r) {
1466 return Some(*r);
1467 }
1468 }
1469 None
1470 }
1471
1472 fn acquire_temp_simd(&mut self) -> Option<NEON> {
1473 let simd = self.pick_temp_simd();
1474 if let Some(x) = simd {
1475 self.used_simd_insert(x);
1476 }
1477 simd
1478 }
1479
1480 fn reserve_simd(&mut self, simd: NEON) {
1481 self.used_simd_insert(simd);
1482 }
1483
1484 fn release_simd(&mut self, simd: NEON) {
1485 assert!(self.used_simd_remove(&simd));
1486 }
1487
1488 fn push_used_simd(&mut self, used_neons: &[NEON]) -> Result<usize, CompileError> {
1489 let stack_adjust = if used_neons.len() % 2 == 1 {
1490 (used_neons.len() * 8) as u32 + 8
1491 } else {
1492 (used_neons.len() * 8) as u32
1493 };
1494 self.extend_stack(stack_adjust)?;
1495
1496 for (i, r) in used_neons.iter().enumerate() {
1497 self.assembler.emit_str(
1498 Size::S64,
1499 Location::SIMD(*r),
1500 Location::Memory(GPR::XzrSp, (i * 8) as i32),
1501 )?;
1502 }
1503 Ok(stack_adjust as usize)
1504 }
1505
1506 fn pop_used_simd(&mut self, used_neons: &[NEON]) -> Result<(), CompileError> {
1507 for (i, r) in used_neons.iter().enumerate() {
1508 self.assembler.emit_ldr(
1509 Size::S64,
1510 Location::SIMD(*r),
1511 Location::Memory(GPR::XzrSp, (i * 8) as i32),
1512 )?;
1513 }
1514 let stack_adjust = if used_neons.len() % 2 == 1 {
1515 (used_neons.len() * 8) as u32 + 8
1516 } else {
1517 (used_neons.len() * 8) as u32
1518 };
1519 self.assembler.emit_add(
1520 Size::S64,
1521 Location::GPR(GPR::XzrSp),
1522 Location::Imm32(stack_adjust as _),
1523 Location::GPR(GPR::XzrSp),
1524 )
1525 }
1526
1527 fn set_srcloc(&mut self, offset: u32) {
1528 self.src_loc = offset;
1529 }
1530
1531 fn mark_address_range_with_trap_code(&mut self, code: TrapCode, begin: usize, end: usize) {
1532 for i in begin..end {
1533 self.trap_table.offset_to_code.insert(i, code);
1534 }
1535 self.mark_instruction_address_end(begin);
1536 }
1537
1538 fn mark_address_with_trap_code(&mut self, code: TrapCode) {
1539 let offset = self.assembler.get_offset().0;
1540 self.trap_table.offset_to_code.insert(offset, code);
1541 self.mark_instruction_address_end(offset);
1542 }
1543
1544 fn mark_instruction_with_trap_code(&mut self, code: TrapCode) -> usize {
1545 let offset = self.assembler.get_offset().0;
1546 self.trap_table.offset_to_code.insert(offset, code);
1547 offset
1548 }
1549
1550 fn mark_instruction_address_end(&mut self, begin: usize) {
1551 self.instructions_address_map.push(InstructionAddressMap {
1552 srcloc: SourceLoc::new(self.src_loc),
1553 code_offset: begin,
1554 code_len: self.assembler.get_offset().0 - begin,
1555 });
1556 }
1557
1558 fn insert_stackoverflow(&mut self) {
1559 let offset = 0;
1560 self.trap_table
1561 .offset_to_code
1562 .insert(offset, TrapCode::StackOverflow);
1563 self.mark_instruction_address_end(offset);
1564 }
1565
1566 fn collect_trap_information(&self) -> Vec<TrapInformation> {
1567 self.trap_table
1568 .offset_to_code
1569 .clone()
1570 .into_iter()
1571 .map(|(offset, code)| TrapInformation {
1572 code_offset: offset as u32,
1573 trap_code: code,
1574 })
1575 .collect()
1576 }
1577
1578 fn instructions_address_map(&self) -> Vec<InstructionAddressMap> {
1579 self.instructions_address_map.clone()
1580 }
1581
1582 fn local_on_stack(&mut self, stack_offset: i32) -> Location {
1583 Location::Memory(GPR::X29, -stack_offset)
1584 }
1585
1586 fn extend_stack(&mut self, delta_stack_offset: u32) -> Result<(), CompileError> {
1587 let delta = if self.compatible_imm(delta_stack_offset as _, ImmType::Bits12) {
1588 Location::Imm32(delta_stack_offset as _)
1589 } else {
1590 let tmp = SCRATCH_REG;
1591 self.assembler
1592 .emit_mov_imm(Location::GPR(tmp), delta_stack_offset as u64)?;
1593 Location::GPR(tmp)
1594 };
1595 self.assembler.emit_sub(
1596 Size::S64,
1597 Location::GPR(GPR::XzrSp),
1598 delta,
1599 Location::GPR(GPR::XzrSp),
1600 )
1601 }
1602
1603 fn truncate_stack(&mut self, delta_stack_offset: u32) -> Result<(), CompileError> {
1604 let delta = if self.compatible_imm(delta_stack_offset as _, ImmType::Bits12) {
1605 Location::Imm32(delta_stack_offset as _)
1606 } else {
1607 let tmp = SCRATCH_REG;
1608 self.assembler
1609 .emit_mov_imm(Location::GPR(tmp), delta_stack_offset as u64)?;
1610 Location::GPR(tmp)
1611 };
1612 self.assembler.emit_add(
1613 Size::S64,
1614 Location::GPR(GPR::XzrSp),
1615 delta,
1616 Location::GPR(GPR::XzrSp),
1617 )
1618 }
1619
1620 fn move_location_for_native(
1621 &mut self,
1622 size: Size,
1623 loc: Location,
1624 dest: Location,
1625 ) -> Result<(), CompileError> {
1626 match loc {
1627 Location::Imm64(_)
1628 | Location::Imm32(_)
1629 | Location::Imm8(_)
1630 | Location::Memory(_, _)
1631 | Location::Memory2(_, _, _, _) => {
1632 self.move_location(size, loc, Location::GPR(SCRATCH_REG))?;
1633 self.move_location(size, Location::GPR(SCRATCH_REG), dest)
1634 }
1635 _ => self.move_location(size, loc, dest),
1636 }
1637 }
1638
1639 fn zero_location(&mut self, size: Size, location: Location) -> Result<(), CompileError> {
1640 self.move_location(size, Location::GPR(GPR::XzrSp), location)
1641 }
1642
1643 fn local_pointer(&self) -> GPR {
1644 GPR::X29
1645 }
1646
1647 fn is_local_on_stack(&self, idx: usize) -> bool {
1648 idx > 7
1649 }
1650
1651 fn get_local_location(&self, idx: usize, callee_saved_regs_size: usize) -> Location {
1652 match idx {
1654 0 => Location::GPR(GPR::X19),
1655 1 => Location::GPR(GPR::X20),
1656 2 => Location::GPR(GPR::X21),
1657 3 => Location::GPR(GPR::X22),
1658 4 => Location::GPR(GPR::X23),
1659 5 => Location::GPR(GPR::X24),
1660 6 => Location::GPR(GPR::X25),
1661 7 => Location::GPR(GPR::X26),
1662 _ => Location::Memory(GPR::X29, -(((idx - 7) * 8 + callee_saved_regs_size) as i32)),
1663 }
1664 }
1665
1666 fn move_local(&mut self, stack_offset: i32, location: Location) -> Result<(), CompileError> {
1667 if stack_offset < 256 {
1668 self.assembler
1669 .emit_stur(Size::S64, location, GPR::X29, -stack_offset)?;
1670 } else {
1671 let tmp = SCRATCH_REG;
1672 if stack_offset < 0x1_0000 {
1673 self.assembler
1674 .emit_mov_imm(Location::GPR(tmp), (-stack_offset as i64) as u64)?;
1675 self.assembler.emit_str(
1676 Size::S64,
1677 location,
1678 Location::Memory2(GPR::X29, tmp, Multiplier::One, 0),
1679 )?;
1680 } else {
1681 self.assembler
1682 .emit_mov_imm(Location::GPR(tmp), (stack_offset as i64) as u64)?;
1683 self.assembler.emit_sub(
1684 Size::S64,
1685 Location::GPR(GPR::X29),
1686 Location::GPR(tmp),
1687 Location::GPR(tmp),
1688 )?;
1689 self.assembler
1690 .emit_str(Size::S64, location, Location::GPR(tmp))?;
1691 }
1692 }
1693 match location {
1694 Location::GPR(x) => self.emit_unwind_op(UnwindOps::SaveRegister {
1695 reg: UnwindRegister::<GPR, NEON>::GPR(x),
1696 bp_neg_offset: stack_offset,
1697 }),
1698 Location::SIMD(x) => self.emit_unwind_op(UnwindOps::SaveRegister {
1699 reg: UnwindRegister::FPR(x),
1700 bp_neg_offset: stack_offset,
1701 }),
1702 _ => (),
1703 }
1704 Ok(())
1705 }
1706
1707 fn get_param_registers(&self) -> &'static [Self::GPR] {
1708 &[
1709 GPR::X0,
1710 GPR::X1,
1711 GPR::X2,
1712 GPR::X3,
1713 GPR::X4,
1714 GPR::X5,
1715 GPR::X6,
1716 GPR::X7,
1717 ]
1718 }
1719
1720 fn get_param_location(
1721 &self,
1722 idx: usize,
1723 sz: Size,
1724 stack_args: &mut usize,
1725 calling_convention: CallingConvention,
1726 ) -> Location {
1727 let register_params = self.get_param_registers();
1728 match calling_convention {
1729 CallingConvention::AppleAarch64 => register_params.get(idx).map_or_else(
1730 || {
1731 let sz = sz.bytes() as usize;
1733 *stack_args = (*stack_args).next_multiple_of(sz);
1734 let loc = Location::Memory(GPR::XzrSp, *stack_args as i32);
1735 *stack_args += sz;
1736 loc
1737 },
1738 |reg| Location::GPR(*reg),
1739 ),
1740 _ => {
1741 if let Some(reg) = register_params.get(idx) {
1742 Location::GPR(*reg)
1743 } else {
1744 let loc = Location::Memory(GPR::XzrSp, *stack_args as i32);
1745 *stack_args += 8;
1746 loc
1747 }
1748 }
1749 }
1750 }
1751
1752 fn get_call_param_location(
1753 &self,
1754 return_slots: usize,
1755 idx: usize,
1756 sz: Size,
1757 stack_args: &mut usize,
1758 calling_convention: CallingConvention,
1759 ) -> Location {
1760 let register_params = self.get_param_registers();
1761 let return_values_memory_size =
1762 8 * return_slots.saturating_sub(ARM64_RETURN_VALUE_REGISTERS.len()) as i32;
1763
1764 match calling_convention {
1765 CallingConvention::AppleAarch64 => register_params.get(idx).map_or_else(
1766 || {
1767 let sz = sz.bytes() as usize;
1768 *stack_args = (*stack_args).next_multiple_of(sz);
1770 let loc = Location::Memory(
1771 GPR::X29,
1772 16 * 2 + return_values_memory_size + *stack_args as i32,
1773 );
1774 *stack_args += sz;
1775 loc
1776 },
1777 |reg| Location::GPR(*reg),
1778 ),
1779 _ => register_params.get(idx).map_or_else(
1780 || {
1781 let loc = Location::Memory(
1782 GPR::X29,
1783 16 * 2 + return_values_memory_size + *stack_args as i32,
1784 );
1785 *stack_args += 8;
1786 loc
1787 },
1788 |reg| Location::GPR(*reg),
1789 ),
1790 }
1791 }
1792
1793 fn get_simple_param_location(&self, idx: usize) -> Self::GPR {
1794 self.get_param_registers()[idx]
1795 }
1796
1797 fn adjust_gpr_param_location(
1798 &mut self,
1799 _register: Self::GPR,
1800 _size: Size,
1801 ) -> Result<(), CompileError> {
1802 Ok(())
1803 }
1804
1805 fn get_return_value_location(
1806 &self,
1807 idx: usize,
1808 stack_location: &mut usize,
1809 ) -> AbstractLocation<Self::GPR, Self::SIMD> {
1810 ARM64_RETURN_VALUE_REGISTERS.get(idx).map_or_else(
1811 || {
1812 let loc = Location::Memory(GPR::XzrSp, *stack_location as i32);
1813 *stack_location += 8;
1814 loc
1815 },
1816 |reg| Location::GPR(*reg),
1817 )
1818 }
1819
1820 fn get_call_return_value_location(
1821 &self,
1822 idx: usize,
1823 ) -> AbstractLocation<Self::GPR, Self::SIMD> {
1824 ARM64_RETURN_VALUE_REGISTERS.get(idx).map_or_else(
1825 || {
1826 Location::Memory(
1827 GPR::X29,
1828 (16 * 2 + (idx - ARM64_RETURN_VALUE_REGISTERS.len()) * 8) as i32,
1829 )
1830 },
1831 |reg| Location::GPR(*reg),
1832 )
1833 }
1834
1835 fn move_location(
1836 &mut self,
1837 size: Size,
1838 source: Location,
1839 dest: Location,
1840 ) -> Result<(), CompileError> {
1841 match source {
1842 Location::GPR(_) | Location::SIMD(_) => match dest {
1843 Location::GPR(_) | Location::SIMD(_) => self.assembler.emit_mov(size, source, dest),
1844 Location::Memory(addr, offs) => {
1845 if self.offset_is_ok(size, offs) {
1846 self.assembler.emit_str(size, source, dest)
1847 } else if self.compatible_imm(offs as i64, ImmType::UnscaledOffset) {
1848 self.assembler.emit_stur(size, source, addr, offs)
1849 } else {
1850 let tmp = SCRATCH_REG;
1851 if offs < 0 {
1852 self.assembler
1853 .emit_mov_imm(Location::GPR(tmp), (-offs) as u64)?;
1854 self.assembler.emit_sub(
1855 Size::S64,
1856 Location::GPR(addr),
1857 Location::GPR(tmp),
1858 Location::GPR(tmp),
1859 )?;
1860 } else {
1861 self.assembler
1862 .emit_mov_imm(Location::GPR(tmp), offs as u64)?;
1863 self.assembler.emit_add(
1864 Size::S64,
1865 Location::GPR(addr),
1866 Location::GPR(tmp),
1867 Location::GPR(tmp),
1868 )?;
1869 }
1870 self.assembler
1871 .emit_str(size, source, Location::Memory(tmp, 0))
1872 }
1873 }
1874 _ => codegen_error!(
1875 "singlepass can't emit move_location {:?} {:?} => {:?}",
1876 size,
1877 source,
1878 dest
1879 ),
1880 },
1881 Location::Imm8(_) => match dest {
1882 Location::GPR(_) => self.assembler.emit_mov(size, source, dest),
1883 Location::Memory(_, _) => match size {
1884 Size::S64 => self.emit_relaxed_str64(source, dest),
1885 Size::S32 => self.emit_relaxed_str32(source, dest),
1886 Size::S16 => self.emit_relaxed_str16(source, dest),
1887 Size::S8 => self.emit_relaxed_str8(source, dest),
1888 },
1889 _ => codegen_error!(
1890 "singlepass can't emit move_location {:?} {:?} => {:?}",
1891 size,
1892 source,
1893 dest
1894 ),
1895 },
1896 Location::Imm32(val) => match dest {
1897 Location::GPR(_) => self.assembler.emit_mov_imm(dest, val as u64),
1898 Location::Memory(_, _) => match size {
1899 Size::S64 => self.emit_relaxed_str64(source, dest),
1900 Size::S32 => self.emit_relaxed_str32(source, dest),
1901 Size::S16 => self.emit_relaxed_str16(source, dest),
1902 Size::S8 => self.emit_relaxed_str8(source, dest),
1903 },
1904 _ => codegen_error!(
1905 "singlepass can't emit move_location {:?} {:?} => {:?}",
1906 size,
1907 source,
1908 dest
1909 ),
1910 },
1911 Location::Imm64(val) => match dest {
1912 Location::GPR(_) => self.assembler.emit_mov_imm(dest, val),
1913 Location::Memory(_, _) => match size {
1914 Size::S64 => self.emit_relaxed_str64(source, dest),
1915 Size::S32 => self.emit_relaxed_str32(source, dest),
1916 Size::S16 => self.emit_relaxed_str16(source, dest),
1917 Size::S8 => self.emit_relaxed_str8(source, dest),
1918 },
1919 _ => codegen_error!(
1920 "singlepass can't emit move_location {:?} {:?} => {:?}",
1921 size,
1922 source,
1923 dest
1924 ),
1925 },
1926 Location::Memory(addr, offs) => match dest {
1927 Location::GPR(_) | Location::SIMD(_) => {
1928 if self.offset_is_ok(size, offs) {
1929 self.assembler.emit_ldr(size, dest, source)
1930 } else if offs > -256 && offs < 256 {
1931 self.assembler.emit_ldur(size, dest, addr, offs)
1932 } else {
1933 let tmp = SCRATCH_REG;
1934 if offs < 0 {
1935 self.assembler
1936 .emit_mov_imm(Location::GPR(tmp), (-offs) as u64)?;
1937 self.assembler.emit_sub(
1938 Size::S64,
1939 Location::GPR(addr),
1940 Location::GPR(tmp),
1941 Location::GPR(tmp),
1942 )?;
1943 } else {
1944 self.assembler
1945 .emit_mov_imm(Location::GPR(tmp), offs as u64)?;
1946 self.assembler.emit_add(
1947 Size::S64,
1948 Location::GPR(addr),
1949 Location::GPR(tmp),
1950 Location::GPR(tmp),
1951 )?;
1952 }
1953 self.assembler
1954 .emit_ldr(size, dest, Location::Memory(tmp, 0))
1955 }
1956 }
1957 _ => {
1958 let mut temps = vec![];
1959 let src =
1960 self.location_to_reg(size, source, &mut temps, ImmType::None, true, None)?;
1961 self.move_location(size, src, dest)?;
1962 for r in temps {
1963 self.release_gpr(r);
1964 }
1965 Ok(())
1966 }
1967 },
1968 _ => codegen_error!(
1969 "singlepass can't emit move_location {:?} {:?} => {:?}",
1970 size,
1971 source,
1972 dest
1973 ),
1974 }
1975 }
1976
1977 fn move_location_extend(
1978 &mut self,
1979 size_val: Size,
1980 signed: bool,
1981 source: Location,
1982 size_op: Size,
1983 dest: Location,
1984 ) -> Result<(), CompileError> {
1985 if size_op != Size::S64 {
1986 codegen_error!("singlepass move_location_extend unreachable");
1987 }
1988 let mut temps = vec![];
1989 let dst = self.location_to_reg(size_op, dest, &mut temps, ImmType::None, false, None)?;
1990 let src = match (size_val, signed, source) {
1991 (Size::S64, _, _) => source,
1992 (Size::S32, false, Location::GPR(_)) => {
1993 self.assembler.emit_mov(size_val, source, dst)?;
1994 dst
1995 }
1996 (Size::S8, false, Location::GPR(_)) => {
1997 self.assembler.emit_uxtb(size_op, source, dst)?;
1998 dst
1999 }
2000 (Size::S16, false, Location::GPR(_)) => {
2001 self.assembler.emit_uxth(size_op, source, dst)?;
2002 dst
2003 }
2004 (Size::S8, true, Location::GPR(_)) => {
2005 self.assembler.emit_sxtb(size_op, source, dst)?;
2006 dst
2007 }
2008 (Size::S16, true, Location::GPR(_)) => {
2009 self.assembler.emit_sxth(size_op, source, dst)?;
2010 dst
2011 }
2012 (Size::S32, true, Location::GPR(_)) => {
2013 self.assembler.emit_sxtw(size_op, source, dst)?;
2014 dst
2015 }
2016 (Size::S32, false, Location::Memory(_, _)) => {
2017 self.emit_relaxed_ldr32(size_op, dst, source)?;
2018 dst
2019 }
2020 (Size::S32, true, Location::Memory(_, _)) => {
2021 self.emit_relaxed_ldr32s(size_op, dst, source)?;
2022 dst
2023 }
2024 (Size::S16, false, Location::Memory(_, _)) => {
2025 self.emit_relaxed_ldr16(size_op, dst, source)?;
2026 dst
2027 }
2028 (Size::S16, true, Location::Memory(_, _)) => {
2029 self.emit_relaxed_ldr16s(size_op, dst, source)?;
2030 dst
2031 }
2032 (Size::S8, false, Location::Memory(_, _)) => {
2033 self.emit_relaxed_ldr8(size_op, dst, source)?;
2034 dst
2035 }
2036 (Size::S8, true, Location::Memory(_, _)) => {
2037 self.emit_relaxed_ldr8s(size_op, dst, source)?;
2038 dst
2039 }
2040 _ => codegen_error!(
2041 "singlepass can't emit move_location_extend {:?} {:?} {:?} => {:?} {:?}",
2042 size_val,
2043 signed,
2044 source,
2045 size_op,
2046 dest
2047 ),
2048 };
2049 if src != dst {
2050 self.move_location(size_op, src, dst)?;
2051 }
2052 if dst != dest {
2053 self.move_location(size_op, dst, dest)?;
2054 }
2055 for r in temps {
2056 self.release_gpr(r);
2057 }
2058 Ok(())
2059 }
2060
2061 fn init_stack_loc(
2062 &mut self,
2063 init_stack_loc_cnt: u64,
2064 last_stack_loc: Location,
2065 ) -> Result<(), CompileError> {
2066 let label = self.assembler.get_label();
2067 let mut temps = vec![];
2068 let dest = self.acquire_temp_gpr().ok_or_else(|| {
2069 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2070 })?;
2071 temps.push(dest);
2072 let cnt = self.location_to_reg(
2073 Size::S64,
2074 Location::Imm64(init_stack_loc_cnt),
2075 &mut temps,
2076 ImmType::None,
2077 true,
2078 None,
2079 )?;
2080 let dest = match last_stack_loc {
2081 Location::GPR(_) => codegen_error!("singlepass init_stack_loc unreachable"),
2082 Location::SIMD(_) => codegen_error!("singlepass init_stack_loc unreachable"),
2083 Location::Memory(reg, offset) => {
2084 if offset < 0 {
2085 let offset = (-offset) as u32;
2086 if self.compatible_imm(offset as i64, ImmType::Bits12) {
2087 self.assembler.emit_sub(
2088 Size::S64,
2089 Location::GPR(reg),
2090 Location::Imm32(offset),
2091 Location::GPR(dest),
2092 )?;
2093 } else {
2094 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2095 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2096 })?;
2097 self.assembler
2098 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
2099 self.assembler.emit_sub(
2100 Size::S64,
2101 Location::GPR(reg),
2102 Location::GPR(tmp),
2103 Location::GPR(dest),
2104 )?;
2105 temps.push(tmp);
2106 }
2107 dest
2108 } else {
2109 let offset = offset as u32;
2110 if self.compatible_imm(offset as i64, ImmType::Bits12) {
2111 self.assembler.emit_add(
2112 Size::S64,
2113 Location::GPR(reg),
2114 Location::Imm32(offset),
2115 Location::GPR(dest),
2116 )?;
2117 } else {
2118 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2119 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2120 })?;
2121 self.assembler
2122 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
2123 self.assembler.emit_add(
2124 Size::S64,
2125 Location::GPR(reg),
2126 Location::GPR(tmp),
2127 Location::GPR(dest),
2128 )?;
2129 temps.push(tmp);
2130 }
2131 dest
2132 }
2133 }
2134 _ => codegen_error!("singlepass can't emit init_stack_loc {:?}", last_stack_loc),
2135 };
2136 self.assembler.emit_label(label)?;
2137 self.assembler
2138 .emit_stria(Size::S64, Location::GPR(GPR::XzrSp), dest, 8)?;
2139 self.assembler
2140 .emit_sub(Size::S64, cnt, Location::Imm8(1), cnt)?;
2141 self.assembler.emit_cbnz_label(Size::S64, cnt, label)?;
2142 for r in temps {
2143 self.release_gpr(r);
2144 }
2145 Ok(())
2146 }
2147
2148 fn restore_saved_area(&mut self, saved_area_offset: i32) -> Result<(), CompileError> {
2149 let real_delta = if saved_area_offset & 15 != 0 {
2150 self.pushed = true;
2151 saved_area_offset + 8
2152 } else {
2153 self.pushed = false;
2154 saved_area_offset
2155 };
2156 if self.compatible_imm(real_delta as _, ImmType::Bits12) {
2157 self.assembler.emit_sub(
2158 Size::S64,
2159 Location::GPR(GPR::X29),
2160 Location::Imm32(real_delta as _),
2161 Location::GPR(GPR::XzrSp),
2162 )?;
2163 } else {
2164 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2165 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2166 })?;
2167 self.assembler
2168 .emit_mov_imm(Location::GPR(tmp), real_delta as u64)?;
2169 self.assembler.emit_sub(
2170 Size::S64,
2171 Location::GPR(GPR::X29),
2172 Location::GPR(tmp),
2173 Location::GPR(GPR::XzrSp),
2174 )?;
2175 self.release_gpr(tmp);
2176 }
2177 Ok(())
2178 }
2179
2180 fn pop_location(&mut self, location: Location) -> Result<(), CompileError> {
2181 self.emit_pop(Size::S64, location)
2182 }
2183
2184 fn assembler_finalize(
2185 self,
2186 assembly_comments: HashMap<usize, AssemblyComment>,
2187 ) -> Result<FinalizedAssembly, CompileError> {
2188 Ok(FinalizedAssembly {
2189 body: self.assembler.finalize().map_err(|e| {
2190 CompileError::Codegen(format!("Assembler failed finalization with: {e:?}"))
2191 })?,
2192 assembly_comments,
2193 })
2194 }
2195
2196 fn get_offset(&self) -> Offset {
2197 self.assembler.get_offset()
2198 }
2199
2200 fn finalize_function(&mut self) -> Result<(), CompileError> {
2201 self.assembler.finalize_function();
2202 Ok(())
2203 }
2204
2205 fn emit_function_prolog(&mut self) -> Result<(), CompileError> {
2206 self.emit_double_push(Size::S64, Location::GPR(GPR::X29), Location::GPR(GPR::X30))?; self.emit_unwind_op(UnwindOps::Push2Regs {
2208 reg1: UnwindRegister::GPR(GPR::X29),
2209 reg2: UnwindRegister::GPR(GPR::X30),
2210 up_to_sp: 16,
2211 });
2212 self.emit_double_push(Size::S64, Location::GPR(GPR::X27), Location::GPR(GPR::X28))?;
2213 self.emit_unwind_op(UnwindOps::Push2Regs {
2214 reg1: UnwindRegister::GPR(GPR::X27),
2215 reg2: UnwindRegister::GPR(GPR::X28),
2216 up_to_sp: 32,
2217 });
2218 self.assembler.emit_add(
2220 Size::S64,
2221 Location::GPR(GPR::XzrSp),
2222 Location::Imm8(0),
2223 Location::GPR(GPR::X29),
2224 )?;
2225 self.emit_unwind_op(UnwindOps::DefineNewFrame);
2226 Ok(())
2227 }
2228
2229 fn emit_function_epilog(&mut self) -> Result<(), CompileError> {
2230 self.assembler.emit_add(
2232 Size::S64,
2233 Location::GPR(GPR::X29),
2234 Location::Imm8(0),
2235 Location::GPR(GPR::XzrSp),
2236 )?;
2237 self.pushed = false; self.emit_double_pop(Size::S64, Location::GPR(GPR::X27), Location::GPR(GPR::X28))?;
2239 self.emit_double_pop(Size::S64, Location::GPR(GPR::X29), Location::GPR(GPR::X30))?;
2240 Ok(())
2241 }
2242
2243 fn emit_function_return_float(&mut self) -> Result<(), CompileError> {
2244 self.assembler
2245 .emit_mov(Size::S64, Location::GPR(GPR::X0), Location::SIMD(NEON::V0))
2246 }
2247
2248 fn canonicalize_nan(
2249 &mut self,
2250 sz: Size,
2251 input: Location,
2252 output: Location,
2253 ) -> Result<(), CompileError> {
2254 let mut tempn = vec![];
2255 let mut temps = vec![];
2256 let old_fpcr = self.set_default_nan(&mut temps)?;
2257 match (sz, input, output) {
2259 (Size::S32, Location::SIMD(_), Location::SIMD(_)) => {
2260 self.assembler.emit_fmax(sz, input, input, output)?;
2261 }
2262 (Size::S64, Location::SIMD(_), Location::SIMD(_)) => {
2263 self.assembler.emit_fmax(sz, input, input, output)?;
2264 }
2265 (Size::S32, Location::SIMD(_), _) | (Size::S64, Location::SIMD(_), _) => {
2266 let tmp = self.location_to_neon(sz, output, &mut tempn, ImmType::None, false)?;
2267 self.assembler.emit_fmax(sz, input, input, tmp)?;
2268 self.move_location(sz, tmp, output)?;
2269 }
2270 (Size::S32, Location::Memory(_, _), _) | (Size::S64, Location::Memory(_, _), _) => {
2271 let src = self.location_to_neon(sz, input, &mut tempn, ImmType::None, true)?;
2272 let tmp = self.location_to_neon(sz, output, &mut tempn, ImmType::None, false)?;
2273 self.assembler.emit_fmax(sz, src, src, tmp)?;
2274 if tmp != output {
2275 self.move_location(sz, tmp, output)?;
2276 }
2277 }
2278 _ => codegen_error!(
2279 "singlepass can't emit canonicalize_nan {:?} {:?} {:?}",
2280 sz,
2281 input,
2282 output
2283 ),
2284 }
2285
2286 self.restore_fpcr(old_fpcr)?;
2287 for r in temps {
2288 self.release_gpr(r);
2289 }
2290 for r in tempn {
2291 self.release_simd(r);
2292 }
2293 Ok(())
2294 }
2295
2296 fn emit_illegal_op(&mut self, trap: TrapCode) -> Result<(), CompileError> {
2297 let offset = self.assembler.get_offset().0;
2298 self.assembler.emit_udf(0xc0 | (trap as u8) as u16)?;
2299 self.mark_instruction_address_end(offset);
2300 Ok(())
2301 }
2302
2303 fn get_label(&mut self) -> Label {
2304 self.assembler.new_dynamic_label()
2305 }
2306
2307 fn emit_label(&mut self, label: Label) -> Result<(), CompileError> {
2308 self.assembler.emit_label(label)
2309 }
2310
2311 fn get_gpr_for_call(&self) -> GPR {
2312 GPR::X27
2313 }
2314
2315 fn emit_call_register(&mut self, reg: GPR) -> Result<(), CompileError> {
2316 self.assembler.emit_call_register(reg)
2317 }
2318
2319 fn emit_call_label(&mut self, label: Label) -> Result<(), CompileError> {
2320 self.assembler.emit_call_label(label)
2321 }
2322
2323 fn arch_emit_indirect_call_with_trampoline(
2324 &mut self,
2325 location: Location,
2326 ) -> Result<(), CompileError> {
2327 self.assembler
2328 .arch_emit_indirect_call_with_trampoline(location)
2329 }
2330
2331 fn emit_debug_breakpoint(&mut self) -> Result<(), CompileError> {
2332 self.assembler.emit_brk()
2333 }
2334
2335 fn emit_call_location(&mut self, location: Location) -> Result<(), CompileError> {
2336 let mut temps = vec![];
2337 let loc = self.location_to_reg(
2338 Size::S64,
2339 location,
2340 &mut temps,
2341 ImmType::None,
2342 true,
2343 Some(self.get_gpr_for_call()),
2344 )?;
2345 match loc {
2346 Location::GPR(reg) => self.assembler.emit_call_register(reg),
2347 _ => codegen_error!("singlepass can't emit CALL Location"),
2348 }?;
2349 for r in temps {
2350 self.release_gpr(r);
2351 }
2352 Ok(())
2353 }
2354
2355 fn location_add(
2356 &mut self,
2357 size: Size,
2358 source: Location,
2359 dest: Location,
2360 flags: bool,
2361 ) -> Result<(), CompileError> {
2362 let mut temps = vec![];
2363 let src = self.location_to_reg(size, source, &mut temps, ImmType::Bits12, true, None)?;
2364 let dst = self.location_to_reg(size, dest, &mut temps, ImmType::None, true, None)?;
2365 if flags {
2366 self.assembler.emit_adds(size, dst, src, dst)?;
2367 } else {
2368 self.assembler.emit_add(size, dst, src, dst)?;
2369 }
2370 if dst != dest {
2371 self.move_location(size, dst, dest)?;
2372 }
2373 for r in temps {
2374 self.release_gpr(r);
2375 }
2376 Ok(())
2377 }
2378
2379 fn location_cmp(
2380 &mut self,
2381 size: Size,
2382 source: Location,
2383 dest: Location,
2384 ) -> Result<(), CompileError> {
2385 self.emit_relaxed_binop(Assembler::emit_cmp, size, source, dest, false)
2386 }
2387
2388 fn jmp_unconditional(&mut self, label: Label) -> Result<(), CompileError> {
2389 self.assembler.emit_b_label(label)
2390 }
2391
2392 fn jmp_on_condition(
2393 &mut self,
2394 cond: UnsignedCondition,
2395 size: Size,
2396 loc_a: AbstractLocation<Self::GPR, Self::SIMD>,
2397 loc_b: AbstractLocation<Self::GPR, Self::SIMD>,
2398 label: Label,
2399 ) -> Result<(), CompileError> {
2400 self.emit_relaxed_binop(Assembler::emit_cmp, size, loc_b, loc_a, false)?;
2401 let cond = match cond {
2402 UnsignedCondition::Equal => Condition::Eq,
2403 UnsignedCondition::NotEqual => Condition::Ne,
2404 UnsignedCondition::Above => Condition::Hi,
2405 UnsignedCondition::AboveEqual => Condition::Cs,
2406 UnsignedCondition::Below => Condition::Cc,
2407 UnsignedCondition::BelowEqual => Condition::Ls,
2408 };
2409 self.assembler.emit_bcond_label_far(cond, label)
2410 }
2411
2412 fn emit_jmp_to_jumptable(&mut self, label: Label, cond: Location) -> Result<(), CompileError> {
2413 let tmp1 = self.acquire_temp_gpr().ok_or_else(|| {
2414 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2415 })?;
2416 let tmp2 = self.acquire_temp_gpr().ok_or_else(|| {
2417 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2418 })?;
2419
2420 self.assembler.emit_load_label(tmp1, label)?;
2421 self.move_location(Size::S32, cond, Location::GPR(tmp2))?;
2422
2423 self.assembler.emit_add_lsl(
2424 Size::S64,
2425 Location::GPR(tmp1),
2426 Location::GPR(tmp2),
2427 2,
2428 Location::GPR(tmp2),
2429 )?;
2430 self.assembler.emit_b_register(tmp2)?;
2431 self.release_gpr(tmp2);
2432 self.release_gpr(tmp1);
2433 Ok(())
2434 }
2435
2436 fn align_for_loop(&mut self) -> Result<(), CompileError> {
2437 Ok(())
2439 }
2440
2441 fn emit_ret(&mut self) -> Result<(), CompileError> {
2442 self.assembler.emit_ret()
2443 }
2444
2445 fn emit_push(&mut self, size: Size, loc: Location) -> Result<(), CompileError> {
2446 self.emit_push(size, loc)
2447 }
2448
2449 fn emit_pop(&mut self, size: Size, loc: Location) -> Result<(), CompileError> {
2450 self.emit_pop(size, loc)
2451 }
2452
2453 fn emit_memory_fence(&mut self) -> Result<(), CompileError> {
2454 self.assembler.emit_dmb()
2455 }
2456
2457 fn emit_imul_imm32(&mut self, size: Size, imm32: u32, gpr: GPR) -> Result<(), CompileError> {
2458 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2459 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2460 })?;
2461 self.assembler
2462 .emit_mov_imm(Location::GPR(tmp), imm32 as u64)?;
2463 self.assembler.emit_mul(
2464 size,
2465 Location::GPR(gpr),
2466 Location::GPR(tmp),
2467 Location::GPR(gpr),
2468 )?;
2469 self.release_gpr(tmp);
2470 Ok(())
2471 }
2472
2473 fn emit_relaxed_mov(
2474 &mut self,
2475 sz: Size,
2476 src: Location,
2477 dst: Location,
2478 ) -> Result<(), CompileError> {
2479 self.emit_relaxed_binop(Assembler::emit_mov, sz, src, dst, true)
2480 }
2481
2482 fn emit_relaxed_cmp(
2483 &mut self,
2484 sz: Size,
2485 src: Location,
2486 dst: Location,
2487 ) -> Result<(), CompileError> {
2488 self.emit_relaxed_binop(Assembler::emit_cmp, sz, src, dst, false)
2489 }
2490
2491 fn emit_relaxed_sign_extension(
2492 &mut self,
2493 sz_src: Size,
2494 src: Location,
2495 sz_dst: Size,
2496 dst: Location,
2497 ) -> Result<(), CompileError> {
2498 match (src, dst) {
2499 (Location::Memory(_, _), Location::GPR(_)) => match sz_src {
2500 Size::S8 => self.emit_relaxed_ldr8s(sz_dst, dst, src),
2501 Size::S16 => self.emit_relaxed_ldr16s(sz_dst, dst, src),
2502 Size::S32 => self.emit_relaxed_ldr32s(sz_dst, dst, src),
2503 _ => codegen_error!("singlepass emit_relaxed_sign_extension unreachable"),
2504 },
2505 _ => {
2506 let mut temps = vec![];
2507 let src =
2508 self.location_to_reg(sz_src, src, &mut temps, ImmType::None, true, None)?;
2509 let dest =
2510 self.location_to_reg(sz_dst, dst, &mut temps, ImmType::None, false, None)?;
2511 match sz_src {
2512 Size::S8 => self.assembler.emit_sxtb(sz_dst, src, dest),
2513 Size::S16 => self.assembler.emit_sxth(sz_dst, src, dest),
2514 Size::S32 => self.assembler.emit_sxtw(sz_dst, src, dest),
2515 _ => codegen_error!("singlepass emit_relaxed_sign_extension unreachable"),
2516 }?;
2517 if dst != dest {
2518 self.move_location(sz_dst, dest, dst)?;
2519 }
2520 for r in temps {
2521 self.release_gpr(r);
2522 }
2523 Ok(())
2524 }
2525 }
2526 }
2527
2528 fn emit_binop_add32(
2529 &mut self,
2530 loc_a: Location,
2531 loc_b: Location,
2532 ret: Location,
2533 ) -> Result<(), CompileError> {
2534 self.emit_relaxed_binop3(
2535 Assembler::emit_add,
2536 Size::S32,
2537 loc_a,
2538 loc_b,
2539 ret,
2540 ImmType::Bits12,
2541 )
2542 }
2543
2544 fn emit_binop_sub32(
2545 &mut self,
2546 loc_a: Location,
2547 loc_b: Location,
2548 ret: Location,
2549 ) -> Result<(), CompileError> {
2550 self.emit_relaxed_binop3(
2551 Assembler::emit_sub,
2552 Size::S32,
2553 loc_a,
2554 loc_b,
2555 ret,
2556 ImmType::Bits12,
2557 )
2558 }
2559
2560 fn emit_binop_mul32(
2561 &mut self,
2562 loc_a: Location,
2563 loc_b: Location,
2564 ret: Location,
2565 ) -> Result<(), CompileError> {
2566 self.emit_relaxed_binop3(
2567 Assembler::emit_mul,
2568 Size::S32,
2569 loc_a,
2570 loc_b,
2571 ret,
2572 ImmType::None,
2573 )
2574 }
2575
2576 fn emit_binop_udiv32(
2577 &mut self,
2578 loc_a: Location,
2579 loc_b: Location,
2580 ret: Location,
2581 integer_division_by_zero: Label,
2582 ) -> Result<usize, CompileError> {
2583 let mut temps = vec![];
2584 let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
2585 let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
2586 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
2587
2588 self.assembler
2589 .emit_cbz_label_far(Size::S32, src2, integer_division_by_zero)?;
2590 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
2591 self.assembler.emit_udiv(Size::S32, src1, src2, dest)?;
2592 if ret != dest {
2593 self.move_location(Size::S32, dest, ret)?;
2594 }
2595 for r in temps {
2596 self.release_gpr(r);
2597 }
2598 Ok(offset)
2599 }
2600
2601 fn emit_binop_sdiv32(
2602 &mut self,
2603 loc_a: Location,
2604 loc_b: Location,
2605 ret: Location,
2606 integer_division_by_zero: Label,
2607 integer_overflow: Label,
2608 ) -> Result<usize, CompileError> {
2609 let mut temps = vec![];
2610 let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
2611 let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
2612 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
2613
2614 self.assembler
2615 .emit_cbz_label_far(Size::S32, src2, integer_division_by_zero)?;
2616 let label_nooverflow = self.assembler.get_label();
2617 let tmp = self.location_to_reg(
2618 Size::S32,
2619 Location::Imm32(0x80000000),
2620 &mut temps,
2621 ImmType::None,
2622 true,
2623 None,
2624 )?;
2625 self.assembler.emit_cmp(Size::S32, tmp, src1)?;
2626 self.assembler
2627 .emit_bcond_label(Condition::Ne, label_nooverflow)?;
2628 self.assembler.emit_movn(Size::S32, tmp, 0)?;
2629 self.assembler.emit_cmp(Size::S32, tmp, src2)?;
2630 self.assembler
2631 .emit_bcond_label_far(Condition::Eq, integer_overflow)?;
2632 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
2633 self.assembler.emit_label(label_nooverflow)?;
2634 self.assembler.emit_sdiv(Size::S32, src1, src2, dest)?;
2635 if ret != dest {
2636 self.move_location(Size::S32, dest, ret)?;
2637 }
2638 for r in temps {
2639 self.release_gpr(r);
2640 }
2641 Ok(offset)
2642 }
2643
2644 fn emit_binop_urem32(
2645 &mut self,
2646 loc_a: Location,
2647 loc_b: Location,
2648 ret: Location,
2649 integer_division_by_zero: Label,
2650 ) -> Result<usize, CompileError> {
2651 let mut temps = vec![];
2652 let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
2653 let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
2654 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
2655 let dest = if dest == src1 || dest == src2 {
2656 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2657 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2658 })?;
2659 temps.push(tmp);
2660 self.assembler
2661 .emit_mov(Size::S32, dest, Location::GPR(tmp))?;
2662 Location::GPR(tmp)
2663 } else {
2664 dest
2665 };
2666 self.assembler
2667 .emit_cbz_label_far(Size::S32, src2, integer_division_by_zero)?;
2668 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
2669 self.assembler.emit_udiv(Size::S32, src1, src2, dest)?;
2670 self.assembler
2672 .emit_msub(Size::S32, dest, src2, src1, dest)?;
2673 if ret != dest {
2674 self.move_location(Size::S32, dest, ret)?;
2675 }
2676 for r in temps {
2677 self.release_gpr(r);
2678 }
2679 Ok(offset)
2680 }
2681
2682 fn emit_binop_srem32(
2683 &mut self,
2684 loc_a: Location,
2685 loc_b: Location,
2686 ret: Location,
2687 integer_division_by_zero: Label,
2688 ) -> Result<usize, CompileError> {
2689 let mut temps = vec![];
2690 let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
2691 let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
2692 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
2693 let dest = if dest == src1 || dest == src2 {
2694 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2695 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2696 })?;
2697 temps.push(tmp);
2698 self.assembler
2699 .emit_mov(Size::S32, dest, Location::GPR(tmp))?;
2700 Location::GPR(tmp)
2701 } else {
2702 dest
2703 };
2704 self.assembler
2705 .emit_cbz_label_far(Size::S32, src2, integer_division_by_zero)?;
2706 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
2707 self.assembler.emit_sdiv(Size::S32, src1, src2, dest)?;
2708 self.assembler
2710 .emit_msub(Size::S32, dest, src2, src1, dest)?;
2711 if ret != dest {
2712 self.move_location(Size::S32, dest, ret)?;
2713 }
2714 for r in temps {
2715 self.release_gpr(r);
2716 }
2717 Ok(offset)
2718 }
2719
2720 fn emit_binop_and32(
2721 &mut self,
2722 loc_a: Location,
2723 loc_b: Location,
2724 ret: Location,
2725 ) -> Result<(), CompileError> {
2726 self.emit_relaxed_binop3(
2727 Assembler::emit_and,
2728 Size::S32,
2729 loc_a,
2730 loc_b,
2731 ret,
2732 ImmType::Logical32,
2733 )
2734 }
2735
2736 fn emit_binop_or32(
2737 &mut self,
2738 loc_a: Location,
2739 loc_b: Location,
2740 ret: Location,
2741 ) -> Result<(), CompileError> {
2742 self.emit_relaxed_binop3(
2743 Assembler::emit_or,
2744 Size::S32,
2745 loc_a,
2746 loc_b,
2747 ret,
2748 ImmType::Logical32,
2749 )
2750 }
2751
2752 fn emit_binop_xor32(
2753 &mut self,
2754 loc_a: Location,
2755 loc_b: Location,
2756 ret: Location,
2757 ) -> Result<(), CompileError> {
2758 self.emit_relaxed_binop3(
2759 Assembler::emit_eor,
2760 Size::S32,
2761 loc_a,
2762 loc_b,
2763 ret,
2764 ImmType::Logical32,
2765 )
2766 }
2767
2768 fn i32_cmp_ge_s(
2769 &mut self,
2770 loc_a: Location,
2771 loc_b: Location,
2772 ret: Location,
2773 ) -> Result<(), CompileError> {
2774 self.emit_cmpop_i32_dynamic_b(Condition::Ge, loc_a, loc_b, ret)
2775 }
2776
2777 fn i32_cmp_gt_s(
2778 &mut self,
2779 loc_a: Location,
2780 loc_b: Location,
2781 ret: Location,
2782 ) -> Result<(), CompileError> {
2783 self.emit_cmpop_i32_dynamic_b(Condition::Gt, loc_a, loc_b, ret)
2784 }
2785
2786 fn i32_cmp_le_s(
2787 &mut self,
2788 loc_a: Location,
2789 loc_b: Location,
2790 ret: Location,
2791 ) -> Result<(), CompileError> {
2792 self.emit_cmpop_i32_dynamic_b(Condition::Le, loc_a, loc_b, ret)
2793 }
2794
2795 fn i32_cmp_lt_s(
2796 &mut self,
2797 loc_a: Location,
2798 loc_b: Location,
2799 ret: Location,
2800 ) -> Result<(), CompileError> {
2801 self.emit_cmpop_i32_dynamic_b(Condition::Lt, loc_a, loc_b, ret)
2802 }
2803
2804 fn i32_cmp_ge_u(
2805 &mut self,
2806 loc_a: Location,
2807 loc_b: Location,
2808 ret: Location,
2809 ) -> Result<(), CompileError> {
2810 self.emit_cmpop_i32_dynamic_b(Condition::Cs, loc_a, loc_b, ret)
2811 }
2812
2813 fn i32_cmp_gt_u(
2814 &mut self,
2815 loc_a: Location,
2816 loc_b: Location,
2817 ret: Location,
2818 ) -> Result<(), CompileError> {
2819 self.emit_cmpop_i32_dynamic_b(Condition::Hi, loc_a, loc_b, ret)
2820 }
2821
2822 fn i32_cmp_le_u(
2823 &mut self,
2824 loc_a: Location,
2825 loc_b: Location,
2826 ret: Location,
2827 ) -> Result<(), CompileError> {
2828 self.emit_cmpop_i32_dynamic_b(Condition::Ls, loc_a, loc_b, ret)
2829 }
2830
2831 fn i32_cmp_lt_u(
2832 &mut self,
2833 loc_a: Location,
2834 loc_b: Location,
2835 ret: Location,
2836 ) -> Result<(), CompileError> {
2837 self.emit_cmpop_i32_dynamic_b(Condition::Cc, loc_a, loc_b, ret)
2838 }
2839
2840 fn i32_cmp_ne(
2841 &mut self,
2842 loc_a: Location,
2843 loc_b: Location,
2844 ret: Location,
2845 ) -> Result<(), CompileError> {
2846 self.emit_cmpop_i32_dynamic_b(Condition::Ne, loc_a, loc_b, ret)
2847 }
2848
2849 fn i32_cmp_eq(
2850 &mut self,
2851 loc_a: Location,
2852 loc_b: Location,
2853 ret: Location,
2854 ) -> Result<(), CompileError> {
2855 self.emit_cmpop_i32_dynamic_b(Condition::Eq, loc_a, loc_b, ret)
2856 }
2857
2858 fn i32_clz(&mut self, src: Location, dst: Location) -> Result<(), CompileError> {
2859 self.emit_relaxed_binop(Assembler::emit_clz, Size::S32, src, dst, true)
2860 }
2861
2862 fn i32_ctz(&mut self, src: Location, dst: Location) -> Result<(), CompileError> {
2863 let mut temps = vec![];
2864 let src = self.location_to_reg(Size::S32, src, &mut temps, ImmType::None, true, None)?;
2865 let dest = self.location_to_reg(Size::S32, dst, &mut temps, ImmType::None, false, None)?;
2866 self.assembler.emit_rbit(Size::S32, src, dest)?;
2867 self.assembler.emit_clz(Size::S32, dest, dest)?;
2868 if dst != dest {
2869 self.move_location(Size::S32, dest, dst)?;
2870 }
2871 for r in temps {
2872 self.release_gpr(r);
2873 }
2874 Ok(())
2875 }
2876
2877 fn i32_popcnt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
2878 if self.has_neon {
2879 let mut temps = vec![];
2880
2881 let src_gpr =
2882 self.location_to_reg(Size::S32, loc, &mut temps, ImmType::None, true, None)?;
2883 let dst_gpr =
2884 self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
2885
2886 let mut neon_temps = vec![];
2887 let neon_temp = self.acquire_temp_simd().ok_or_else(|| {
2888 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2889 })?;
2890 neon_temps.push(neon_temp);
2891
2892 self.assembler
2893 .emit_fmov(Size::S32, src_gpr, Size::S32, Location::SIMD(neon_temp))?;
2894 self.assembler.emit_cnt(neon_temp, neon_temp)?;
2895 self.assembler.emit_addv(neon_temp, neon_temp)?;
2896 self.assembler
2897 .emit_fmov(Size::S32, Location::SIMD(neon_temp), Size::S32, dst_gpr)?;
2898
2899 if ret != dst_gpr {
2900 self.move_location(Size::S32, dst_gpr, ret)?;
2901 }
2902
2903 for r in temps {
2904 self.release_gpr(r);
2905 }
2906
2907 for r in neon_temps {
2908 self.release_simd(r);
2909 }
2910 } else {
2911 let mut temps = vec![];
2912 let src =
2913 self.location_to_reg(Size::S32, loc, &mut temps, ImmType::None, true, None)?;
2914 let dest =
2915 self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
2916 let src = if src == loc {
2917 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2918 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2919 })?;
2920 temps.push(tmp);
2921 self.assembler
2922 .emit_mov(Size::S32, src, Location::GPR(tmp))?;
2923 Location::GPR(tmp)
2924 } else {
2925 src
2926 };
2927 let tmp = {
2928 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2929 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2930 })?;
2931 temps.push(tmp);
2932 Location::GPR(tmp)
2933 };
2934 let label_loop = self.assembler.get_label();
2935 let label_exit = self.assembler.get_label();
2936 self.assembler
2937 .emit_mov(Size::S32, Location::GPR(GPR::XzrSp), dest)?; self.assembler.emit_cbz_label(Size::S32, src, label_exit)?; self.assembler.emit_label(label_loop)?; self.assembler
2941 .emit_add(Size::S32, dest, Location::Imm8(1), dest)?; self.assembler.emit_clz(Size::S32, src, tmp)?; self.assembler.emit_lsl(Size::S32, src, tmp, src)?; self.assembler
2945 .emit_lsl(Size::S32, src, Location::Imm8(1), src)?; self.assembler.emit_cbnz_label(Size::S32, src, label_loop)?; self.assembler.emit_label(label_exit)?;
2948 if ret != dest {
2949 self.move_location(Size::S32, dest, ret)?;
2950 }
2951 for r in temps {
2952 self.release_gpr(r);
2953 }
2954 }
2955 Ok(())
2956 }
2957
2958 fn i32_shl(
2959 &mut self,
2960 loc_a: Location,
2961 loc_b: Location,
2962 ret: Location,
2963 ) -> Result<(), CompileError> {
2964 self.emit_relaxed_binop3(
2965 Assembler::emit_lsl,
2966 Size::S32,
2967 loc_a,
2968 loc_b,
2969 ret,
2970 ImmType::Shift32No0,
2971 )
2972 }
2973
2974 fn i32_shr(
2975 &mut self,
2976 loc_a: Location,
2977 loc_b: Location,
2978 ret: Location,
2979 ) -> Result<(), CompileError> {
2980 self.emit_relaxed_binop3(
2981 Assembler::emit_lsr,
2982 Size::S32,
2983 loc_a,
2984 loc_b,
2985 ret,
2986 ImmType::Shift32No0,
2987 )
2988 }
2989
2990 fn i32_sar(
2991 &mut self,
2992 loc_a: Location,
2993 loc_b: Location,
2994 ret: Location,
2995 ) -> Result<(), CompileError> {
2996 self.emit_relaxed_binop3(
2997 Assembler::emit_asr,
2998 Size::S32,
2999 loc_a,
3000 loc_b,
3001 ret,
3002 ImmType::Shift32No0,
3003 )
3004 }
3005
3006 fn i32_rol(
3007 &mut self,
3008 loc_a: Location,
3009 loc_b: Location,
3010 ret: Location,
3011 ) -> Result<(), CompileError> {
3012 let mut temps = vec![];
3013 let src2 = match loc_b {
3014 Location::Imm8(imm) => Location::Imm8(32 - (imm & 31)),
3015 Location::Imm32(imm) => Location::Imm8(32 - (imm & 31) as u8),
3016 Location::Imm64(imm) => Location::Imm8(32 - (imm & 31) as u8),
3017 _ => {
3018 let tmp1 = self.location_to_reg(
3019 Size::S32,
3020 Location::Imm32(32),
3021 &mut temps,
3022 ImmType::None,
3023 true,
3024 None,
3025 )?;
3026 let tmp2 =
3027 self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
3028 self.assembler.emit_sub(Size::S32, tmp1, tmp2, tmp1)?;
3029 tmp1
3030 }
3031 };
3032 self.emit_relaxed_binop3(
3033 Assembler::emit_ror,
3034 Size::S32,
3035 loc_a,
3036 src2,
3037 ret,
3038 ImmType::Shift32No0,
3039 )?;
3040 for r in temps {
3041 self.release_gpr(r);
3042 }
3043 Ok(())
3044 }
3045
3046 fn i32_ror(
3047 &mut self,
3048 loc_a: Location,
3049 loc_b: Location,
3050 ret: Location,
3051 ) -> Result<(), CompileError> {
3052 self.emit_relaxed_binop3(
3053 Assembler::emit_ror,
3054 Size::S32,
3055 loc_a,
3056 loc_b,
3057 ret,
3058 ImmType::Shift32No0,
3059 )
3060 }
3061
3062 fn i32_load(
3063 &mut self,
3064 addr: Location,
3065 memarg: &MemArg,
3066 ret: Location,
3067 need_check: bool,
3068 imported_memories: bool,
3069 offset: i32,
3070 heap_access_oob: Label,
3071 unaligned_atomic: Label,
3072 ) -> Result<(), CompileError> {
3073 self.memory_op(
3074 addr,
3075 memarg,
3076 false,
3077 4,
3078 need_check,
3079 imported_memories,
3080 offset,
3081 heap_access_oob,
3082 unaligned_atomic,
3083 |this, addr| this.emit_relaxed_ldr32(Size::S32, ret, Location::Memory(addr, 0)),
3084 )
3085 }
3086
3087 fn i32_load_8u(
3088 &mut self,
3089 addr: Location,
3090 memarg: &MemArg,
3091 ret: Location,
3092 need_check: bool,
3093 imported_memories: bool,
3094 offset: i32,
3095 heap_access_oob: Label,
3096 unaligned_atomic: Label,
3097 ) -> Result<(), CompileError> {
3098 self.memory_op(
3099 addr,
3100 memarg,
3101 false,
3102 1,
3103 need_check,
3104 imported_memories,
3105 offset,
3106 heap_access_oob,
3107 unaligned_atomic,
3108 |this, addr| this.emit_relaxed_ldr8(Size::S32, ret, Location::Memory(addr, 0)),
3109 )
3110 }
3111
3112 fn i32_load_8s(
3113 &mut self,
3114 addr: Location,
3115 memarg: &MemArg,
3116 ret: Location,
3117 need_check: bool,
3118 imported_memories: bool,
3119 offset: i32,
3120 heap_access_oob: Label,
3121 unaligned_atomic: Label,
3122 ) -> Result<(), CompileError> {
3123 self.memory_op(
3124 addr,
3125 memarg,
3126 false,
3127 1,
3128 need_check,
3129 imported_memories,
3130 offset,
3131 heap_access_oob,
3132 unaligned_atomic,
3133 |this, addr| this.emit_relaxed_ldr8s(Size::S32, ret, Location::Memory(addr, 0)),
3134 )
3135 }
3136
3137 fn i32_load_16u(
3138 &mut self,
3139 addr: Location,
3140 memarg: &MemArg,
3141 ret: Location,
3142 need_check: bool,
3143 imported_memories: bool,
3144 offset: i32,
3145 heap_access_oob: Label,
3146 unaligned_atomic: Label,
3147 ) -> Result<(), CompileError> {
3148 self.memory_op(
3149 addr,
3150 memarg,
3151 false,
3152 2,
3153 need_check,
3154 imported_memories,
3155 offset,
3156 heap_access_oob,
3157 unaligned_atomic,
3158 |this, addr| this.emit_relaxed_ldr16(Size::S32, ret, Location::Memory(addr, 0)),
3159 )
3160 }
3161
3162 fn i32_load_16s(
3163 &mut self,
3164 addr: Location,
3165 memarg: &MemArg,
3166 ret: Location,
3167 need_check: bool,
3168 imported_memories: bool,
3169 offset: i32,
3170 heap_access_oob: Label,
3171 unaligned_atomic: Label,
3172 ) -> Result<(), CompileError> {
3173 self.memory_op(
3174 addr,
3175 memarg,
3176 false,
3177 2,
3178 need_check,
3179 imported_memories,
3180 offset,
3181 heap_access_oob,
3182 unaligned_atomic,
3183 |this, addr| this.emit_relaxed_ldr16s(Size::S32, ret, Location::Memory(addr, 0)),
3184 )
3185 }
3186
3187 fn i32_atomic_load(
3188 &mut self,
3189 addr: Location,
3190 memarg: &MemArg,
3191 ret: Location,
3192 need_check: bool,
3193 imported_memories: bool,
3194 offset: i32,
3195 heap_access_oob: Label,
3196 unaligned_atomic: Label,
3197 ) -> Result<(), CompileError> {
3198 self.memory_op(
3199 addr,
3200 memarg,
3201 true,
3202 4,
3203 need_check,
3204 imported_memories,
3205 offset,
3206 heap_access_oob,
3207 unaligned_atomic,
3208 |this, addr| this.emit_relaxed_ldr32(Size::S32, ret, Location::Memory(addr, 0)),
3209 )
3210 }
3211
3212 fn i32_atomic_load_8u(
3213 &mut self,
3214 addr: Location,
3215 memarg: &MemArg,
3216 ret: Location,
3217 need_check: bool,
3218 imported_memories: bool,
3219 offset: i32,
3220 heap_access_oob: Label,
3221 unaligned_atomic: Label,
3222 ) -> Result<(), CompileError> {
3223 self.memory_op(
3224 addr,
3225 memarg,
3226 true,
3227 1,
3228 need_check,
3229 imported_memories,
3230 offset,
3231 heap_access_oob,
3232 unaligned_atomic,
3233 |this, addr| this.emit_relaxed_ldr8(Size::S32, ret, Location::Memory(addr, 0)),
3234 )
3235 }
3236
3237 fn i32_atomic_load_16u(
3238 &mut self,
3239 addr: Location,
3240 memarg: &MemArg,
3241 ret: Location,
3242 need_check: bool,
3243 imported_memories: bool,
3244 offset: i32,
3245 heap_access_oob: Label,
3246 unaligned_atomic: Label,
3247 ) -> Result<(), CompileError> {
3248 self.memory_op(
3249 addr,
3250 memarg,
3251 true,
3252 2,
3253 need_check,
3254 imported_memories,
3255 offset,
3256 heap_access_oob,
3257 unaligned_atomic,
3258 |this, addr| this.emit_relaxed_ldr16(Size::S32, ret, Location::Memory(addr, 0)),
3259 )
3260 }
3261
3262 fn i32_save(
3263 &mut self,
3264 target_value: Location,
3265 memarg: &MemArg,
3266 target_addr: Location,
3267 need_check: bool,
3268 imported_memories: bool,
3269 offset: i32,
3270 heap_access_oob: Label,
3271 unaligned_atomic: Label,
3272 ) -> Result<(), CompileError> {
3273 self.memory_op(
3274 target_addr,
3275 memarg,
3276 false,
3277 4,
3278 need_check,
3279 imported_memories,
3280 offset,
3281 heap_access_oob,
3282 unaligned_atomic,
3283 |this, addr| this.emit_relaxed_str32(target_value, Location::Memory(addr, 0)),
3284 )
3285 }
3286
3287 fn i32_save_8(
3288 &mut self,
3289 target_value: Location,
3290 memarg: &MemArg,
3291 target_addr: Location,
3292 need_check: bool,
3293 imported_memories: bool,
3294 offset: i32,
3295 heap_access_oob: Label,
3296 unaligned_atomic: Label,
3297 ) -> Result<(), CompileError> {
3298 self.memory_op(
3299 target_addr,
3300 memarg,
3301 false,
3302 1,
3303 need_check,
3304 imported_memories,
3305 offset,
3306 heap_access_oob,
3307 unaligned_atomic,
3308 |this, addr| this.emit_relaxed_str8(target_value, Location::Memory(addr, 0)),
3309 )
3310 }
3311
3312 fn i32_save_16(
3313 &mut self,
3314 target_value: Location,
3315 memarg: &MemArg,
3316 target_addr: Location,
3317 need_check: bool,
3318 imported_memories: bool,
3319 offset: i32,
3320 heap_access_oob: Label,
3321 unaligned_atomic: Label,
3322 ) -> Result<(), CompileError> {
3323 self.memory_op(
3324 target_addr,
3325 memarg,
3326 false,
3327 2,
3328 need_check,
3329 imported_memories,
3330 offset,
3331 heap_access_oob,
3332 unaligned_atomic,
3333 |this, addr| this.emit_relaxed_str16(target_value, Location::Memory(addr, 0)),
3334 )
3335 }
3336
3337 fn i32_atomic_save(
3338 &mut self,
3339 target_value: Location,
3340 memarg: &MemArg,
3341 target_addr: Location,
3342 need_check: bool,
3343 imported_memories: bool,
3344 offset: i32,
3345 heap_access_oob: Label,
3346 unaligned_atomic: Label,
3347 ) -> Result<(), CompileError> {
3348 self.memory_op(
3349 target_addr,
3350 memarg,
3351 true,
3352 4,
3353 need_check,
3354 imported_memories,
3355 offset,
3356 heap_access_oob,
3357 unaligned_atomic,
3358 |this, addr| this.emit_relaxed_str32(target_value, Location::Memory(addr, 0)),
3359 )?;
3360 self.assembler.emit_dmb()
3361 }
3362
3363 fn i32_atomic_save_8(
3364 &mut self,
3365 target_value: Location,
3366 memarg: &MemArg,
3367 target_addr: Location,
3368 need_check: bool,
3369 imported_memories: bool,
3370 offset: i32,
3371 heap_access_oob: Label,
3372 unaligned_atomic: Label,
3373 ) -> Result<(), CompileError> {
3374 self.memory_op(
3375 target_addr,
3376 memarg,
3377 true,
3378 1,
3379 need_check,
3380 imported_memories,
3381 offset,
3382 heap_access_oob,
3383 unaligned_atomic,
3384 |this, addr| this.emit_relaxed_str8(target_value, Location::Memory(addr, 0)),
3385 )?;
3386 self.assembler.emit_dmb()
3387 }
3388
3389 fn i32_atomic_save_16(
3390 &mut self,
3391 target_value: Location,
3392 memarg: &MemArg,
3393 target_addr: Location,
3394 need_check: bool,
3395 imported_memories: bool,
3396 offset: i32,
3397 heap_access_oob: Label,
3398 unaligned_atomic: Label,
3399 ) -> Result<(), CompileError> {
3400 self.memory_op(
3401 target_addr,
3402 memarg,
3403 true,
3404 2,
3405 need_check,
3406 imported_memories,
3407 offset,
3408 heap_access_oob,
3409 unaligned_atomic,
3410 |this, addr| this.emit_relaxed_str16(target_value, Location::Memory(addr, 0)),
3411 )?;
3412 self.assembler.emit_dmb()
3413 }
3414
3415 fn i32_atomic_add(
3416 &mut self,
3417 loc: Location,
3418 target: Location,
3419 memarg: &MemArg,
3420 ret: Location,
3421 need_check: bool,
3422 imported_memories: bool,
3423 offset: i32,
3424 heap_access_oob: Label,
3425 unaligned_atomic: Label,
3426 ) -> Result<(), CompileError> {
3427 self.memory_op(
3428 target,
3429 memarg,
3430 true,
3431 4,
3432 need_check,
3433 imported_memories,
3434 offset,
3435 heap_access_oob,
3436 unaligned_atomic,
3437 |this, addr| {
3438 let mut temps = vec![];
3439 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3440 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3441 })?;
3442 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3443 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3444 })?;
3445 let dst =
3446 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3447 let reread = this.get_label();
3448
3449 this.emit_label(reread)?;
3450 this.assembler
3451 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
3452 this.emit_binop_add32(dst, loc, Location::GPR(tmp1))?;
3453 this.assembler.emit_stlxr(
3454 Size::S32,
3455 Location::GPR(tmp2),
3456 Location::GPR(tmp1),
3457 Location::GPR(addr),
3458 )?;
3459 this.assembler
3460 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3461 this.assembler.emit_dmb()?;
3462
3463 if dst != ret {
3464 this.move_location(Size::S32, ret, dst)?;
3465 }
3466 for r in temps {
3467 this.release_gpr(r);
3468 }
3469 this.release_gpr(tmp1);
3470 this.release_gpr(tmp2);
3471 Ok(())
3472 },
3473 )
3474 }
3475
3476 fn i32_atomic_add_8u(
3477 &mut self,
3478 loc: Location,
3479 target: Location,
3480 memarg: &MemArg,
3481 ret: Location,
3482 need_check: bool,
3483 imported_memories: bool,
3484 offset: i32,
3485 heap_access_oob: Label,
3486 unaligned_atomic: Label,
3487 ) -> Result<(), CompileError> {
3488 self.memory_op(
3489 target,
3490 memarg,
3491 true,
3492 1,
3493 need_check,
3494 imported_memories,
3495 offset,
3496 heap_access_oob,
3497 unaligned_atomic,
3498 |this, addr| {
3499 let mut temps = vec![];
3500 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3501 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3502 })?;
3503 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3504 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3505 })?;
3506 let dst =
3507 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3508 let reread = this.get_label();
3509
3510 this.emit_label(reread)?;
3511 this.assembler
3512 .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?;
3513 this.emit_binop_add32(dst, loc, Location::GPR(tmp1))?;
3514 this.assembler.emit_stlxrb(
3515 Size::S32,
3516 Location::GPR(tmp2),
3517 Location::GPR(tmp1),
3518 Location::GPR(addr),
3519 )?;
3520 this.assembler
3521 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3522 this.assembler.emit_dmb()?;
3523
3524 if dst != ret {
3525 this.move_location(Size::S32, ret, dst)?;
3526 }
3527 for r in temps {
3528 this.release_gpr(r);
3529 }
3530 this.release_gpr(tmp1);
3531 this.release_gpr(tmp2);
3532 Ok(())
3533 },
3534 )
3535 }
3536
3537 fn i32_atomic_add_16u(
3538 &mut self,
3539 loc: Location,
3540 target: Location,
3541 memarg: &MemArg,
3542 ret: Location,
3543 need_check: bool,
3544 imported_memories: bool,
3545 offset: i32,
3546 heap_access_oob: Label,
3547 unaligned_atomic: Label,
3548 ) -> Result<(), CompileError> {
3549 self.memory_op(
3550 target,
3551 memarg,
3552 true,
3553 2,
3554 need_check,
3555 imported_memories,
3556 offset,
3557 heap_access_oob,
3558 unaligned_atomic,
3559 |this, addr| {
3560 let mut temps = vec![];
3561 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3562 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3563 })?;
3564 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3565 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3566 })?;
3567 let dst =
3568 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3569 let reread = this.get_label();
3570
3571 this.emit_label(reread)?;
3572 this.assembler
3573 .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?;
3574 this.emit_binop_add32(dst, loc, Location::GPR(tmp1))?;
3575 this.assembler.emit_stlxrh(
3576 Size::S32,
3577 Location::GPR(tmp2),
3578 Location::GPR(tmp1),
3579 Location::GPR(addr),
3580 )?;
3581 this.assembler
3582 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3583 this.assembler.emit_dmb()?;
3584
3585 if dst != ret {
3586 this.move_location(Size::S32, ret, dst)?;
3587 }
3588 for r in temps {
3589 this.release_gpr(r);
3590 }
3591 this.release_gpr(tmp1);
3592 this.release_gpr(tmp2);
3593 Ok(())
3594 },
3595 )
3596 }
3597
3598 fn i32_atomic_sub(
3599 &mut self,
3600 loc: Location,
3601 target: Location,
3602 memarg: &MemArg,
3603 ret: Location,
3604 need_check: bool,
3605 imported_memories: bool,
3606 offset: i32,
3607 heap_access_oob: Label,
3608 unaligned_atomic: Label,
3609 ) -> Result<(), CompileError> {
3610 self.memory_op(
3611 target,
3612 memarg,
3613 true,
3614 4,
3615 need_check,
3616 imported_memories,
3617 offset,
3618 heap_access_oob,
3619 unaligned_atomic,
3620 |this, addr| {
3621 let mut temps = vec![];
3622 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3623 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3624 })?;
3625 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3626 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3627 })?;
3628 let dst =
3629 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3630 let reread = this.get_label();
3631
3632 this.emit_label(reread)?;
3633 this.assembler
3634 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
3635 this.emit_binop_sub32(dst, loc, Location::GPR(tmp1))?;
3636 this.assembler.emit_stlxr(
3637 Size::S32,
3638 Location::GPR(tmp2),
3639 Location::GPR(tmp1),
3640 Location::GPR(addr),
3641 )?;
3642 this.assembler
3643 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3644 this.assembler.emit_dmb()?;
3645
3646 if dst != ret {
3647 this.move_location(Size::S32, ret, dst)?;
3648 }
3649 for r in temps {
3650 this.release_gpr(r);
3651 }
3652 this.release_gpr(tmp1);
3653 this.release_gpr(tmp2);
3654 Ok(())
3655 },
3656 )
3657 }
3658
3659 fn i32_atomic_sub_8u(
3660 &mut self,
3661 loc: Location,
3662 target: Location,
3663 memarg: &MemArg,
3664 ret: Location,
3665 need_check: bool,
3666 imported_memories: bool,
3667 offset: i32,
3668 heap_access_oob: Label,
3669 unaligned_atomic: Label,
3670 ) -> Result<(), CompileError> {
3671 self.memory_op(
3672 target,
3673 memarg,
3674 true,
3675 1,
3676 need_check,
3677 imported_memories,
3678 offset,
3679 heap_access_oob,
3680 unaligned_atomic,
3681 |this, addr| {
3682 let mut temps = vec![];
3683 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3684 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3685 })?;
3686 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3687 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3688 })?;
3689 let dst =
3690 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3691 let reread = this.get_label();
3692
3693 this.emit_label(reread)?;
3694 this.assembler
3695 .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?;
3696 this.emit_binop_sub32(dst, loc, Location::GPR(tmp1))?;
3697 this.assembler.emit_stlxrb(
3698 Size::S32,
3699 Location::GPR(tmp2),
3700 Location::GPR(tmp1),
3701 Location::GPR(addr),
3702 )?;
3703 this.assembler
3704 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3705 this.assembler.emit_dmb()?;
3706
3707 if dst != ret {
3708 this.move_location(Size::S32, ret, dst)?;
3709 }
3710 for r in temps {
3711 this.release_gpr(r);
3712 }
3713 this.release_gpr(tmp1);
3714 this.release_gpr(tmp2);
3715 Ok(())
3716 },
3717 )
3718 }
3719
3720 fn i32_atomic_sub_16u(
3721 &mut self,
3722 loc: Location,
3723 target: Location,
3724 memarg: &MemArg,
3725 ret: Location,
3726 need_check: bool,
3727 imported_memories: bool,
3728 offset: i32,
3729 heap_access_oob: Label,
3730 unaligned_atomic: Label,
3731 ) -> Result<(), CompileError> {
3732 self.memory_op(
3733 target,
3734 memarg,
3735 true,
3736 2,
3737 need_check,
3738 imported_memories,
3739 offset,
3740 heap_access_oob,
3741 unaligned_atomic,
3742 |this, addr| {
3743 let mut temps = vec![];
3744 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3745 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3746 })?;
3747 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3748 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3749 })?;
3750 let dst =
3751 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3752 let reread = this.get_label();
3753
3754 this.emit_label(reread)?;
3755 this.assembler
3756 .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?;
3757 this.emit_binop_sub32(dst, loc, Location::GPR(tmp1))?;
3758 this.assembler.emit_stlxrh(
3759 Size::S32,
3760 Location::GPR(tmp2),
3761 Location::GPR(tmp1),
3762 Location::GPR(addr),
3763 )?;
3764 this.assembler
3765 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3766 this.assembler.emit_dmb()?;
3767
3768 if dst != ret {
3769 this.move_location(Size::S32, ret, dst)?;
3770 }
3771 for r in temps {
3772 this.release_gpr(r);
3773 }
3774 this.release_gpr(tmp1);
3775 this.release_gpr(tmp2);
3776 Ok(())
3777 },
3778 )
3779 }
3780
3781 fn i32_atomic_and(
3782 &mut self,
3783 loc: Location,
3784 target: Location,
3785 memarg: &MemArg,
3786 ret: Location,
3787 need_check: bool,
3788 imported_memories: bool,
3789 offset: i32,
3790 heap_access_oob: Label,
3791 unaligned_atomic: Label,
3792 ) -> Result<(), CompileError> {
3793 self.memory_op(
3794 target,
3795 memarg,
3796 true,
3797 4,
3798 need_check,
3799 imported_memories,
3800 offset,
3801 heap_access_oob,
3802 unaligned_atomic,
3803 |this, addr| {
3804 let mut temps = vec![];
3805 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3806 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3807 })?;
3808 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3809 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3810 })?;
3811 let dst =
3812 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3813 let reread = this.get_label();
3814
3815 this.emit_label(reread)?;
3816 this.assembler
3817 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
3818 this.emit_binop_and32(dst, loc, Location::GPR(tmp1))?;
3819 this.assembler.emit_stlxr(
3820 Size::S32,
3821 Location::GPR(tmp2),
3822 Location::GPR(tmp1),
3823 Location::GPR(addr),
3824 )?;
3825 this.assembler
3826 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3827 this.assembler.emit_dmb()?;
3828
3829 if dst != ret {
3830 this.move_location(Size::S32, ret, dst)?;
3831 }
3832 for r in temps {
3833 this.release_gpr(r);
3834 }
3835 this.release_gpr(tmp1);
3836 this.release_gpr(tmp2);
3837 Ok(())
3838 },
3839 )
3840 }
3841
3842 fn i32_atomic_and_8u(
3843 &mut self,
3844 loc: Location,
3845 target: Location,
3846 memarg: &MemArg,
3847 ret: Location,
3848 need_check: bool,
3849 imported_memories: bool,
3850 offset: i32,
3851 heap_access_oob: Label,
3852 unaligned_atomic: Label,
3853 ) -> Result<(), CompileError> {
3854 self.memory_op(
3855 target,
3856 memarg,
3857 true,
3858 1,
3859 need_check,
3860 imported_memories,
3861 offset,
3862 heap_access_oob,
3863 unaligned_atomic,
3864 |this, addr| {
3865 let mut temps = vec![];
3866 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3867 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3868 })?;
3869 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3870 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3871 })?;
3872 let dst =
3873 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3874 let reread = this.get_label();
3875
3876 this.emit_label(reread)?;
3877 this.assembler
3878 .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?;
3879 this.emit_binop_and32(dst, loc, Location::GPR(tmp1))?;
3880 this.assembler.emit_stlxrb(
3881 Size::S32,
3882 Location::GPR(tmp2),
3883 Location::GPR(tmp1),
3884 Location::GPR(addr),
3885 )?;
3886 this.assembler
3887 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3888 this.assembler.emit_dmb()?;
3889
3890 if dst != ret {
3891 this.move_location(Size::S32, ret, dst)?;
3892 }
3893 for r in temps {
3894 this.release_gpr(r);
3895 }
3896 this.release_gpr(tmp1);
3897 this.release_gpr(tmp2);
3898 Ok(())
3899 },
3900 )
3901 }
3902
3903 fn i32_atomic_and_16u(
3904 &mut self,
3905 loc: Location,
3906 target: Location,
3907 memarg: &MemArg,
3908 ret: Location,
3909 need_check: bool,
3910 imported_memories: bool,
3911 offset: i32,
3912 heap_access_oob: Label,
3913 unaligned_atomic: Label,
3914 ) -> Result<(), CompileError> {
3915 self.memory_op(
3916 target,
3917 memarg,
3918 true,
3919 2,
3920 need_check,
3921 imported_memories,
3922 offset,
3923 heap_access_oob,
3924 unaligned_atomic,
3925 |this, addr| {
3926 let mut temps = vec![];
3927 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3928 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3929 })?;
3930 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3931 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3932 })?;
3933 let dst =
3934 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3935 let reread = this.get_label();
3936
3937 this.emit_label(reread)?;
3938 this.assembler
3939 .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?;
3940 this.emit_binop_and32(dst, loc, Location::GPR(tmp1))?;
3941 this.assembler.emit_stlxrh(
3942 Size::S32,
3943 Location::GPR(tmp2),
3944 Location::GPR(tmp1),
3945 Location::GPR(addr),
3946 )?;
3947 this.assembler
3948 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3949 this.assembler.emit_dmb()?;
3950
3951 if dst != ret {
3952 this.move_location(Size::S32, ret, dst)?;
3953 }
3954 for r in temps {
3955 this.release_gpr(r);
3956 }
3957 this.release_gpr(tmp1);
3958 this.release_gpr(tmp2);
3959 Ok(())
3960 },
3961 )
3962 }
3963
3964 fn i32_atomic_or(
3965 &mut self,
3966 loc: Location,
3967 target: Location,
3968 memarg: &MemArg,
3969 ret: Location,
3970 need_check: bool,
3971 imported_memories: bool,
3972 offset: i32,
3973 heap_access_oob: Label,
3974 unaligned_atomic: Label,
3975 ) -> Result<(), CompileError> {
3976 self.memory_op(
3977 target,
3978 memarg,
3979 true,
3980 4,
3981 need_check,
3982 imported_memories,
3983 offset,
3984 heap_access_oob,
3985 unaligned_atomic,
3986 |this, addr| {
3987 let mut temps = vec![];
3988 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3989 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3990 })?;
3991 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3992 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3993 })?;
3994 let dst =
3995 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3996 let reread = this.get_label();
3997
3998 this.emit_label(reread)?;
3999 this.assembler
4000 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
4001 this.emit_binop_or32(dst, loc, Location::GPR(tmp1))?;
4002 this.assembler.emit_stlxr(
4003 Size::S32,
4004 Location::GPR(tmp2),
4005 Location::GPR(tmp1),
4006 Location::GPR(addr),
4007 )?;
4008 this.assembler
4009 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
4010 this.assembler.emit_dmb()?;
4011
4012 if dst != ret {
4013 this.move_location(Size::S32, ret, dst)?;
4014 }
4015 for r in temps {
4016 this.release_gpr(r);
4017 }
4018 this.release_gpr(tmp1);
4019 this.release_gpr(tmp2);
4020 Ok(())
4021 },
4022 )
4023 }
4024
4025 fn i32_atomic_or_8u(
4026 &mut self,
4027 loc: Location,
4028 target: Location,
4029 memarg: &MemArg,
4030 ret: Location,
4031 need_check: bool,
4032 imported_memories: bool,
4033 offset: i32,
4034 heap_access_oob: Label,
4035 unaligned_atomic: Label,
4036 ) -> Result<(), CompileError> {
4037 self.memory_op(
4038 target,
4039 memarg,
4040 true,
4041 1,
4042 need_check,
4043 imported_memories,
4044 offset,
4045 heap_access_oob,
4046 unaligned_atomic,
4047 |this, addr| {
4048 let mut temps = vec![];
4049 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
4050 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4051 })?;
4052 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
4053 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4054 })?;
4055 let dst =
4056 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4057 let reread = this.get_label();
4058
4059 this.emit_label(reread)?;
4060 this.assembler
4061 .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?;
4062 this.emit_binop_or32(dst, loc, Location::GPR(tmp1))?;
4063 this.assembler.emit_stlxrb(
4064 Size::S32,
4065 Location::GPR(tmp2),
4066 Location::GPR(tmp1),
4067 Location::GPR(addr),
4068 )?;
4069 this.assembler
4070 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
4071 this.assembler.emit_dmb()?;
4072
4073 if dst != ret {
4074 this.move_location(Size::S32, ret, dst)?;
4075 }
4076 for r in temps {
4077 this.release_gpr(r);
4078 }
4079 this.release_gpr(tmp1);
4080 this.release_gpr(tmp2);
4081 Ok(())
4082 },
4083 )
4084 }
4085
4086 fn i32_atomic_or_16u(
4087 &mut self,
4088 loc: Location,
4089 target: Location,
4090 memarg: &MemArg,
4091 ret: Location,
4092 need_check: bool,
4093 imported_memories: bool,
4094 offset: i32,
4095 heap_access_oob: Label,
4096 unaligned_atomic: Label,
4097 ) -> Result<(), CompileError> {
4098 self.memory_op(
4099 target,
4100 memarg,
4101 true,
4102 2,
4103 need_check,
4104 imported_memories,
4105 offset,
4106 heap_access_oob,
4107 unaligned_atomic,
4108 |this, addr| {
4109 let mut temps = vec![];
4110 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
4111 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4112 })?;
4113 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
4114 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4115 })?;
4116 let dst =
4117 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4118 let reread = this.get_label();
4119
4120 this.emit_label(reread)?;
4121 this.assembler
4122 .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?;
4123 this.emit_binop_or32(dst, loc, Location::GPR(tmp1))?;
4124 this.assembler.emit_stlxrh(
4125 Size::S32,
4126 Location::GPR(tmp2),
4127 Location::GPR(tmp1),
4128 Location::GPR(addr),
4129 )?;
4130 this.assembler
4131 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
4132 this.assembler.emit_dmb()?;
4133
4134 if dst != ret {
4135 this.move_location(Size::S32, ret, dst)?;
4136 }
4137 for r in temps {
4138 this.release_gpr(r);
4139 }
4140 this.release_gpr(tmp1);
4141 this.release_gpr(tmp2);
4142 Ok(())
4143 },
4144 )
4145 }
4146
4147 fn i32_atomic_xor(
4148 &mut self,
4149 loc: Location,
4150 target: Location,
4151 memarg: &MemArg,
4152 ret: Location,
4153 need_check: bool,
4154 imported_memories: bool,
4155 offset: i32,
4156 heap_access_oob: Label,
4157 unaligned_atomic: Label,
4158 ) -> Result<(), CompileError> {
4159 self.memory_op(
4160 target,
4161 memarg,
4162 true,
4163 4,
4164 need_check,
4165 imported_memories,
4166 offset,
4167 heap_access_oob,
4168 unaligned_atomic,
4169 |this, addr| {
4170 let mut temps = vec![];
4171 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
4172 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4173 })?;
4174 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
4175 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4176 })?;
4177 let dst =
4178 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4179 let reread = this.get_label();
4180
4181 this.emit_label(reread)?;
4182 this.assembler
4183 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
4184 this.emit_binop_xor32(dst, loc, Location::GPR(tmp1))?;
4185 this.assembler.emit_stlxr(
4186 Size::S32,
4187 Location::GPR(tmp2),
4188 Location::GPR(tmp1),
4189 Location::GPR(addr),
4190 )?;
4191 this.assembler
4192 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
4193 this.assembler.emit_dmb()?;
4194
4195 if dst != ret {
4196 this.move_location(Size::S32, ret, dst)?;
4197 }
4198 for r in temps {
4199 this.release_gpr(r);
4200 }
4201 this.release_gpr(tmp1);
4202 this.release_gpr(tmp2);
4203 Ok(())
4204 },
4205 )
4206 }
4207
4208 fn i32_atomic_xor_8u(
4209 &mut self,
4210 loc: Location,
4211 target: Location,
4212 memarg: &MemArg,
4213 ret: Location,
4214 need_check: bool,
4215 imported_memories: bool,
4216 offset: i32,
4217 heap_access_oob: Label,
4218 unaligned_atomic: Label,
4219 ) -> Result<(), CompileError> {
4220 self.memory_op(
4221 target,
4222 memarg,
4223 true,
4224 1,
4225 need_check,
4226 imported_memories,
4227 offset,
4228 heap_access_oob,
4229 unaligned_atomic,
4230 |this, addr| {
4231 let mut temps = vec![];
4232 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
4233 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4234 })?;
4235 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
4236 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4237 })?;
4238 let dst =
4239 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4240 let reread = this.get_label();
4241
4242 this.emit_label(reread)?;
4243 this.assembler
4244 .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?;
4245 this.emit_binop_xor32(dst, loc, Location::GPR(tmp1))?;
4246 this.assembler.emit_stlxrb(
4247 Size::S32,
4248 Location::GPR(tmp2),
4249 Location::GPR(tmp1),
4250 Location::GPR(addr),
4251 )?;
4252 this.assembler
4253 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
4254 this.assembler.emit_dmb()?;
4255
4256 if dst != ret {
4257 this.move_location(Size::S32, ret, dst)?;
4258 }
4259 for r in temps {
4260 this.release_gpr(r);
4261 }
4262 this.release_gpr(tmp1);
4263 this.release_gpr(tmp2);
4264 Ok(())
4265 },
4266 )
4267 }
4268
4269 fn i32_atomic_xor_16u(
4270 &mut self,
4271 loc: Location,
4272 target: Location,
4273 memarg: &MemArg,
4274 ret: Location,
4275 need_check: bool,
4276 imported_memories: bool,
4277 offset: i32,
4278 heap_access_oob: Label,
4279 unaligned_atomic: Label,
4280 ) -> Result<(), CompileError> {
4281 self.memory_op(
4282 target,
4283 memarg,
4284 true,
4285 2,
4286 need_check,
4287 imported_memories,
4288 offset,
4289 heap_access_oob,
4290 unaligned_atomic,
4291 |this, addr| {
4292 let mut temps = vec![];
4293 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
4294 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4295 })?;
4296 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
4297 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4298 })?;
4299 let dst =
4300 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4301 let reread = this.get_label();
4302
4303 this.emit_label(reread)?;
4304 this.assembler
4305 .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?;
4306 this.emit_binop_xor32(dst, loc, Location::GPR(tmp1))?;
4307 this.assembler.emit_stlxrh(
4308 Size::S32,
4309 Location::GPR(tmp2),
4310 Location::GPR(tmp1),
4311 Location::GPR(addr),
4312 )?;
4313 this.assembler
4314 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
4315 this.assembler.emit_dmb()?;
4316
4317 if dst != ret {
4318 this.move_location(Size::S32, ret, dst)?;
4319 }
4320 for r in temps {
4321 this.release_gpr(r);
4322 }
4323 this.release_gpr(tmp1);
4324 this.release_gpr(tmp2);
4325 Ok(())
4326 },
4327 )
4328 }
4329
4330 fn i32_atomic_xchg(
4331 &mut self,
4332 loc: Location,
4333 target: Location,
4334 memarg: &MemArg,
4335 ret: Location,
4336 need_check: bool,
4337 imported_memories: bool,
4338 offset: i32,
4339 heap_access_oob: Label,
4340 unaligned_atomic: Label,
4341 ) -> Result<(), CompileError> {
4342 self.memory_op(
4343 target,
4344 memarg,
4345 true,
4346 4,
4347 need_check,
4348 imported_memories,
4349 offset,
4350 heap_access_oob,
4351 unaligned_atomic,
4352 |this, addr| {
4353 let mut temps = vec![];
4354 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
4355 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4356 })?;
4357 let dst =
4358 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4359 let org =
4360 this.location_to_reg(Size::S32, loc, &mut temps, ImmType::None, false, None)?;
4361 let reread = this.get_label();
4362
4363 this.emit_label(reread)?;
4364 this.assembler
4365 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
4366 this.assembler.emit_stlxr(
4367 Size::S32,
4368 Location::GPR(tmp),
4369 org,
4370 Location::GPR(addr),
4371 )?;
4372 this.assembler
4373 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
4374 this.assembler.emit_dmb()?;
4375
4376 if dst != ret {
4377 this.move_location(Size::S32, ret, dst)?;
4378 }
4379 for r in temps {
4380 this.release_gpr(r);
4381 }
4382 this.release_gpr(tmp);
4383 Ok(())
4384 },
4385 )
4386 }
4387
4388 fn i32_atomic_xchg_8u(
4389 &mut self,
4390 loc: Location,
4391 target: Location,
4392 memarg: &MemArg,
4393 ret: Location,
4394 need_check: bool,
4395 imported_memories: bool,
4396 offset: i32,
4397 heap_access_oob: Label,
4398 unaligned_atomic: Label,
4399 ) -> Result<(), CompileError> {
4400 self.memory_op(
4401 target,
4402 memarg,
4403 true,
4404 1,
4405 need_check,
4406 imported_memories,
4407 offset,
4408 heap_access_oob,
4409 unaligned_atomic,
4410 |this, addr| {
4411 let mut temps = vec![];
4412 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
4413 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4414 })?;
4415 let dst =
4416 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4417 let org =
4418 this.location_to_reg(Size::S32, loc, &mut temps, ImmType::None, false, None)?;
4419 let reread = this.get_label();
4420
4421 this.emit_label(reread)?;
4422 this.assembler
4423 .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?;
4424 this.assembler.emit_stlxrb(
4425 Size::S32,
4426 Location::GPR(tmp),
4427 org,
4428 Location::GPR(addr),
4429 )?;
4430 this.assembler
4431 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
4432 this.assembler.emit_dmb()?;
4433
4434 if dst != ret {
4435 this.move_location(Size::S32, ret, dst)?;
4436 }
4437 for r in temps {
4438 this.release_gpr(r);
4439 }
4440 this.release_gpr(tmp);
4441 Ok(())
4442 },
4443 )
4444 }
4445
4446 fn i32_atomic_xchg_16u(
4447 &mut self,
4448 loc: Location,
4449 target: Location,
4450 memarg: &MemArg,
4451 ret: Location,
4452 need_check: bool,
4453 imported_memories: bool,
4454 offset: i32,
4455 heap_access_oob: Label,
4456 unaligned_atomic: Label,
4457 ) -> Result<(), CompileError> {
4458 self.memory_op(
4459 target,
4460 memarg,
4461 true,
4462 2,
4463 need_check,
4464 imported_memories,
4465 offset,
4466 heap_access_oob,
4467 unaligned_atomic,
4468 |this, addr| {
4469 let mut temps = vec![];
4470 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
4471 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4472 })?;
4473 let dst =
4474 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4475 let org =
4476 this.location_to_reg(Size::S32, loc, &mut temps, ImmType::None, false, None)?;
4477 let reread = this.get_label();
4478
4479 this.emit_label(reread)?;
4480 this.assembler
4481 .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?;
4482 this.assembler.emit_stlxrh(
4483 Size::S32,
4484 Location::GPR(tmp),
4485 org,
4486 Location::GPR(addr),
4487 )?;
4488 this.assembler
4489 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
4490 this.assembler.emit_dmb()?;
4491
4492 if dst != ret {
4493 this.move_location(Size::S32, ret, dst)?;
4494 }
4495 for r in temps {
4496 this.release_gpr(r);
4497 }
4498 this.release_gpr(tmp);
4499 Ok(())
4500 },
4501 )
4502 }
4503
4504 fn i32_atomic_cmpxchg(
4505 &mut self,
4506 new: Location,
4507 cmp: Location,
4508 target: Location,
4509 memarg: &MemArg,
4510 ret: Location,
4511 need_check: bool,
4512 imported_memories: bool,
4513 offset: i32,
4514 heap_access_oob: Label,
4515 unaligned_atomic: Label,
4516 ) -> Result<(), CompileError> {
4517 self.memory_op(
4518 target,
4519 memarg,
4520 true,
4521 4,
4522 need_check,
4523 imported_memories,
4524 offset,
4525 heap_access_oob,
4526 unaligned_atomic,
4527 |this, addr| {
4528 let mut temps = vec![];
4529 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
4530 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4531 })?;
4532 let dst =
4533 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4534 let org =
4535 this.location_to_reg(Size::S32, new, &mut temps, ImmType::None, false, None)?;
4536 let reread = this.get_label();
4537 let nosame = this.get_label();
4538
4539 this.emit_label(reread)?;
4540 this.assembler
4541 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
4542 this.emit_relaxed_cmp(Size::S32, dst, cmp)?;
4543 this.assembler.emit_bcond_label(Condition::Ne, nosame)?;
4544 this.assembler.emit_stlxr(
4545 Size::S32,
4546 Location::GPR(tmp),
4547 org,
4548 Location::GPR(addr),
4549 )?;
4550 this.assembler
4551 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
4552 this.assembler.emit_dmb()?;
4553
4554 this.emit_label(nosame)?;
4555 if dst != ret {
4556 this.move_location(Size::S32, ret, dst)?;
4557 }
4558 for r in temps {
4559 this.release_gpr(r);
4560 }
4561 this.release_gpr(tmp);
4562 Ok(())
4563 },
4564 )
4565 }
4566
4567 fn i32_atomic_cmpxchg_8u(
4568 &mut self,
4569 new: Location,
4570 cmp: Location,
4571 target: Location,
4572 memarg: &MemArg,
4573 ret: Location,
4574 need_check: bool,
4575 imported_memories: bool,
4576 offset: i32,
4577 heap_access_oob: Label,
4578 unaligned_atomic: Label,
4579 ) -> Result<(), CompileError> {
4580 self.memory_op(
4581 target,
4582 memarg,
4583 true,
4584 1,
4585 need_check,
4586 imported_memories,
4587 offset,
4588 heap_access_oob,
4589 unaligned_atomic,
4590 |this, addr| {
4591 let mut temps = vec![];
4592 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
4593 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4594 })?;
4595 let dst =
4596 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4597 let org =
4598 this.location_to_reg(Size::S32, new, &mut temps, ImmType::None, false, None)?;
4599 let reread = this.get_label();
4600 let nosame = this.get_label();
4601
4602 this.emit_label(reread)?;
4603 this.assembler
4604 .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?;
4605 this.emit_relaxed_cmp(Size::S32, dst, cmp)?;
4606 this.assembler.emit_bcond_label(Condition::Ne, nosame)?;
4607 this.assembler.emit_stlxrb(
4608 Size::S32,
4609 Location::GPR(tmp),
4610 org,
4611 Location::GPR(addr),
4612 )?;
4613 this.assembler
4614 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
4615 this.assembler.emit_dmb()?;
4616
4617 this.emit_label(nosame)?;
4618 if dst != ret {
4619 this.move_location(Size::S32, ret, dst)?;
4620 }
4621 for r in temps {
4622 this.release_gpr(r);
4623 }
4624 this.release_gpr(tmp);
4625 Ok(())
4626 },
4627 )
4628 }
4629
4630 fn i32_atomic_cmpxchg_16u(
4631 &mut self,
4632 new: Location,
4633 cmp: Location,
4634 target: Location,
4635 memarg: &MemArg,
4636 ret: Location,
4637 need_check: bool,
4638 imported_memories: bool,
4639 offset: i32,
4640 heap_access_oob: Label,
4641 unaligned_atomic: Label,
4642 ) -> Result<(), CompileError> {
4643 self.memory_op(
4644 target,
4645 memarg,
4646 true,
4647 2,
4648 need_check,
4649 imported_memories,
4650 offset,
4651 heap_access_oob,
4652 unaligned_atomic,
4653 |this, addr| {
4654 let mut temps = vec![];
4655 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
4656 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4657 })?;
4658 let dst =
4659 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4660 let org =
4661 this.location_to_reg(Size::S32, new, &mut temps, ImmType::None, false, None)?;
4662 let reread = this.get_label();
4663 let nosame = this.get_label();
4664
4665 this.emit_label(reread)?;
4666 this.assembler
4667 .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?;
4668 this.emit_relaxed_cmp(Size::S32, dst, cmp)?;
4669 this.assembler.emit_bcond_label(Condition::Ne, nosame)?;
4670 this.assembler.emit_stlxrh(
4671 Size::S32,
4672 Location::GPR(tmp),
4673 org,
4674 Location::GPR(addr),
4675 )?;
4676 this.assembler
4677 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
4678 this.assembler.emit_dmb()?;
4679
4680 this.emit_label(nosame)?;
4681 if dst != ret {
4682 this.move_location(Size::S32, ret, dst)?;
4683 }
4684 for r in temps {
4685 this.release_gpr(r);
4686 }
4687 this.release_gpr(tmp);
4688 Ok(())
4689 },
4690 )
4691 }
4692
4693 fn emit_call_with_reloc(
4694 &mut self,
4695 reloc_target: RelocationTarget,
4696 ) -> Result<Vec<Relocation>, CompileError> {
4697 let mut relocations = vec![];
4698 let next = self.get_label();
4699 let reloc_at = self.assembler.get_offset().0;
4700 self.emit_label(next)?; self.assembler.emit_call_label(next)?;
4702 relocations.push(Relocation {
4703 kind: RelocationKind::Arm64Call,
4704 reloc_target,
4705 offset: reloc_at as u32,
4706 addend: 0,
4707 });
4708 Ok(relocations)
4709 }
4710
4711 fn emit_binop_add64(
4712 &mut self,
4713 loc_a: Location,
4714 loc_b: Location,
4715 ret: Location,
4716 ) -> Result<(), CompileError> {
4717 self.emit_relaxed_binop3(
4718 Assembler::emit_add,
4719 Size::S64,
4720 loc_a,
4721 loc_b,
4722 ret,
4723 ImmType::Bits12,
4724 )
4725 }
4726
4727 fn emit_binop_sub64(
4728 &mut self,
4729 loc_a: Location,
4730 loc_b: Location,
4731 ret: Location,
4732 ) -> Result<(), CompileError> {
4733 self.emit_relaxed_binop3(
4734 Assembler::emit_sub,
4735 Size::S64,
4736 loc_a,
4737 loc_b,
4738 ret,
4739 ImmType::Bits12,
4740 )
4741 }
4742
4743 fn emit_binop_mul64(
4744 &mut self,
4745 loc_a: Location,
4746 loc_b: Location,
4747 ret: Location,
4748 ) -> Result<(), CompileError> {
4749 self.emit_relaxed_binop3(
4750 Assembler::emit_mul,
4751 Size::S64,
4752 loc_a,
4753 loc_b,
4754 ret,
4755 ImmType::None,
4756 )
4757 }
4758
4759 fn emit_binop_udiv64(
4760 &mut self,
4761 loc_a: Location,
4762 loc_b: Location,
4763 ret: Location,
4764 integer_division_by_zero: Label,
4765 ) -> Result<usize, CompileError> {
4766 let mut temps = vec![];
4767 let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4768 let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4769 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4770
4771 self.assembler
4772 .emit_cbz_label_far(Size::S64, src2, integer_division_by_zero)?;
4773 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
4774 self.assembler.emit_udiv(Size::S64, src1, src2, dest)?;
4775 if ret != dest {
4776 self.move_location(Size::S64, dest, ret)?;
4777 }
4778 for r in temps {
4779 self.release_gpr(r);
4780 }
4781 Ok(offset)
4782 }
4783
4784 fn emit_binop_sdiv64(
4785 &mut self,
4786 loc_a: Location,
4787 loc_b: Location,
4788 ret: Location,
4789 integer_division_by_zero: Label,
4790 integer_overflow: Label,
4791 ) -> Result<usize, CompileError> {
4792 let mut temps = vec![];
4793 let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4794 let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4795 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4796
4797 self.assembler
4798 .emit_cbz_label_far(Size::S64, src2, integer_division_by_zero)?;
4799 let label_nooverflow = self.assembler.get_label();
4800 let tmp = self.location_to_reg(
4801 Size::S64,
4802 Location::Imm64(0x8000000000000000),
4803 &mut temps,
4804 ImmType::None,
4805 true,
4806 None,
4807 )?;
4808 self.assembler.emit_cmp(Size::S64, tmp, src1)?;
4809 self.assembler
4810 .emit_bcond_label(Condition::Ne, label_nooverflow)?;
4811 self.assembler.emit_movn(Size::S64, tmp, 0)?;
4812 self.assembler.emit_cmp(Size::S64, tmp, src2)?;
4813 self.assembler
4814 .emit_bcond_label_far(Condition::Eq, integer_overflow)?;
4815 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
4816 self.assembler.emit_label(label_nooverflow)?;
4817 self.assembler.emit_sdiv(Size::S64, src1, src2, dest)?;
4818 if ret != dest {
4819 self.move_location(Size::S64, dest, ret)?;
4820 }
4821 for r in temps {
4822 self.release_gpr(r);
4823 }
4824 Ok(offset)
4825 }
4826
4827 fn emit_binop_urem64(
4828 &mut self,
4829 loc_a: Location,
4830 loc_b: Location,
4831 ret: Location,
4832 integer_division_by_zero: Label,
4833 ) -> Result<usize, CompileError> {
4834 let mut temps = vec![];
4835 let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4836 let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4837 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4838 let dest = if dest == src1 || dest == src2 {
4839 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
4840 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4841 })?;
4842 temps.push(tmp);
4843 self.assembler
4844 .emit_mov(Size::S32, dest, Location::GPR(tmp))?;
4845 Location::GPR(tmp)
4846 } else {
4847 dest
4848 };
4849 self.assembler
4850 .emit_cbz_label_far(Size::S64, src2, integer_division_by_zero)?;
4851 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
4852 self.assembler.emit_udiv(Size::S64, src1, src2, dest)?;
4853 self.assembler
4855 .emit_msub(Size::S64, dest, src2, src1, dest)?;
4856 if ret != dest {
4857 self.move_location(Size::S64, dest, ret)?;
4858 }
4859 for r in temps {
4860 self.release_gpr(r);
4861 }
4862 Ok(offset)
4863 }
4864
4865 fn emit_binop_srem64(
4866 &mut self,
4867 loc_a: Location,
4868 loc_b: Location,
4869 ret: Location,
4870 integer_division_by_zero: Label,
4871 ) -> Result<usize, CompileError> {
4872 let mut temps = vec![];
4873 let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4874 let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4875 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4876 let dest = if dest == src1 || dest == src2 {
4877 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
4878 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4879 })?;
4880 temps.push(tmp);
4881 self.assembler
4882 .emit_mov(Size::S64, dest, Location::GPR(tmp))?;
4883 Location::GPR(tmp)
4884 } else {
4885 dest
4886 };
4887 self.assembler
4888 .emit_cbz_label_far(Size::S64, src2, integer_division_by_zero)?;
4889 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
4890 self.assembler.emit_sdiv(Size::S64, src1, src2, dest)?;
4891 self.assembler
4893 .emit_msub(Size::S64, dest, src2, src1, dest)?;
4894 if ret != dest {
4895 self.move_location(Size::S64, dest, ret)?;
4896 }
4897 for r in temps {
4898 self.release_gpr(r);
4899 }
4900 Ok(offset)
4901 }
4902
4903 fn emit_binop_and64(
4904 &mut self,
4905 loc_a: Location,
4906 loc_b: Location,
4907 ret: Location,
4908 ) -> Result<(), CompileError> {
4909 self.emit_relaxed_binop3(
4910 Assembler::emit_and,
4911 Size::S64,
4912 loc_a,
4913 loc_b,
4914 ret,
4915 ImmType::Logical64,
4916 )
4917 }
4918
4919 fn emit_binop_or64(
4920 &mut self,
4921 loc_a: Location,
4922 loc_b: Location,
4923 ret: Location,
4924 ) -> Result<(), CompileError> {
4925 self.emit_relaxed_binop3(
4926 Assembler::emit_or,
4927 Size::S64,
4928 loc_a,
4929 loc_b,
4930 ret,
4931 ImmType::Logical64,
4932 )
4933 }
4934
4935 fn emit_binop_xor64(
4936 &mut self,
4937 loc_a: Location,
4938 loc_b: Location,
4939 ret: Location,
4940 ) -> Result<(), CompileError> {
4941 self.emit_relaxed_binop3(
4942 Assembler::emit_eor,
4943 Size::S64,
4944 loc_a,
4945 loc_b,
4946 ret,
4947 ImmType::Logical64,
4948 )
4949 }
4950
4951 fn i64_cmp_ge_s(
4952 &mut self,
4953 loc_a: Location,
4954 loc_b: Location,
4955 ret: Location,
4956 ) -> Result<(), CompileError> {
4957 self.emit_cmpop_i64_dynamic_b(Condition::Ge, loc_a, loc_b, ret)
4958 }
4959
4960 fn i64_cmp_gt_s(
4961 &mut self,
4962 loc_a: Location,
4963 loc_b: Location,
4964 ret: Location,
4965 ) -> Result<(), CompileError> {
4966 self.emit_cmpop_i64_dynamic_b(Condition::Gt, loc_a, loc_b, ret)
4967 }
4968
4969 fn i64_cmp_le_s(
4970 &mut self,
4971 loc_a: Location,
4972 loc_b: Location,
4973 ret: Location,
4974 ) -> Result<(), CompileError> {
4975 self.emit_cmpop_i64_dynamic_b(Condition::Le, loc_a, loc_b, ret)
4976 }
4977
4978 fn i64_cmp_lt_s(
4979 &mut self,
4980 loc_a: Location,
4981 loc_b: Location,
4982 ret: Location,
4983 ) -> Result<(), CompileError> {
4984 self.emit_cmpop_i64_dynamic_b(Condition::Lt, loc_a, loc_b, ret)
4985 }
4986
4987 fn i64_cmp_ge_u(
4988 &mut self,
4989 loc_a: Location,
4990 loc_b: Location,
4991 ret: Location,
4992 ) -> Result<(), CompileError> {
4993 self.emit_cmpop_i64_dynamic_b(Condition::Cs, loc_a, loc_b, ret)
4994 }
4995
4996 fn i64_cmp_gt_u(
4997 &mut self,
4998 loc_a: Location,
4999 loc_b: Location,
5000 ret: Location,
5001 ) -> Result<(), CompileError> {
5002 self.emit_cmpop_i64_dynamic_b(Condition::Hi, loc_a, loc_b, ret)
5003 }
5004
5005 fn i64_cmp_le_u(
5006 &mut self,
5007 loc_a: Location,
5008 loc_b: Location,
5009 ret: Location,
5010 ) -> Result<(), CompileError> {
5011 self.emit_cmpop_i64_dynamic_b(Condition::Ls, loc_a, loc_b, ret)
5012 }
5013
5014 fn i64_cmp_lt_u(
5015 &mut self,
5016 loc_a: Location,
5017 loc_b: Location,
5018 ret: Location,
5019 ) -> Result<(), CompileError> {
5020 self.emit_cmpop_i64_dynamic_b(Condition::Cc, loc_a, loc_b, ret)
5021 }
5022
5023 fn i64_cmp_ne(
5024 &mut self,
5025 loc_a: Location,
5026 loc_b: Location,
5027 ret: Location,
5028 ) -> Result<(), CompileError> {
5029 self.emit_cmpop_i64_dynamic_b(Condition::Ne, loc_a, loc_b, ret)
5030 }
5031
5032 fn i64_cmp_eq(
5033 &mut self,
5034 loc_a: Location,
5035 loc_b: Location,
5036 ret: Location,
5037 ) -> Result<(), CompileError> {
5038 self.emit_cmpop_i64_dynamic_b(Condition::Eq, loc_a, loc_b, ret)
5039 }
5040
5041 fn i64_clz(&mut self, src: Location, dst: Location) -> Result<(), CompileError> {
5042 self.emit_relaxed_binop(Assembler::emit_clz, Size::S64, src, dst, true)
5043 }
5044
5045 fn i64_ctz(&mut self, src: Location, dst: Location) -> Result<(), CompileError> {
5046 let mut temps = vec![];
5047 let src = self.location_to_reg(Size::S64, src, &mut temps, ImmType::None, true, None)?;
5048 let dest = self.location_to_reg(Size::S64, dst, &mut temps, ImmType::None, false, None)?;
5049 self.assembler.emit_rbit(Size::S64, src, dest)?;
5050 self.assembler.emit_clz(Size::S64, dest, dest)?;
5051 if dst != dest {
5052 self.move_location(Size::S64, dest, dst)?;
5053 }
5054 for r in temps {
5055 self.release_gpr(r);
5056 }
5057 Ok(())
5058 }
5059
5060 fn i64_popcnt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
5061 if self.has_neon {
5062 let mut temps = vec![];
5063
5064 let src_gpr =
5065 self.location_to_reg(Size::S64, loc, &mut temps, ImmType::None, true, None)?;
5066 let dst_gpr =
5067 self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
5068
5069 let mut neon_temps = vec![];
5070 let neon_temp = self.acquire_temp_simd().ok_or_else(|| {
5071 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5072 })?;
5073 neon_temps.push(neon_temp);
5074
5075 self.assembler
5076 .emit_fmov(Size::S64, src_gpr, Size::S64, Location::SIMD(neon_temp))?;
5077 self.assembler.emit_cnt(neon_temp, neon_temp)?;
5078 self.assembler.emit_addv(neon_temp, neon_temp)?;
5079 self.assembler
5080 .emit_fmov(Size::S64, Location::SIMD(neon_temp), Size::S64, dst_gpr)?;
5081
5082 if ret != dst_gpr {
5083 self.move_location(Size::S64, dst_gpr, ret)?;
5084 }
5085
5086 for r in temps {
5087 self.release_gpr(r);
5088 }
5089
5090 for r in neon_temps {
5091 self.release_simd(r);
5092 }
5093 } else {
5094 let mut temps = vec![];
5095 let src =
5096 self.location_to_reg(Size::S64, loc, &mut temps, ImmType::None, true, None)?;
5097 let dest =
5098 self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
5099 let src = if src == loc {
5100 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
5101 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5102 })?;
5103 temps.push(tmp);
5104 self.assembler
5105 .emit_mov(Size::S64, src, Location::GPR(tmp))?;
5106 Location::GPR(tmp)
5107 } else {
5108 src
5109 };
5110 let tmp = {
5111 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
5112 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5113 })?;
5114 temps.push(tmp);
5115 Location::GPR(tmp)
5116 };
5117 let label_loop = self.assembler.get_label();
5118 let label_exit = self.assembler.get_label();
5119 self.assembler
5120 .emit_mov(Size::S32, Location::GPR(GPR::XzrSp), dest)?; self.assembler.emit_cbz_label(Size::S64, src, label_exit)?; self.assembler.emit_label(label_loop)?;
5123 self.assembler
5124 .emit_add(Size::S32, dest, Location::Imm8(1), dest)?; self.assembler.emit_clz(Size::S64, src, tmp)?; self.assembler.emit_lsl(Size::S64, src, tmp, src)?; self.assembler
5128 .emit_lsl(Size::S64, src, Location::Imm8(1), src)?; self.assembler.emit_cbnz_label(Size::S64, src, label_loop)?; self.assembler.emit_label(label_exit)?;
5131 if ret != dest {
5132 self.move_location(Size::S64, dest, ret)?;
5133 }
5134 for r in temps {
5135 self.release_gpr(r);
5136 }
5137 }
5138
5139 Ok(())
5140 }
5141
5142 fn i64_shl(
5143 &mut self,
5144 loc_a: Location,
5145 loc_b: Location,
5146 ret: Location,
5147 ) -> Result<(), CompileError> {
5148 self.emit_relaxed_binop3(
5149 Assembler::emit_lsl,
5150 Size::S64,
5151 loc_a,
5152 loc_b,
5153 ret,
5154 ImmType::Shift64No0,
5155 )
5156 }
5157
5158 fn i64_shr(
5159 &mut self,
5160 loc_a: Location,
5161 loc_b: Location,
5162 ret: Location,
5163 ) -> Result<(), CompileError> {
5164 self.emit_relaxed_binop3(
5165 Assembler::emit_lsr,
5166 Size::S64,
5167 loc_a,
5168 loc_b,
5169 ret,
5170 ImmType::Shift64No0,
5171 )
5172 }
5173
5174 fn i64_sar(
5175 &mut self,
5176 loc_a: Location,
5177 loc_b: Location,
5178 ret: Location,
5179 ) -> Result<(), CompileError> {
5180 self.emit_relaxed_binop3(
5181 Assembler::emit_asr,
5182 Size::S64,
5183 loc_a,
5184 loc_b,
5185 ret,
5186 ImmType::Shift64No0,
5187 )
5188 }
5189
5190 fn i64_rol(
5191 &mut self,
5192 loc_a: Location,
5193 loc_b: Location,
5194 ret: Location,
5195 ) -> Result<(), CompileError> {
5196 let mut temps = vec![];
5198 let src2 = match loc_b {
5199 Location::Imm8(imm) => Location::Imm8(64 - (imm & 63)),
5200 Location::Imm32(imm) => Location::Imm8(64 - (imm & 63) as u8),
5201 Location::Imm64(imm) => Location::Imm8(64 - (imm & 63) as u8),
5202 _ => {
5203 let tmp1 = self.location_to_reg(
5204 Size::S64,
5205 Location::Imm32(64),
5206 &mut temps,
5207 ImmType::None,
5208 true,
5209 None,
5210 )?;
5211 let tmp2 =
5212 self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
5213 self.assembler.emit_sub(Size::S64, tmp1, tmp2, tmp1)?;
5214 tmp1
5215 }
5216 };
5217 self.emit_relaxed_binop3(
5218 Assembler::emit_ror,
5219 Size::S64,
5220 loc_a,
5221 src2,
5222 ret,
5223 ImmType::Shift64No0,
5224 )?;
5225 for r in temps {
5226 self.release_gpr(r);
5227 }
5228 Ok(())
5229 }
5230
5231 fn i64_ror(
5232 &mut self,
5233 loc_a: Location,
5234 loc_b: Location,
5235 ret: Location,
5236 ) -> Result<(), CompileError> {
5237 self.emit_relaxed_binop3(
5238 Assembler::emit_ror,
5239 Size::S64,
5240 loc_a,
5241 loc_b,
5242 ret,
5243 ImmType::Shift64No0,
5244 )
5245 }
5246
5247 fn i64_load(
5248 &mut self,
5249 addr: Location,
5250 memarg: &MemArg,
5251 ret: Location,
5252 need_check: bool,
5253 imported_memories: bool,
5254 offset: i32,
5255 heap_access_oob: Label,
5256 unaligned_atomic: Label,
5257 ) -> Result<(), CompileError> {
5258 self.memory_op(
5259 addr,
5260 memarg,
5261 false,
5262 8,
5263 need_check,
5264 imported_memories,
5265 offset,
5266 heap_access_oob,
5267 unaligned_atomic,
5268 |this, addr| this.emit_relaxed_ldr64(Size::S64, ret, Location::Memory(addr, 0)),
5269 )
5270 }
5271
5272 fn i64_load_8u(
5273 &mut self,
5274 addr: Location,
5275 memarg: &MemArg,
5276 ret: Location,
5277 need_check: bool,
5278 imported_memories: bool,
5279 offset: i32,
5280 heap_access_oob: Label,
5281 unaligned_atomic: Label,
5282 ) -> Result<(), CompileError> {
5283 self.memory_op(
5284 addr,
5285 memarg,
5286 false,
5287 1,
5288 need_check,
5289 imported_memories,
5290 offset,
5291 heap_access_oob,
5292 unaligned_atomic,
5293 |this, addr| this.emit_relaxed_ldr8(Size::S64, ret, Location::Memory(addr, 0)),
5294 )
5295 }
5296
5297 fn i64_load_8s(
5298 &mut self,
5299 addr: Location,
5300 memarg: &MemArg,
5301 ret: Location,
5302 need_check: bool,
5303 imported_memories: bool,
5304 offset: i32,
5305 heap_access_oob: Label,
5306 unaligned_atomic: Label,
5307 ) -> Result<(), CompileError> {
5308 self.memory_op(
5309 addr,
5310 memarg,
5311 false,
5312 1,
5313 need_check,
5314 imported_memories,
5315 offset,
5316 heap_access_oob,
5317 unaligned_atomic,
5318 |this, addr| this.emit_relaxed_ldr8s(Size::S64, ret, Location::Memory(addr, 0)),
5319 )
5320 }
5321
5322 fn i64_load_16u(
5323 &mut self,
5324 addr: Location,
5325 memarg: &MemArg,
5326 ret: Location,
5327 need_check: bool,
5328 imported_memories: bool,
5329 offset: i32,
5330 heap_access_oob: Label,
5331 unaligned_atomic: Label,
5332 ) -> Result<(), CompileError> {
5333 self.memory_op(
5334 addr,
5335 memarg,
5336 false,
5337 2,
5338 need_check,
5339 imported_memories,
5340 offset,
5341 heap_access_oob,
5342 unaligned_atomic,
5343 |this, addr| this.emit_relaxed_ldr16(Size::S64, ret, Location::Memory(addr, 0)),
5344 )
5345 }
5346
5347 fn i64_load_16s(
5348 &mut self,
5349 addr: Location,
5350 memarg: &MemArg,
5351 ret: Location,
5352 need_check: bool,
5353 imported_memories: bool,
5354 offset: i32,
5355 heap_access_oob: Label,
5356 unaligned_atomic: Label,
5357 ) -> Result<(), CompileError> {
5358 self.memory_op(
5359 addr,
5360 memarg,
5361 false,
5362 2,
5363 need_check,
5364 imported_memories,
5365 offset,
5366 heap_access_oob,
5367 unaligned_atomic,
5368 |this, addr| this.emit_relaxed_ldr16s(Size::S64, ret, Location::Memory(addr, 0)),
5369 )
5370 }
5371
5372 fn i64_load_32u(
5373 &mut self,
5374 addr: Location,
5375 memarg: &MemArg,
5376 ret: Location,
5377 need_check: bool,
5378 imported_memories: bool,
5379 offset: i32,
5380 heap_access_oob: Label,
5381 unaligned_atomic: Label,
5382 ) -> Result<(), CompileError> {
5383 self.memory_op(
5384 addr,
5385 memarg,
5386 false,
5387 4,
5388 need_check,
5389 imported_memories,
5390 offset,
5391 heap_access_oob,
5392 unaligned_atomic,
5393 |this, addr| this.emit_relaxed_ldr32(Size::S64, ret, Location::Memory(addr, 0)),
5394 )
5395 }
5396
5397 fn i64_load_32s(
5398 &mut self,
5399 addr: Location,
5400 memarg: &MemArg,
5401 ret: Location,
5402 need_check: bool,
5403 imported_memories: bool,
5404 offset: i32,
5405 heap_access_oob: Label,
5406 unaligned_atomic: Label,
5407 ) -> Result<(), CompileError> {
5408 self.memory_op(
5409 addr,
5410 memarg,
5411 false,
5412 4,
5413 need_check,
5414 imported_memories,
5415 offset,
5416 heap_access_oob,
5417 unaligned_atomic,
5418 |this, addr| this.emit_relaxed_ldr32s(Size::S64, ret, Location::Memory(addr, 0)),
5419 )
5420 }
5421
5422 fn i64_atomic_load(
5423 &mut self,
5424 addr: Location,
5425 memarg: &MemArg,
5426 ret: Location,
5427 need_check: bool,
5428 imported_memories: bool,
5429 offset: i32,
5430 heap_access_oob: Label,
5431 unaligned_atomic: Label,
5432 ) -> Result<(), CompileError> {
5433 self.memory_op(
5434 addr,
5435 memarg,
5436 true,
5437 8,
5438 need_check,
5439 imported_memories,
5440 offset,
5441 heap_access_oob,
5442 unaligned_atomic,
5443 |this, addr| this.emit_relaxed_ldr64(Size::S64, ret, Location::Memory(addr, 0)),
5444 )
5445 }
5446
5447 fn i64_atomic_load_8u(
5448 &mut self,
5449 addr: Location,
5450 memarg: &MemArg,
5451 ret: Location,
5452 need_check: bool,
5453 imported_memories: bool,
5454 offset: i32,
5455 heap_access_oob: Label,
5456 unaligned_atomic: Label,
5457 ) -> Result<(), CompileError> {
5458 self.memory_op(
5459 addr,
5460 memarg,
5461 true,
5462 1,
5463 need_check,
5464 imported_memories,
5465 offset,
5466 heap_access_oob,
5467 unaligned_atomic,
5468 |this, addr| this.emit_relaxed_ldr8(Size::S64, ret, Location::Memory(addr, 0)),
5469 )
5470 }
5471
5472 fn i64_atomic_load_16u(
5473 &mut self,
5474 addr: Location,
5475 memarg: &MemArg,
5476 ret: Location,
5477 need_check: bool,
5478 imported_memories: bool,
5479 offset: i32,
5480 heap_access_oob: Label,
5481 unaligned_atomic: Label,
5482 ) -> Result<(), CompileError> {
5483 self.memory_op(
5484 addr,
5485 memarg,
5486 true,
5487 2,
5488 need_check,
5489 imported_memories,
5490 offset,
5491 heap_access_oob,
5492 unaligned_atomic,
5493 |this, addr| this.emit_relaxed_ldr16(Size::S64, ret, Location::Memory(addr, 0)),
5494 )
5495 }
5496
5497 fn i64_atomic_load_32u(
5498 &mut self,
5499 addr: Location,
5500 memarg: &MemArg,
5501 ret: Location,
5502 need_check: bool,
5503 imported_memories: bool,
5504 offset: i32,
5505 heap_access_oob: Label,
5506 unaligned_atomic: Label,
5507 ) -> Result<(), CompileError> {
5508 self.memory_op(
5509 addr,
5510 memarg,
5511 true,
5512 4,
5513 need_check,
5514 imported_memories,
5515 offset,
5516 heap_access_oob,
5517 unaligned_atomic,
5518 |this, addr| this.emit_relaxed_ldr32(Size::S64, ret, Location::Memory(addr, 0)),
5519 )
5520 }
5521
5522 fn i64_save(
5523 &mut self,
5524 target_value: Location,
5525 memarg: &MemArg,
5526 target_addr: Location,
5527 need_check: bool,
5528 imported_memories: bool,
5529 offset: i32,
5530 heap_access_oob: Label,
5531 unaligned_atomic: Label,
5532 ) -> Result<(), CompileError> {
5533 self.memory_op(
5534 target_addr,
5535 memarg,
5536 false,
5537 8,
5538 need_check,
5539 imported_memories,
5540 offset,
5541 heap_access_oob,
5542 unaligned_atomic,
5543 |this, addr| this.emit_relaxed_str64(target_value, Location::Memory(addr, 0)),
5544 )
5545 }
5546
5547 fn i64_save_8(
5548 &mut self,
5549 target_value: Location,
5550 memarg: &MemArg,
5551 target_addr: Location,
5552 need_check: bool,
5553 imported_memories: bool,
5554 offset: i32,
5555 heap_access_oob: Label,
5556 unaligned_atomic: Label,
5557 ) -> Result<(), CompileError> {
5558 self.memory_op(
5559 target_addr,
5560 memarg,
5561 false,
5562 1,
5563 need_check,
5564 imported_memories,
5565 offset,
5566 heap_access_oob,
5567 unaligned_atomic,
5568 |this, addr| this.emit_relaxed_str8(target_value, Location::Memory(addr, 0)),
5569 )
5570 }
5571
5572 fn i64_save_16(
5573 &mut self,
5574 target_value: Location,
5575 memarg: &MemArg,
5576 target_addr: Location,
5577 need_check: bool,
5578 imported_memories: bool,
5579 offset: i32,
5580 heap_access_oob: Label,
5581 unaligned_atomic: Label,
5582 ) -> Result<(), CompileError> {
5583 self.memory_op(
5584 target_addr,
5585 memarg,
5586 false,
5587 2,
5588 need_check,
5589 imported_memories,
5590 offset,
5591 heap_access_oob,
5592 unaligned_atomic,
5593 |this, addr| this.emit_relaxed_str16(target_value, Location::Memory(addr, 0)),
5594 )
5595 }
5596
5597 fn i64_save_32(
5598 &mut self,
5599 target_value: Location,
5600 memarg: &MemArg,
5601 target_addr: Location,
5602 need_check: bool,
5603 imported_memories: bool,
5604 offset: i32,
5605 heap_access_oob: Label,
5606 unaligned_atomic: Label,
5607 ) -> Result<(), CompileError> {
5608 self.memory_op(
5609 target_addr,
5610 memarg,
5611 false,
5612 4,
5613 need_check,
5614 imported_memories,
5615 offset,
5616 heap_access_oob,
5617 unaligned_atomic,
5618 |this, addr| this.emit_relaxed_str32(target_value, Location::Memory(addr, 0)),
5619 )
5620 }
5621
5622 fn i64_atomic_save(
5623 &mut self,
5624 target_value: Location,
5625 memarg: &MemArg,
5626 target_addr: Location,
5627 need_check: bool,
5628 imported_memories: bool,
5629 offset: i32,
5630 heap_access_oob: Label,
5631 unaligned_atomic: Label,
5632 ) -> Result<(), CompileError> {
5633 self.memory_op(
5634 target_addr,
5635 memarg,
5636 true,
5637 8,
5638 need_check,
5639 imported_memories,
5640 offset,
5641 heap_access_oob,
5642 unaligned_atomic,
5643 |this, addr| this.emit_relaxed_str64(target_value, Location::Memory(addr, 0)),
5644 )?;
5645 self.assembler.emit_dmb()
5646 }
5647
5648 fn i64_atomic_save_8(
5649 &mut self,
5650 target_value: Location,
5651 memarg: &MemArg,
5652 target_addr: Location,
5653 need_check: bool,
5654 imported_memories: bool,
5655 offset: i32,
5656 heap_access_oob: Label,
5657 unaligned_atomic: Label,
5658 ) -> Result<(), CompileError> {
5659 self.memory_op(
5660 target_addr,
5661 memarg,
5662 true,
5663 1,
5664 need_check,
5665 imported_memories,
5666 offset,
5667 heap_access_oob,
5668 unaligned_atomic,
5669 |this, addr| this.emit_relaxed_str8(target_value, Location::Memory(addr, 0)),
5670 )?;
5671 self.assembler.emit_dmb()
5672 }
5673
5674 fn i64_atomic_save_16(
5675 &mut self,
5676 target_value: Location,
5677 memarg: &MemArg,
5678 target_addr: Location,
5679 need_check: bool,
5680 imported_memories: bool,
5681 offset: i32,
5682 heap_access_oob: Label,
5683 unaligned_atomic: Label,
5684 ) -> Result<(), CompileError> {
5685 self.memory_op(
5686 target_addr,
5687 memarg,
5688 true,
5689 2,
5690 need_check,
5691 imported_memories,
5692 offset,
5693 heap_access_oob,
5694 unaligned_atomic,
5695 |this, addr| this.emit_relaxed_str16(target_value, Location::Memory(addr, 0)),
5696 )?;
5697 self.assembler.emit_dmb()
5698 }
5699
5700 fn i64_atomic_save_32(
5701 &mut self,
5702 target_value: Location,
5703 memarg: &MemArg,
5704 target_addr: Location,
5705 need_check: bool,
5706 imported_memories: bool,
5707 offset: i32,
5708 heap_access_oob: Label,
5709 unaligned_atomic: Label,
5710 ) -> Result<(), CompileError> {
5711 self.memory_op(
5712 target_addr,
5713 memarg,
5714 true,
5715 4,
5716 need_check,
5717 imported_memories,
5718 offset,
5719 heap_access_oob,
5720 unaligned_atomic,
5721 |this, addr| this.emit_relaxed_str32(target_value, Location::Memory(addr, 0)),
5722 )?;
5723 self.assembler.emit_dmb()
5724 }
5725
5726 fn i64_atomic_add(
5727 &mut self,
5728 loc: Location,
5729 target: Location,
5730 memarg: &MemArg,
5731 ret: Location,
5732 need_check: bool,
5733 imported_memories: bool,
5734 offset: i32,
5735 heap_access_oob: Label,
5736 unaligned_atomic: Label,
5737 ) -> Result<(), CompileError> {
5738 self.memory_op(
5739 target,
5740 memarg,
5741 true,
5742 8,
5743 need_check,
5744 imported_memories,
5745 offset,
5746 heap_access_oob,
5747 unaligned_atomic,
5748 |this, addr| {
5749 let mut temps = vec![];
5750 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
5751 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5752 })?;
5753 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
5754 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5755 })?;
5756 let dst =
5757 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
5758 let reread = this.get_label();
5759
5760 this.emit_label(reread)?;
5761 this.assembler
5762 .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?;
5763 this.emit_binop_add64(dst, loc, Location::GPR(tmp1))?;
5764 this.assembler.emit_stlxr(
5765 Size::S64,
5766 Location::GPR(tmp2),
5767 Location::GPR(tmp1),
5768 Location::GPR(addr),
5769 )?;
5770 this.assembler
5771 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
5772 this.assembler.emit_dmb()?;
5773
5774 if dst != ret {
5775 this.move_location(Size::S64, ret, dst)?;
5776 }
5777 for r in temps {
5778 this.release_gpr(r);
5779 }
5780 this.release_gpr(tmp1);
5781 this.release_gpr(tmp2);
5782 Ok(())
5783 },
5784 )
5785 }
5786
5787 fn i64_atomic_add_8u(
5788 &mut self,
5789 loc: Location,
5790 target: Location,
5791 memarg: &MemArg,
5792 ret: Location,
5793 need_check: bool,
5794 imported_memories: bool,
5795 offset: i32,
5796 heap_access_oob: Label,
5797 unaligned_atomic: Label,
5798 ) -> Result<(), CompileError> {
5799 self.memory_op(
5800 target,
5801 memarg,
5802 true,
5803 1,
5804 need_check,
5805 imported_memories,
5806 offset,
5807 heap_access_oob,
5808 unaligned_atomic,
5809 |this, addr| {
5810 let mut temps = vec![];
5811 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
5812 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5813 })?;
5814 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
5815 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5816 })?;
5817 let dst =
5818 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
5819 let reread = this.get_label();
5820
5821 this.emit_label(reread)?;
5822 this.assembler
5823 .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?;
5824 this.emit_binop_add64(dst, loc, Location::GPR(tmp1))?;
5825 this.assembler.emit_stlxrb(
5826 Size::S64,
5827 Location::GPR(tmp2),
5828 Location::GPR(tmp1),
5829 Location::GPR(addr),
5830 )?;
5831 this.assembler
5832 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
5833 this.assembler.emit_dmb()?;
5834
5835 if dst != ret {
5836 this.move_location(Size::S64, ret, dst)?;
5837 }
5838 for r in temps {
5839 this.release_gpr(r);
5840 }
5841 this.release_gpr(tmp1);
5842 this.release_gpr(tmp2);
5843 Ok(())
5844 },
5845 )
5846 }
5847
5848 fn i64_atomic_add_16u(
5849 &mut self,
5850 loc: Location,
5851 target: Location,
5852 memarg: &MemArg,
5853 ret: Location,
5854 need_check: bool,
5855 imported_memories: bool,
5856 offset: i32,
5857 heap_access_oob: Label,
5858 unaligned_atomic: Label,
5859 ) -> Result<(), CompileError> {
5860 self.memory_op(
5861 target,
5862 memarg,
5863 true,
5864 2,
5865 need_check,
5866 imported_memories,
5867 offset,
5868 heap_access_oob,
5869 unaligned_atomic,
5870 |this, addr| {
5871 let mut temps = vec![];
5872 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
5873 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5874 })?;
5875 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
5876 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5877 })?;
5878 let dst =
5879 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
5880 let reread = this.get_label();
5881
5882 this.emit_label(reread)?;
5883 this.assembler
5884 .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?;
5885 this.emit_binop_add64(dst, loc, Location::GPR(tmp1))?;
5886 this.assembler.emit_stlxrh(
5887 Size::S64,
5888 Location::GPR(tmp2),
5889 Location::GPR(tmp1),
5890 Location::GPR(addr),
5891 )?;
5892 this.assembler
5893 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
5894 this.assembler.emit_dmb()?;
5895
5896 if dst != ret {
5897 this.move_location(Size::S64, ret, dst)?;
5898 }
5899 for r in temps {
5900 this.release_gpr(r);
5901 }
5902 this.release_gpr(tmp1);
5903 this.release_gpr(tmp2);
5904 Ok(())
5905 },
5906 )
5907 }
5908
5909 fn i64_atomic_add_32u(
5910 &mut self,
5911 loc: Location,
5912 target: Location,
5913 memarg: &MemArg,
5914 ret: Location,
5915 need_check: bool,
5916 imported_memories: bool,
5917 offset: i32,
5918 heap_access_oob: Label,
5919 unaligned_atomic: Label,
5920 ) -> Result<(), CompileError> {
5921 self.memory_op(
5922 target,
5923 memarg,
5924 true,
5925 4,
5926 need_check,
5927 imported_memories,
5928 offset,
5929 heap_access_oob,
5930 unaligned_atomic,
5931 |this, addr| {
5932 let mut temps = vec![];
5933 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
5934 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5935 })?;
5936 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
5937 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5938 })?;
5939 let dst =
5940 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
5941 let reread = this.get_label();
5942
5943 this.emit_label(reread)?;
5944 this.assembler
5945 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
5946 this.emit_binop_add64(dst, loc, Location::GPR(tmp1))?;
5947 this.assembler.emit_stlxr(
5948 Size::S32,
5949 Location::GPR(tmp2),
5950 Location::GPR(tmp1),
5951 Location::GPR(addr),
5952 )?;
5953 this.assembler
5954 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
5955 this.assembler.emit_dmb()?;
5956
5957 if dst != ret {
5958 this.move_location(Size::S64, ret, dst)?;
5959 }
5960 for r in temps {
5961 this.release_gpr(r);
5962 }
5963 this.release_gpr(tmp1);
5964 this.release_gpr(tmp2);
5965 Ok(())
5966 },
5967 )
5968 }
5969
5970 fn i64_atomic_sub(
5971 &mut self,
5972 loc: Location,
5973 target: Location,
5974 memarg: &MemArg,
5975 ret: Location,
5976 need_check: bool,
5977 imported_memories: bool,
5978 offset: i32,
5979 heap_access_oob: Label,
5980 unaligned_atomic: Label,
5981 ) -> Result<(), CompileError> {
5982 self.memory_op(
5983 target,
5984 memarg,
5985 true,
5986 8,
5987 need_check,
5988 imported_memories,
5989 offset,
5990 heap_access_oob,
5991 unaligned_atomic,
5992 |this, addr| {
5993 let mut temps = vec![];
5994 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
5995 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5996 })?;
5997 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
5998 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5999 })?;
6000 let dst =
6001 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6002 let reread = this.get_label();
6003
6004 this.emit_label(reread)?;
6005 this.assembler
6006 .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?;
6007 this.emit_binop_sub64(dst, loc, Location::GPR(tmp1))?;
6008 this.assembler.emit_stlxr(
6009 Size::S64,
6010 Location::GPR(tmp2),
6011 Location::GPR(tmp1),
6012 Location::GPR(addr),
6013 )?;
6014 this.assembler
6015 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6016 this.assembler.emit_dmb()?;
6017
6018 if dst != ret {
6019 this.move_location(Size::S64, ret, dst)?;
6020 }
6021 for r in temps {
6022 this.release_gpr(r);
6023 }
6024 this.release_gpr(tmp1);
6025 this.release_gpr(tmp2);
6026 Ok(())
6027 },
6028 )
6029 }
6030
6031 fn i64_atomic_sub_8u(
6032 &mut self,
6033 loc: Location,
6034 target: Location,
6035 memarg: &MemArg,
6036 ret: Location,
6037 need_check: bool,
6038 imported_memories: bool,
6039 offset: i32,
6040 heap_access_oob: Label,
6041 unaligned_atomic: Label,
6042 ) -> Result<(), CompileError> {
6043 self.memory_op(
6044 target,
6045 memarg,
6046 true,
6047 1,
6048 need_check,
6049 imported_memories,
6050 offset,
6051 heap_access_oob,
6052 unaligned_atomic,
6053 |this, addr| {
6054 let mut temps = vec![];
6055 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6056 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6057 })?;
6058 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6059 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6060 })?;
6061 let dst =
6062 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6063 let reread = this.get_label();
6064
6065 this.emit_label(reread)?;
6066 this.assembler
6067 .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?;
6068 this.emit_binop_sub64(dst, loc, Location::GPR(tmp1))?;
6069 this.assembler.emit_stlxrb(
6070 Size::S64,
6071 Location::GPR(tmp2),
6072 Location::GPR(tmp1),
6073 Location::GPR(addr),
6074 )?;
6075 this.assembler
6076 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6077 this.assembler.emit_dmb()?;
6078
6079 if dst != ret {
6080 this.move_location(Size::S64, ret, dst)?;
6081 }
6082 for r in temps {
6083 this.release_gpr(r);
6084 }
6085 this.release_gpr(tmp1);
6086 this.release_gpr(tmp2);
6087 Ok(())
6088 },
6089 )
6090 }
6091
6092 fn i64_atomic_sub_16u(
6093 &mut self,
6094 loc: Location,
6095 target: Location,
6096 memarg: &MemArg,
6097 ret: Location,
6098 need_check: bool,
6099 imported_memories: bool,
6100 offset: i32,
6101 heap_access_oob: Label,
6102 unaligned_atomic: Label,
6103 ) -> Result<(), CompileError> {
6104 self.memory_op(
6105 target,
6106 memarg,
6107 true,
6108 2,
6109 need_check,
6110 imported_memories,
6111 offset,
6112 heap_access_oob,
6113 unaligned_atomic,
6114 |this, addr| {
6115 let mut temps = vec![];
6116 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6117 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6118 })?;
6119 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6120 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6121 })?;
6122 let dst =
6123 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6124 let reread = this.get_label();
6125
6126 this.emit_label(reread)?;
6127 this.assembler
6128 .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?;
6129 this.emit_binop_sub64(dst, loc, Location::GPR(tmp1))?;
6130 this.assembler.emit_stlxrh(
6131 Size::S64,
6132 Location::GPR(tmp2),
6133 Location::GPR(tmp1),
6134 Location::GPR(addr),
6135 )?;
6136 this.assembler
6137 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6138 this.assembler.emit_dmb()?;
6139
6140 if dst != ret {
6141 this.move_location(Size::S64, ret, dst)?;
6142 }
6143 for r in temps {
6144 this.release_gpr(r);
6145 }
6146 this.release_gpr(tmp1);
6147 this.release_gpr(tmp2);
6148 Ok(())
6149 },
6150 )
6151 }
6152
6153 fn i64_atomic_sub_32u(
6154 &mut self,
6155 loc: Location,
6156 target: Location,
6157 memarg: &MemArg,
6158 ret: Location,
6159 need_check: bool,
6160 imported_memories: bool,
6161 offset: i32,
6162 heap_access_oob: Label,
6163 unaligned_atomic: Label,
6164 ) -> Result<(), CompileError> {
6165 self.memory_op(
6166 target,
6167 memarg,
6168 true,
6169 4,
6170 need_check,
6171 imported_memories,
6172 offset,
6173 heap_access_oob,
6174 unaligned_atomic,
6175 |this, addr| {
6176 let mut temps = vec![];
6177 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6178 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6179 })?;
6180 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6181 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6182 })?;
6183 let dst =
6184 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6185 let reread = this.get_label();
6186
6187 this.emit_label(reread)?;
6188 this.assembler
6189 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
6190 this.emit_binop_sub64(dst, loc, Location::GPR(tmp1))?;
6191 this.assembler.emit_stlxr(
6192 Size::S32,
6193 Location::GPR(tmp2),
6194 Location::GPR(tmp1),
6195 Location::GPR(addr),
6196 )?;
6197 this.assembler
6198 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6199 this.assembler.emit_dmb()?;
6200
6201 if dst != ret {
6202 this.move_location(Size::S64, ret, dst)?;
6203 }
6204 for r in temps {
6205 this.release_gpr(r);
6206 }
6207 this.release_gpr(tmp1);
6208 this.release_gpr(tmp2);
6209 Ok(())
6210 },
6211 )
6212 }
6213
6214 fn i64_atomic_and(
6215 &mut self,
6216 loc: Location,
6217 target: Location,
6218 memarg: &MemArg,
6219 ret: Location,
6220 need_check: bool,
6221 imported_memories: bool,
6222 offset: i32,
6223 heap_access_oob: Label,
6224 unaligned_atomic: Label,
6225 ) -> Result<(), CompileError> {
6226 self.memory_op(
6227 target,
6228 memarg,
6229 true,
6230 8,
6231 need_check,
6232 imported_memories,
6233 offset,
6234 heap_access_oob,
6235 unaligned_atomic,
6236 |this, addr| {
6237 let mut temps = vec![];
6238 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6239 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6240 })?;
6241 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6242 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6243 })?;
6244 let dst =
6245 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6246 let reread = this.get_label();
6247
6248 this.emit_label(reread)?;
6249 this.assembler
6250 .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?;
6251 this.emit_binop_and64(dst, loc, Location::GPR(tmp1))?;
6252 this.assembler.emit_stlxr(
6253 Size::S64,
6254 Location::GPR(tmp2),
6255 Location::GPR(tmp1),
6256 Location::GPR(addr),
6257 )?;
6258 this.assembler
6259 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6260 this.assembler.emit_dmb()?;
6261
6262 if dst != ret {
6263 this.move_location(Size::S64, ret, dst)?;
6264 }
6265 for r in temps {
6266 this.release_gpr(r);
6267 }
6268 this.release_gpr(tmp1);
6269 this.release_gpr(tmp2);
6270 Ok(())
6271 },
6272 )
6273 }
6274
6275 fn i64_atomic_and_8u(
6276 &mut self,
6277 loc: Location,
6278 target: Location,
6279 memarg: &MemArg,
6280 ret: Location,
6281 need_check: bool,
6282 imported_memories: bool,
6283 offset: i32,
6284 heap_access_oob: Label,
6285 unaligned_atomic: Label,
6286 ) -> Result<(), CompileError> {
6287 self.memory_op(
6288 target,
6289 memarg,
6290 true,
6291 1,
6292 need_check,
6293 imported_memories,
6294 offset,
6295 heap_access_oob,
6296 unaligned_atomic,
6297 |this, addr| {
6298 let mut temps = vec![];
6299 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6300 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6301 })?;
6302 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6303 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6304 })?;
6305 let dst =
6306 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6307 let reread = this.get_label();
6308
6309 this.emit_label(reread)?;
6310 this.assembler
6311 .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?;
6312 this.emit_binop_and64(dst, loc, Location::GPR(tmp1))?;
6313 this.assembler.emit_stlxrb(
6314 Size::S64,
6315 Location::GPR(tmp2),
6316 Location::GPR(tmp1),
6317 Location::GPR(addr),
6318 )?;
6319 this.assembler
6320 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6321 this.assembler.emit_dmb()?;
6322
6323 if dst != ret {
6324 this.move_location(Size::S64, ret, dst)?;
6325 }
6326 for r in temps {
6327 this.release_gpr(r);
6328 }
6329 this.release_gpr(tmp1);
6330 this.release_gpr(tmp2);
6331 Ok(())
6332 },
6333 )
6334 }
6335
6336 fn i64_atomic_and_16u(
6337 &mut self,
6338 loc: Location,
6339 target: Location,
6340 memarg: &MemArg,
6341 ret: Location,
6342 need_check: bool,
6343 imported_memories: bool,
6344 offset: i32,
6345 heap_access_oob: Label,
6346 unaligned_atomic: Label,
6347 ) -> Result<(), CompileError> {
6348 self.memory_op(
6349 target,
6350 memarg,
6351 true,
6352 2,
6353 need_check,
6354 imported_memories,
6355 offset,
6356 heap_access_oob,
6357 unaligned_atomic,
6358 |this, addr| {
6359 let mut temps = vec![];
6360 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6361 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6362 })?;
6363 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6364 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6365 })?;
6366 let dst =
6367 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6368 let reread = this.get_label();
6369
6370 this.emit_label(reread)?;
6371 this.assembler
6372 .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?;
6373 this.emit_binop_and64(dst, loc, Location::GPR(tmp1))?;
6374 this.assembler.emit_stlxrh(
6375 Size::S64,
6376 Location::GPR(tmp2),
6377 Location::GPR(tmp1),
6378 Location::GPR(addr),
6379 )?;
6380 this.assembler
6381 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6382 this.assembler.emit_dmb()?;
6383
6384 if dst != ret {
6385 this.move_location(Size::S64, ret, dst)?;
6386 }
6387 for r in temps {
6388 this.release_gpr(r);
6389 }
6390 this.release_gpr(tmp1);
6391 this.release_gpr(tmp2);
6392 Ok(())
6393 },
6394 )
6395 }
6396
6397 fn i64_atomic_and_32u(
6398 &mut self,
6399 loc: Location,
6400 target: Location,
6401 memarg: &MemArg,
6402 ret: Location,
6403 need_check: bool,
6404 imported_memories: bool,
6405 offset: i32,
6406 heap_access_oob: Label,
6407 unaligned_atomic: Label,
6408 ) -> Result<(), CompileError> {
6409 self.memory_op(
6410 target,
6411 memarg,
6412 true,
6413 4,
6414 need_check,
6415 imported_memories,
6416 offset,
6417 heap_access_oob,
6418 unaligned_atomic,
6419 |this, addr| {
6420 let mut temps = vec![];
6421 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6422 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6423 })?;
6424 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6425 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6426 })?;
6427 let dst =
6428 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6429 let reread = this.get_label();
6430
6431 this.emit_label(reread)?;
6432 this.assembler
6433 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
6434 this.emit_binop_and64(dst, loc, Location::GPR(tmp1))?;
6435 this.assembler.emit_stlxr(
6436 Size::S32,
6437 Location::GPR(tmp2),
6438 Location::GPR(tmp1),
6439 Location::GPR(addr),
6440 )?;
6441 this.assembler
6442 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6443 this.assembler.emit_dmb()?;
6444
6445 if dst != ret {
6446 this.move_location(Size::S64, ret, dst)?;
6447 }
6448 for r in temps {
6449 this.release_gpr(r);
6450 }
6451 this.release_gpr(tmp1);
6452 this.release_gpr(tmp2);
6453 Ok(())
6454 },
6455 )
6456 }
6457
6458 fn i64_atomic_or(
6459 &mut self,
6460 loc: Location,
6461 target: Location,
6462 memarg: &MemArg,
6463 ret: Location,
6464 need_check: bool,
6465 imported_memories: bool,
6466 offset: i32,
6467 heap_access_oob: Label,
6468 unaligned_atomic: Label,
6469 ) -> Result<(), CompileError> {
6470 self.memory_op(
6471 target,
6472 memarg,
6473 true,
6474 8,
6475 need_check,
6476 imported_memories,
6477 offset,
6478 heap_access_oob,
6479 unaligned_atomic,
6480 |this, addr| {
6481 let mut temps = vec![];
6482 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6483 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6484 })?;
6485 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6486 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6487 })?;
6488 let dst =
6489 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6490 let reread = this.get_label();
6491
6492 this.emit_label(reread)?;
6493 this.assembler
6494 .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?;
6495 this.emit_binop_or64(dst, loc, Location::GPR(tmp1))?;
6496 this.assembler.emit_stlxr(
6497 Size::S64,
6498 Location::GPR(tmp2),
6499 Location::GPR(tmp1),
6500 Location::GPR(addr),
6501 )?;
6502 this.assembler
6503 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6504 this.assembler.emit_dmb()?;
6505
6506 if dst != ret {
6507 this.move_location(Size::S64, ret, dst)?;
6508 }
6509 for r in temps {
6510 this.release_gpr(r);
6511 }
6512 this.release_gpr(tmp1);
6513 this.release_gpr(tmp2);
6514 Ok(())
6515 },
6516 )
6517 }
6518
6519 fn i64_atomic_or_8u(
6520 &mut self,
6521 loc: Location,
6522 target: Location,
6523 memarg: &MemArg,
6524 ret: Location,
6525 need_check: bool,
6526 imported_memories: bool,
6527 offset: i32,
6528 heap_access_oob: Label,
6529 unaligned_atomic: Label,
6530 ) -> Result<(), CompileError> {
6531 self.memory_op(
6532 target,
6533 memarg,
6534 true,
6535 1,
6536 need_check,
6537 imported_memories,
6538 offset,
6539 heap_access_oob,
6540 unaligned_atomic,
6541 |this, addr| {
6542 let mut temps = vec![];
6543 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6544 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6545 })?;
6546 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6547 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6548 })?;
6549 let dst =
6550 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6551 let reread = this.get_label();
6552
6553 this.emit_label(reread)?;
6554 this.assembler
6555 .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?;
6556 this.emit_binop_or64(dst, loc, Location::GPR(tmp1))?;
6557 this.assembler.emit_stlxrb(
6558 Size::S64,
6559 Location::GPR(tmp2),
6560 Location::GPR(tmp1),
6561 Location::GPR(addr),
6562 )?;
6563 this.assembler
6564 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6565 this.assembler.emit_dmb()?;
6566
6567 if dst != ret {
6568 this.move_location(Size::S64, ret, dst)?;
6569 }
6570 for r in temps {
6571 this.release_gpr(r);
6572 }
6573 this.release_gpr(tmp1);
6574 this.release_gpr(tmp2);
6575 Ok(())
6576 },
6577 )
6578 }
6579
6580 fn i64_atomic_or_16u(
6581 &mut self,
6582 loc: Location,
6583 target: Location,
6584 memarg: &MemArg,
6585 ret: Location,
6586 need_check: bool,
6587 imported_memories: bool,
6588 offset: i32,
6589 heap_access_oob: Label,
6590 unaligned_atomic: Label,
6591 ) -> Result<(), CompileError> {
6592 self.memory_op(
6593 target,
6594 memarg,
6595 true,
6596 2,
6597 need_check,
6598 imported_memories,
6599 offset,
6600 heap_access_oob,
6601 unaligned_atomic,
6602 |this, addr| {
6603 let mut temps = vec![];
6604 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6605 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6606 })?;
6607 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6608 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6609 })?;
6610 let dst =
6611 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6612 let reread = this.get_label();
6613
6614 this.emit_label(reread)?;
6615 this.assembler
6616 .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?;
6617 this.emit_binop_or64(dst, loc, Location::GPR(tmp1))?;
6618 this.assembler.emit_stlxrh(
6619 Size::S64,
6620 Location::GPR(tmp2),
6621 Location::GPR(tmp1),
6622 Location::GPR(addr),
6623 )?;
6624 this.assembler
6625 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6626 this.assembler.emit_dmb()?;
6627
6628 if dst != ret {
6629 this.move_location(Size::S64, ret, dst)?;
6630 }
6631 for r in temps {
6632 this.release_gpr(r);
6633 }
6634 this.release_gpr(tmp1);
6635 this.release_gpr(tmp2);
6636 Ok(())
6637 },
6638 )
6639 }
6640
6641 fn i64_atomic_or_32u(
6642 &mut self,
6643 loc: Location,
6644 target: Location,
6645 memarg: &MemArg,
6646 ret: Location,
6647 need_check: bool,
6648 imported_memories: bool,
6649 offset: i32,
6650 heap_access_oob: Label,
6651 unaligned_atomic: Label,
6652 ) -> Result<(), CompileError> {
6653 self.memory_op(
6654 target,
6655 memarg,
6656 true,
6657 4,
6658 need_check,
6659 imported_memories,
6660 offset,
6661 heap_access_oob,
6662 unaligned_atomic,
6663 |this, addr| {
6664 let mut temps = vec![];
6665 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6666 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6667 })?;
6668 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6669 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6670 })?;
6671 let dst =
6672 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6673 let reread = this.get_label();
6674
6675 this.emit_label(reread)?;
6676 this.assembler
6677 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
6678 this.emit_binop_or64(dst, loc, Location::GPR(tmp1))?;
6679 this.assembler.emit_stlxr(
6680 Size::S32,
6681 Location::GPR(tmp2),
6682 Location::GPR(tmp1),
6683 Location::GPR(addr),
6684 )?;
6685 this.assembler
6686 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6687 this.assembler.emit_dmb()?;
6688
6689 if dst != ret {
6690 this.move_location(Size::S64, ret, dst)?;
6691 }
6692 for r in temps {
6693 this.release_gpr(r);
6694 }
6695 this.release_gpr(tmp1);
6696 this.release_gpr(tmp2);
6697 Ok(())
6698 },
6699 )
6700 }
6701
6702 fn i64_atomic_xor(
6703 &mut self,
6704 loc: Location,
6705 target: Location,
6706 memarg: &MemArg,
6707 ret: Location,
6708 need_check: bool,
6709 imported_memories: bool,
6710 offset: i32,
6711 heap_access_oob: Label,
6712 unaligned_atomic: Label,
6713 ) -> Result<(), CompileError> {
6714 self.memory_op(
6715 target,
6716 memarg,
6717 true,
6718 8,
6719 need_check,
6720 imported_memories,
6721 offset,
6722 heap_access_oob,
6723 unaligned_atomic,
6724 |this, addr| {
6725 let mut temps = vec![];
6726 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6727 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6728 })?;
6729 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6730 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6731 })?;
6732 let dst =
6733 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6734 let reread = this.get_label();
6735
6736 this.emit_label(reread)?;
6737 this.assembler
6738 .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?;
6739 this.emit_binop_xor64(dst, loc, Location::GPR(tmp1))?;
6740 this.assembler.emit_stlxr(
6741 Size::S64,
6742 Location::GPR(tmp2),
6743 Location::GPR(tmp1),
6744 Location::GPR(addr),
6745 )?;
6746 this.assembler
6747 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6748 this.assembler.emit_dmb()?;
6749
6750 if dst != ret {
6751 this.move_location(Size::S64, ret, dst)?;
6752 }
6753 for r in temps {
6754 this.release_gpr(r);
6755 }
6756 this.release_gpr(tmp1);
6757 this.release_gpr(tmp2);
6758 Ok(())
6759 },
6760 )
6761 }
6762
6763 fn i64_atomic_xor_8u(
6764 &mut self,
6765 loc: Location,
6766 target: Location,
6767 memarg: &MemArg,
6768 ret: Location,
6769 need_check: bool,
6770 imported_memories: bool,
6771 offset: i32,
6772 heap_access_oob: Label,
6773 unaligned_atomic: Label,
6774 ) -> Result<(), CompileError> {
6775 self.memory_op(
6776 target,
6777 memarg,
6778 true,
6779 1,
6780 need_check,
6781 imported_memories,
6782 offset,
6783 heap_access_oob,
6784 unaligned_atomic,
6785 |this, addr| {
6786 let mut temps = vec![];
6787 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6788 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6789 })?;
6790 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6791 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6792 })?;
6793 let dst =
6794 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6795 let reread = this.get_label();
6796
6797 this.emit_label(reread)?;
6798 this.assembler
6799 .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?;
6800 this.emit_binop_xor64(dst, loc, Location::GPR(tmp1))?;
6801 this.assembler.emit_stlxrb(
6802 Size::S64,
6803 Location::GPR(tmp2),
6804 Location::GPR(tmp1),
6805 Location::GPR(addr),
6806 )?;
6807 this.assembler
6808 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6809 this.assembler.emit_dmb()?;
6810
6811 if dst != ret {
6812 this.move_location(Size::S64, ret, dst)?;
6813 }
6814 for r in temps {
6815 this.release_gpr(r);
6816 }
6817 this.release_gpr(tmp1);
6818 this.release_gpr(tmp2);
6819 Ok(())
6820 },
6821 )
6822 }
6823
6824 fn i64_atomic_xor_16u(
6825 &mut self,
6826 loc: Location,
6827 target: Location,
6828 memarg: &MemArg,
6829 ret: Location,
6830 need_check: bool,
6831 imported_memories: bool,
6832 offset: i32,
6833 heap_access_oob: Label,
6834 unaligned_atomic: Label,
6835 ) -> Result<(), CompileError> {
6836 self.memory_op(
6837 target,
6838 memarg,
6839 true,
6840 2,
6841 need_check,
6842 imported_memories,
6843 offset,
6844 heap_access_oob,
6845 unaligned_atomic,
6846 |this, addr| {
6847 let mut temps = vec![];
6848 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6849 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6850 })?;
6851 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6852 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6853 })?;
6854 let dst =
6855 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6856 let reread = this.get_label();
6857
6858 this.emit_label(reread)?;
6859 this.assembler
6860 .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?;
6861 this.emit_binop_xor64(dst, loc, Location::GPR(tmp1))?;
6862 this.assembler.emit_stlxrh(
6863 Size::S64,
6864 Location::GPR(tmp2),
6865 Location::GPR(tmp1),
6866 Location::GPR(addr),
6867 )?;
6868 this.assembler
6869 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6870 this.assembler.emit_dmb()?;
6871
6872 if dst != ret {
6873 this.move_location(Size::S64, ret, dst)?;
6874 }
6875 for r in temps {
6876 this.release_gpr(r);
6877 }
6878 this.release_gpr(tmp1);
6879 this.release_gpr(tmp2);
6880 Ok(())
6881 },
6882 )
6883 }
6884
6885 fn i64_atomic_xor_32u(
6886 &mut self,
6887 loc: Location,
6888 target: Location,
6889 memarg: &MemArg,
6890 ret: Location,
6891 need_check: bool,
6892 imported_memories: bool,
6893 offset: i32,
6894 heap_access_oob: Label,
6895 unaligned_atomic: Label,
6896 ) -> Result<(), CompileError> {
6897 self.memory_op(
6898 target,
6899 memarg,
6900 true,
6901 4,
6902 need_check,
6903 imported_memories,
6904 offset,
6905 heap_access_oob,
6906 unaligned_atomic,
6907 |this, addr| {
6908 let mut temps = vec![];
6909 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6910 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6911 })?;
6912 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6913 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6914 })?;
6915 let dst =
6916 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6917 let reread = this.get_label();
6918
6919 this.emit_label(reread)?;
6920 this.assembler
6921 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
6922 this.emit_binop_xor64(dst, loc, Location::GPR(tmp1))?;
6923 this.assembler.emit_stlxr(
6924 Size::S32,
6925 Location::GPR(tmp2),
6926 Location::GPR(tmp1),
6927 Location::GPR(addr),
6928 )?;
6929 this.assembler
6930 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6931 this.assembler.emit_dmb()?;
6932
6933 if dst != ret {
6934 this.move_location(Size::S64, ret, dst)?;
6935 }
6936 for r in temps {
6937 this.release_gpr(r);
6938 }
6939 this.release_gpr(tmp1);
6940 this.release_gpr(tmp2);
6941 Ok(())
6942 },
6943 )
6944 }
6945
6946 fn i64_atomic_xchg(
6947 &mut self,
6948 loc: Location,
6949 target: Location,
6950 memarg: &MemArg,
6951 ret: Location,
6952 need_check: bool,
6953 imported_memories: bool,
6954 offset: i32,
6955 heap_access_oob: Label,
6956 unaligned_atomic: Label,
6957 ) -> Result<(), CompileError> {
6958 self.memory_op(
6959 target,
6960 memarg,
6961 true,
6962 8,
6963 need_check,
6964 imported_memories,
6965 offset,
6966 heap_access_oob,
6967 unaligned_atomic,
6968 |this, addr| {
6969 let mut temps = vec![];
6970 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
6971 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6972 })?;
6973 let dst =
6974 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6975 let org =
6976 this.location_to_reg(Size::S64, loc, &mut temps, ImmType::None, false, None)?;
6977 let reread = this.get_label();
6978
6979 this.emit_label(reread)?;
6980 this.assembler
6981 .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?;
6982 this.assembler.emit_stlxr(
6983 Size::S64,
6984 Location::GPR(tmp),
6985 org,
6986 Location::GPR(addr),
6987 )?;
6988 this.assembler
6989 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
6990 this.assembler.emit_dmb()?;
6991
6992 if dst != ret {
6993 this.move_location(Size::S64, ret, dst)?;
6994 }
6995 for r in temps {
6996 this.release_gpr(r);
6997 }
6998 this.release_gpr(tmp);
6999 Ok(())
7000 },
7001 )
7002 }
7003
7004 fn i64_atomic_xchg_8u(
7005 &mut self,
7006 loc: Location,
7007 target: Location,
7008 memarg: &MemArg,
7009 ret: Location,
7010 need_check: bool,
7011 imported_memories: bool,
7012 offset: i32,
7013 heap_access_oob: Label,
7014 unaligned_atomic: Label,
7015 ) -> Result<(), CompileError> {
7016 self.memory_op(
7017 target,
7018 memarg,
7019 true,
7020 1,
7021 need_check,
7022 imported_memories,
7023 offset,
7024 heap_access_oob,
7025 unaligned_atomic,
7026 |this, addr| {
7027 let mut temps = vec![];
7028 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
7029 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7030 })?;
7031 let dst =
7032 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7033 let org =
7034 this.location_to_reg(Size::S64, loc, &mut temps, ImmType::None, false, None)?;
7035 let reread = this.get_label();
7036
7037 this.emit_label(reread)?;
7038 this.assembler
7039 .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?;
7040 this.assembler.emit_stlxrb(
7041 Size::S64,
7042 Location::GPR(tmp),
7043 org,
7044 Location::GPR(addr),
7045 )?;
7046 this.assembler
7047 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7048 this.assembler.emit_dmb()?;
7049
7050 if dst != ret {
7051 this.move_location(Size::S64, ret, dst)?;
7052 }
7053 for r in temps {
7054 this.release_gpr(r);
7055 }
7056 this.release_gpr(tmp);
7057 Ok(())
7058 },
7059 )
7060 }
7061
7062 fn i64_atomic_xchg_16u(
7063 &mut self,
7064 loc: Location,
7065 target: Location,
7066 memarg: &MemArg,
7067 ret: Location,
7068 need_check: bool,
7069 imported_memories: bool,
7070 offset: i32,
7071 heap_access_oob: Label,
7072 unaligned_atomic: Label,
7073 ) -> Result<(), CompileError> {
7074 self.memory_op(
7075 target,
7076 memarg,
7077 true,
7078 2,
7079 need_check,
7080 imported_memories,
7081 offset,
7082 heap_access_oob,
7083 unaligned_atomic,
7084 |this, addr| {
7085 let mut temps = vec![];
7086 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
7087 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7088 })?;
7089 let dst =
7090 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7091 let org =
7092 this.location_to_reg(Size::S64, loc, &mut temps, ImmType::None, false, None)?;
7093 let reread = this.get_label();
7094
7095 this.emit_label(reread)?;
7096 this.assembler
7097 .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?;
7098 this.assembler.emit_stlxrh(
7099 Size::S64,
7100 Location::GPR(tmp),
7101 org,
7102 Location::GPR(addr),
7103 )?;
7104 this.assembler
7105 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7106 this.assembler.emit_dmb()?;
7107
7108 if dst != ret {
7109 this.move_location(Size::S64, ret, dst)?;
7110 }
7111 for r in temps {
7112 this.release_gpr(r);
7113 }
7114 this.release_gpr(tmp);
7115 Ok(())
7116 },
7117 )
7118 }
7119
7120 fn i64_atomic_xchg_32u(
7121 &mut self,
7122 loc: Location,
7123 target: Location,
7124 memarg: &MemArg,
7125 ret: Location,
7126 need_check: bool,
7127 imported_memories: bool,
7128 offset: i32,
7129 heap_access_oob: Label,
7130 unaligned_atomic: Label,
7131 ) -> Result<(), CompileError> {
7132 self.memory_op(
7133 target,
7134 memarg,
7135 true,
7136 4,
7137 need_check,
7138 imported_memories,
7139 offset,
7140 heap_access_oob,
7141 unaligned_atomic,
7142 |this, addr| {
7143 let mut temps = vec![];
7144 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
7145 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7146 })?;
7147 let dst =
7148 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7149 let org =
7150 this.location_to_reg(Size::S64, loc, &mut temps, ImmType::None, false, None)?;
7151 let reread = this.get_label();
7152
7153 this.emit_label(reread)?;
7154 this.assembler
7155 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
7156 this.assembler.emit_stlxr(
7157 Size::S32,
7158 Location::GPR(tmp),
7159 org,
7160 Location::GPR(addr),
7161 )?;
7162 this.assembler
7163 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7164 this.assembler.emit_dmb()?;
7165
7166 if dst != ret {
7167 this.move_location(Size::S64, ret, dst)?;
7168 }
7169 for r in temps {
7170 this.release_gpr(r);
7171 }
7172 this.release_gpr(tmp);
7173 Ok(())
7174 },
7175 )
7176 }
7177
7178 fn i64_atomic_cmpxchg(
7179 &mut self,
7180 new: Location,
7181 cmp: Location,
7182 target: Location,
7183 memarg: &MemArg,
7184 ret: Location,
7185 need_check: bool,
7186 imported_memories: bool,
7187 offset: i32,
7188 heap_access_oob: Label,
7189 unaligned_atomic: Label,
7190 ) -> Result<(), CompileError> {
7191 self.memory_op(
7192 target,
7193 memarg,
7194 true,
7195 8,
7196 need_check,
7197 imported_memories,
7198 offset,
7199 heap_access_oob,
7200 unaligned_atomic,
7201 |this, addr| {
7202 let mut temps = vec![];
7203 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
7204 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7205 })?;
7206 let dst =
7207 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7208 let org =
7209 this.location_to_reg(Size::S64, new, &mut temps, ImmType::None, false, None)?;
7210 let reread = this.get_label();
7211 let nosame = this.get_label();
7212
7213 this.emit_label(reread)?;
7214 this.assembler
7215 .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?;
7216 this.emit_relaxed_cmp(Size::S64, dst, cmp)?;
7217 this.assembler.emit_bcond_label(Condition::Ne, nosame)?;
7218 this.assembler.emit_stlxr(
7219 Size::S64,
7220 Location::GPR(tmp),
7221 org,
7222 Location::GPR(addr),
7223 )?;
7224 this.assembler
7225 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7226 this.assembler.emit_dmb()?;
7227
7228 this.emit_label(nosame)?;
7229 if dst != ret {
7230 this.move_location(Size::S64, ret, dst)?;
7231 }
7232 for r in temps {
7233 this.release_gpr(r);
7234 }
7235 this.release_gpr(tmp);
7236 Ok(())
7237 },
7238 )
7239 }
7240
7241 fn i64_atomic_cmpxchg_8u(
7242 &mut self,
7243 new: Location,
7244 cmp: Location,
7245 target: Location,
7246 memarg: &MemArg,
7247 ret: Location,
7248 need_check: bool,
7249 imported_memories: bool,
7250 offset: i32,
7251 heap_access_oob: Label,
7252 unaligned_atomic: Label,
7253 ) -> Result<(), CompileError> {
7254 self.memory_op(
7255 target,
7256 memarg,
7257 true,
7258 1,
7259 need_check,
7260 imported_memories,
7261 offset,
7262 heap_access_oob,
7263 unaligned_atomic,
7264 |this, addr| {
7265 let mut temps = vec![];
7266 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
7267 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7268 })?;
7269 let dst =
7270 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7271 let org =
7272 this.location_to_reg(Size::S64, new, &mut temps, ImmType::None, false, None)?;
7273 let reread = this.get_label();
7274 let nosame = this.get_label();
7275
7276 this.emit_label(reread)?;
7277 this.assembler
7278 .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?;
7279 this.emit_relaxed_cmp(Size::S64, dst, cmp)?;
7280 this.assembler.emit_bcond_label(Condition::Ne, nosame)?;
7281 this.assembler.emit_stlxrb(
7282 Size::S64,
7283 Location::GPR(tmp),
7284 org,
7285 Location::GPR(addr),
7286 )?;
7287 this.assembler
7288 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7289 this.assembler.emit_dmb()?;
7290
7291 this.emit_label(nosame)?;
7292 if dst != ret {
7293 this.move_location(Size::S64, ret, dst)?;
7294 }
7295 for r in temps {
7296 this.release_gpr(r);
7297 }
7298 this.release_gpr(tmp);
7299 Ok(())
7300 },
7301 )
7302 }
7303
7304 fn i64_atomic_cmpxchg_16u(
7305 &mut self,
7306 new: Location,
7307 cmp: Location,
7308 target: Location,
7309 memarg: &MemArg,
7310 ret: Location,
7311 need_check: bool,
7312 imported_memories: bool,
7313 offset: i32,
7314 heap_access_oob: Label,
7315 unaligned_atomic: Label,
7316 ) -> Result<(), CompileError> {
7317 self.memory_op(
7318 target,
7319 memarg,
7320 true,
7321 2,
7322 need_check,
7323 imported_memories,
7324 offset,
7325 heap_access_oob,
7326 unaligned_atomic,
7327 |this, addr| {
7328 let mut temps = vec![];
7329 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
7330 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7331 })?;
7332 let dst =
7333 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7334 let org =
7335 this.location_to_reg(Size::S64, new, &mut temps, ImmType::None, false, None)?;
7336 let reread = this.get_label();
7337 let nosame = this.get_label();
7338
7339 this.emit_label(reread)?;
7340 this.assembler
7341 .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?;
7342 this.emit_relaxed_cmp(Size::S64, dst, cmp)?;
7343 this.assembler.emit_bcond_label(Condition::Ne, nosame)?;
7344 this.assembler.emit_stlxrh(
7345 Size::S64,
7346 Location::GPR(tmp),
7347 org,
7348 Location::GPR(addr),
7349 )?;
7350 this.assembler
7351 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7352 this.assembler.emit_dmb()?;
7353
7354 this.emit_label(nosame)?;
7355 if dst != ret {
7356 this.move_location(Size::S64, ret, dst)?;
7357 }
7358 for r in temps {
7359 this.release_gpr(r);
7360 }
7361 this.release_gpr(tmp);
7362 Ok(())
7363 },
7364 )
7365 }
7366
7367 fn i64_atomic_cmpxchg_32u(
7368 &mut self,
7369 new: Location,
7370 cmp: Location,
7371 target: Location,
7372 memarg: &MemArg,
7373 ret: Location,
7374 need_check: bool,
7375 imported_memories: bool,
7376 offset: i32,
7377 heap_access_oob: Label,
7378 unaligned_atomic: Label,
7379 ) -> Result<(), CompileError> {
7380 self.memory_op(
7381 target,
7382 memarg,
7383 true,
7384 4,
7385 need_check,
7386 imported_memories,
7387 offset,
7388 heap_access_oob,
7389 unaligned_atomic,
7390 |this, addr| {
7391 let mut temps = vec![];
7392 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
7393 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7394 })?;
7395 let dst =
7396 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7397 let org =
7398 this.location_to_reg(Size::S64, new, &mut temps, ImmType::None, false, None)?;
7399 let reread = this.get_label();
7400 let nosame = this.get_label();
7401
7402 this.emit_label(reread)?;
7403 this.assembler
7404 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
7405 this.emit_relaxed_cmp(Size::S64, dst, cmp)?;
7406 this.assembler.emit_bcond_label(Condition::Ne, nosame)?;
7407 this.assembler.emit_stlxr(
7408 Size::S32,
7409 Location::GPR(tmp),
7410 org,
7411 Location::GPR(addr),
7412 )?;
7413 this.assembler
7414 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7415 this.assembler.emit_dmb()?;
7416
7417 this.emit_label(nosame)?;
7418 if dst != ret {
7419 this.move_location(Size::S64, ret, dst)?;
7420 }
7421 for r in temps {
7422 this.release_gpr(r);
7423 }
7424 this.release_gpr(tmp);
7425 Ok(())
7426 },
7427 )
7428 }
7429
7430 fn f32_load(
7431 &mut self,
7432 addr: Location,
7433 memarg: &MemArg,
7434 ret: Location,
7435 need_check: bool,
7436 imported_memories: bool,
7437 offset: i32,
7438 heap_access_oob: Label,
7439 unaligned_atomic: Label,
7440 ) -> Result<(), CompileError> {
7441 self.memory_op(
7442 addr,
7443 memarg,
7444 false,
7445 4,
7446 need_check,
7447 imported_memories,
7448 offset,
7449 heap_access_oob,
7450 unaligned_atomic,
7451 |this, addr| this.emit_relaxed_ldr32(Size::S32, ret, Location::Memory(addr, 0)),
7452 )
7453 }
7454
7455 fn f32_save(
7456 &mut self,
7457 target_value: Location,
7458 memarg: &MemArg,
7459 target_addr: Location,
7460 canonicalize: bool,
7461 need_check: bool,
7462 imported_memories: bool,
7463 offset: i32,
7464 heap_access_oob: Label,
7465 unaligned_atomic: Label,
7466 ) -> Result<(), CompileError> {
7467 self.memory_op(
7468 target_addr,
7469 memarg,
7470 false,
7471 4,
7472 need_check,
7473 imported_memories,
7474 offset,
7475 heap_access_oob,
7476 unaligned_atomic,
7477 |this, addr| {
7478 if !canonicalize {
7479 this.emit_relaxed_str32(target_value, Location::Memory(addr, 0))
7480 } else {
7481 this.canonicalize_nan(Size::S32, target_value, Location::Memory(addr, 0))
7482 }
7483 },
7484 )
7485 }
7486
7487 fn f64_load(
7488 &mut self,
7489 addr: Location,
7490 memarg: &MemArg,
7491 ret: Location,
7492 need_check: bool,
7493 imported_memories: bool,
7494 offset: i32,
7495 heap_access_oob: Label,
7496 unaligned_atomic: Label,
7497 ) -> Result<(), CompileError> {
7498 self.memory_op(
7499 addr,
7500 memarg,
7501 false,
7502 8,
7503 need_check,
7504 imported_memories,
7505 offset,
7506 heap_access_oob,
7507 unaligned_atomic,
7508 |this, addr| this.emit_relaxed_ldr64(Size::S64, ret, Location::Memory(addr, 0)),
7509 )
7510 }
7511
7512 fn f64_save(
7513 &mut self,
7514 target_value: Location,
7515 memarg: &MemArg,
7516 target_addr: Location,
7517 canonicalize: bool,
7518 need_check: bool,
7519 imported_memories: bool,
7520 offset: i32,
7521 heap_access_oob: Label,
7522 unaligned_atomic: Label,
7523 ) -> Result<(), CompileError> {
7524 self.memory_op(
7525 target_addr,
7526 memarg,
7527 false,
7528 8,
7529 need_check,
7530 imported_memories,
7531 offset,
7532 heap_access_oob,
7533 unaligned_atomic,
7534 |this, addr| {
7535 if !canonicalize {
7536 this.emit_relaxed_str64(target_value, Location::Memory(addr, 0))
7537 } else {
7538 this.canonicalize_nan(Size::S64, target_value, Location::Memory(addr, 0))
7539 }
7540 },
7541 )
7542 }
7543
7544 fn convert_f64_i64(
7545 &mut self,
7546 loc: Location,
7547 signed: bool,
7548 ret: Location,
7549 ) -> Result<(), CompileError> {
7550 let mut gprs = vec![];
7551 let mut neons = vec![];
7552 let src = self.location_to_reg(Size::S64, loc, &mut gprs, ImmType::NoneXzr, true, None)?;
7553 let dest = self.location_to_neon(Size::S64, ret, &mut neons, ImmType::None, false)?;
7554 if signed {
7555 self.assembler.emit_scvtf(Size::S64, src, Size::S64, dest)?;
7556 } else {
7557 self.assembler.emit_ucvtf(Size::S64, src, Size::S64, dest)?;
7558 }
7559 if ret != dest {
7560 self.move_location(Size::S64, dest, ret)?;
7561 }
7562 for r in gprs {
7563 self.release_gpr(r);
7564 }
7565 for r in neons {
7566 self.release_simd(r);
7567 }
7568 Ok(())
7569 }
7570
7571 fn convert_f64_i32(
7572 &mut self,
7573 loc: Location,
7574 signed: bool,
7575 ret: Location,
7576 ) -> Result<(), CompileError> {
7577 let mut gprs = vec![];
7578 let mut neons = vec![];
7579 let src = self.location_to_reg(Size::S32, loc, &mut gprs, ImmType::NoneXzr, true, None)?;
7580 let dest = self.location_to_neon(Size::S64, ret, &mut neons, ImmType::None, false)?;
7581 if signed {
7582 self.assembler.emit_scvtf(Size::S32, src, Size::S64, dest)?;
7583 } else {
7584 self.assembler.emit_ucvtf(Size::S32, src, Size::S64, dest)?;
7585 }
7586 if ret != dest {
7587 self.move_location(Size::S64, dest, ret)?;
7588 }
7589 for r in gprs {
7590 self.release_gpr(r);
7591 }
7592 for r in neons {
7593 self.release_simd(r);
7594 }
7595 Ok(())
7596 }
7597
7598 fn convert_f32_i64(
7599 &mut self,
7600 loc: Location,
7601 signed: bool,
7602 ret: Location,
7603 ) -> Result<(), CompileError> {
7604 let mut gprs = vec![];
7605 let mut neons = vec![];
7606 let src = self.location_to_reg(Size::S64, loc, &mut gprs, ImmType::NoneXzr, true, None)?;
7607 let dest = self.location_to_neon(Size::S32, ret, &mut neons, ImmType::None, false)?;
7608 if signed {
7609 self.assembler.emit_scvtf(Size::S64, src, Size::S32, dest)?;
7610 } else {
7611 self.assembler.emit_ucvtf(Size::S64, src, Size::S32, dest)?;
7612 }
7613 if ret != dest {
7614 self.move_location(Size::S32, dest, ret)?;
7615 }
7616 for r in gprs {
7617 self.release_gpr(r);
7618 }
7619 for r in neons {
7620 self.release_simd(r);
7621 }
7622 Ok(())
7623 }
7624
7625 fn convert_f32_i32(
7626 &mut self,
7627 loc: Location,
7628 signed: bool,
7629 ret: Location,
7630 ) -> Result<(), CompileError> {
7631 let mut gprs = vec![];
7632 let mut neons = vec![];
7633 let src = self.location_to_reg(Size::S32, loc, &mut gprs, ImmType::NoneXzr, true, None)?;
7634 let dest = self.location_to_neon(Size::S32, ret, &mut neons, ImmType::None, false)?;
7635 if signed {
7636 self.assembler.emit_scvtf(Size::S32, src, Size::S32, dest)?;
7637 } else {
7638 self.assembler.emit_ucvtf(Size::S32, src, Size::S32, dest)?;
7639 }
7640 if ret != dest {
7641 self.move_location(Size::S32, dest, ret)?;
7642 }
7643 for r in gprs {
7644 self.release_gpr(r);
7645 }
7646 for r in neons {
7647 self.release_simd(r);
7648 }
7649 Ok(())
7650 }
7651
7652 fn convert_i64_f64(
7653 &mut self,
7654 loc: Location,
7655 ret: Location,
7656 signed: bool,
7657 sat: bool,
7658 ) -> Result<(), CompileError> {
7659 let mut gprs = vec![];
7660 let mut neons = vec![];
7661 let src = self.location_to_neon(Size::S64, loc, &mut neons, ImmType::None, true)?;
7662 let dest = self.location_to_reg(Size::S64, ret, &mut gprs, ImmType::None, false, None)?;
7663 let old_fpcr = if !sat {
7664 self.reset_exception_fpsr()?;
7665 self.set_trap_enabled(&mut gprs)?
7666 } else {
7667 GPR::XzrSp
7668 };
7669 if signed {
7670 self.assembler
7671 .emit_fcvtzs(Size::S64, src, Size::S64, dest)?;
7672 } else {
7673 self.assembler
7674 .emit_fcvtzu(Size::S64, src, Size::S64, dest)?;
7675 }
7676 if !sat {
7677 self.trap_float_conversion_errors(old_fpcr, Size::S64, src, &mut gprs)?;
7678 }
7679 if ret != dest {
7680 self.move_location(Size::S64, dest, ret)?;
7681 }
7682 for r in gprs {
7683 self.release_gpr(r);
7684 }
7685 for r in neons {
7686 self.release_simd(r);
7687 }
7688 Ok(())
7689 }
7690
7691 fn convert_i32_f64(
7692 &mut self,
7693 loc: Location,
7694 ret: Location,
7695 signed: bool,
7696 sat: bool,
7697 ) -> Result<(), CompileError> {
7698 let mut gprs = vec![];
7699 let mut neons = vec![];
7700 let src = self.location_to_neon(Size::S64, loc, &mut neons, ImmType::None, true)?;
7701 let dest = self.location_to_reg(Size::S32, ret, &mut gprs, ImmType::None, false, None)?;
7702 let old_fpcr = if !sat {
7703 self.reset_exception_fpsr()?;
7704 self.set_trap_enabled(&mut gprs)?
7705 } else {
7706 GPR::XzrSp
7707 };
7708 if signed {
7709 self.assembler
7710 .emit_fcvtzs(Size::S64, src, Size::S32, dest)?;
7711 } else {
7712 self.assembler
7713 .emit_fcvtzu(Size::S64, src, Size::S32, dest)?;
7714 }
7715 if !sat {
7716 self.trap_float_conversion_errors(old_fpcr, Size::S64, src, &mut gprs)?;
7717 }
7718 if ret != dest {
7719 self.move_location(Size::S32, dest, ret)?;
7720 }
7721 for r in gprs {
7722 self.release_gpr(r);
7723 }
7724 for r in neons {
7725 self.release_simd(r);
7726 }
7727 Ok(())
7728 }
7729
7730 fn convert_i64_f32(
7731 &mut self,
7732 loc: Location,
7733 ret: Location,
7734 signed: bool,
7735 sat: bool,
7736 ) -> Result<(), CompileError> {
7737 let mut gprs = vec![];
7738 let mut neons = vec![];
7739 let src = self.location_to_neon(Size::S32, loc, &mut neons, ImmType::None, true)?;
7740 let dest = self.location_to_reg(Size::S64, ret, &mut gprs, ImmType::None, false, None)?;
7741 let old_fpcr = if !sat {
7742 self.reset_exception_fpsr()?;
7743 self.set_trap_enabled(&mut gprs)?
7744 } else {
7745 GPR::XzrSp
7746 };
7747 if signed {
7748 self.assembler
7749 .emit_fcvtzs(Size::S32, src, Size::S64, dest)?;
7750 } else {
7751 self.assembler
7752 .emit_fcvtzu(Size::S32, src, Size::S64, dest)?;
7753 }
7754 if !sat {
7755 self.trap_float_conversion_errors(old_fpcr, Size::S32, src, &mut gprs)?;
7756 }
7757 if ret != dest {
7758 self.move_location(Size::S64, dest, ret)?;
7759 }
7760 for r in gprs {
7761 self.release_gpr(r);
7762 }
7763 for r in neons {
7764 self.release_simd(r);
7765 }
7766 Ok(())
7767 }
7768
7769 fn convert_i32_f32(
7770 &mut self,
7771 loc: Location,
7772 ret: Location,
7773 signed: bool,
7774 sat: bool,
7775 ) -> Result<(), CompileError> {
7776 let mut gprs = vec![];
7777 let mut neons = vec![];
7778 let src = self.location_to_neon(Size::S32, loc, &mut neons, ImmType::None, true)?;
7779 let dest = self.location_to_reg(Size::S32, ret, &mut gprs, ImmType::None, false, None)?;
7780 let old_fpcr = if !sat {
7781 self.reset_exception_fpsr()?;
7782 self.set_trap_enabled(&mut gprs)?
7783 } else {
7784 GPR::XzrSp
7785 };
7786 if signed {
7787 self.assembler
7788 .emit_fcvtzs(Size::S32, src, Size::S32, dest)?;
7789 } else {
7790 self.assembler
7791 .emit_fcvtzu(Size::S32, src, Size::S32, dest)?;
7792 }
7793 if !sat {
7794 self.trap_float_conversion_errors(old_fpcr, Size::S32, src, &mut gprs)?;
7795 }
7796 if ret != dest {
7797 self.move_location(Size::S32, dest, ret)?;
7798 }
7799 for r in gprs {
7800 self.release_gpr(r);
7801 }
7802 for r in neons {
7803 self.release_simd(r);
7804 }
7805 Ok(())
7806 }
7807
7808 fn convert_f64_f32(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7809 self.emit_relaxed_fcvt(Size::S32, Size::S64, loc, ret)
7810 }
7811
7812 fn convert_f32_f64(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7813 self.emit_relaxed_fcvt(Size::S64, Size::S32, loc, ret)
7814 }
7815
7816 fn f64_neg(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7817 self.emit_relaxed_binop_neon(Assembler::emit_fneg, Size::S64, loc, ret, true)
7818 }
7819
7820 fn f64_abs(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7821 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
7822 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7823 })?;
7824
7825 self.move_location(Size::S64, loc, Location::GPR(tmp))?;
7826 self.assembler.emit_and(
7827 Size::S64,
7828 Location::GPR(tmp),
7829 Location::Imm64(0x7fffffffffffffffu64),
7830 Location::GPR(tmp),
7831 )?;
7832 self.move_location(Size::S64, Location::GPR(tmp), ret)?;
7833
7834 self.release_gpr(tmp);
7835 Ok(())
7836 }
7837
7838 fn emit_i64_copysign(&mut self, tmp1: GPR, tmp2: GPR) -> Result<(), CompileError> {
7839 self.assembler.emit_and(
7840 Size::S64,
7841 Location::GPR(tmp1),
7842 Location::Imm64(0x7fffffffffffffffu64),
7843 Location::GPR(tmp1),
7844 )?;
7845
7846 self.assembler.emit_and(
7847 Size::S64,
7848 Location::GPR(tmp2),
7849 Location::Imm64(0x8000000000000000u64),
7850 Location::GPR(tmp2),
7851 )?;
7852
7853 self.assembler.emit_or(
7854 Size::S64,
7855 Location::GPR(tmp1),
7856 Location::GPR(tmp2),
7857 Location::GPR(tmp1),
7858 )
7859 }
7860
7861 fn f64_sqrt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7862 self.emit_relaxed_binop_neon(Assembler::emit_fsqrt, Size::S64, loc, ret, true)
7863 }
7864
7865 fn f64_trunc(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7866 self.emit_relaxed_binop_neon(Assembler::emit_frintz, Size::S64, loc, ret, true)
7867 }
7868
7869 fn f64_ceil(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7870 self.emit_relaxed_binop_neon(Assembler::emit_frintp, Size::S64, loc, ret, true)
7871 }
7872
7873 fn f64_floor(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7874 self.emit_relaxed_binop_neon(Assembler::emit_frintm, Size::S64, loc, ret, true)
7875 }
7876
7877 fn f64_nearest(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7878 self.emit_relaxed_binop_neon(Assembler::emit_frintn, Size::S64, loc, ret, true)
7879 }
7880
7881 fn f64_cmp_ge(
7882 &mut self,
7883 loc_a: Location,
7884 loc_b: Location,
7885 ret: Location,
7886 ) -> Result<(), CompileError> {
7887 let mut temps = vec![];
7888 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7889 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S64, loc_b, loc_a, false)?;
7890 self.assembler.emit_cset(Size::S32, dest, Condition::Ls)?;
7891 if ret != dest {
7892 self.move_location(Size::S32, dest, ret)?;
7893 }
7894 for r in temps {
7895 self.release_gpr(r);
7896 }
7897 Ok(())
7898 }
7899
7900 fn f64_cmp_gt(
7901 &mut self,
7902 loc_a: Location,
7903 loc_b: Location,
7904 ret: Location,
7905 ) -> Result<(), CompileError> {
7906 let mut temps = vec![];
7907 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7908 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S64, loc_b, loc_a, false)?;
7909 self.assembler.emit_cset(Size::S32, dest, Condition::Cc)?;
7910 if ret != dest {
7911 self.move_location(Size::S32, dest, ret)?;
7912 }
7913 for r in temps {
7914 self.release_gpr(r);
7915 }
7916 Ok(())
7917 }
7918
7919 fn f64_cmp_le(
7920 &mut self,
7921 loc_a: Location,
7922 loc_b: Location,
7923 ret: Location,
7924 ) -> Result<(), CompileError> {
7925 let mut temps = vec![];
7926 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7927 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S64, loc_a, loc_b, false)?;
7928 self.assembler.emit_cset(Size::S32, dest, Condition::Ls)?;
7929 if ret != dest {
7930 self.move_location(Size::S32, dest, ret)?;
7931 }
7932 for r in temps {
7933 self.release_gpr(r);
7934 }
7935 Ok(())
7936 }
7937
7938 fn f64_cmp_lt(
7939 &mut self,
7940 loc_a: Location,
7941 loc_b: Location,
7942 ret: Location,
7943 ) -> Result<(), CompileError> {
7944 let mut temps = vec![];
7945 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7946 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S64, loc_a, loc_b, false)?;
7947 self.assembler.emit_cset(Size::S32, dest, Condition::Cc)?;
7948 if ret != dest {
7949 self.move_location(Size::S32, dest, ret)?;
7950 }
7951 for r in temps {
7952 self.release_gpr(r);
7953 }
7954 Ok(())
7955 }
7956
7957 fn f64_cmp_ne(
7958 &mut self,
7959 loc_a: Location,
7960 loc_b: Location,
7961 ret: Location,
7962 ) -> Result<(), CompileError> {
7963 let mut temps = vec![];
7964 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7965 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S64, loc_a, loc_b, false)?;
7966 self.assembler.emit_cset(Size::S32, dest, Condition::Ne)?;
7967 if ret != dest {
7968 self.move_location(Size::S32, dest, ret)?;
7969 }
7970 for r in temps {
7971 self.release_gpr(r);
7972 }
7973 Ok(())
7974 }
7975
7976 fn f64_cmp_eq(
7977 &mut self,
7978 loc_a: Location,
7979 loc_b: Location,
7980 ret: Location,
7981 ) -> Result<(), CompileError> {
7982 let mut temps = vec![];
7983 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7984 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S64, loc_a, loc_b, false)?;
7985 self.assembler.emit_cset(Size::S32, dest, Condition::Eq)?;
7986 if ret != dest {
7987 self.move_location(Size::S32, dest, ret)?;
7988 }
7989 for r in temps {
7990 self.release_gpr(r);
7991 }
7992 Ok(())
7993 }
7994
7995 fn f64_min(
7996 &mut self,
7997 loc_a: Location,
7998 loc_b: Location,
7999 ret: Location,
8000 ) -> Result<(), CompileError> {
8001 let mut temps = vec![];
8002 let old_fpcr = self.set_default_nan(&mut temps)?;
8003 self.emit_relaxed_binop3_neon(
8004 Assembler::emit_fmin,
8005 Size::S64,
8006 loc_a,
8007 loc_b,
8008 ret,
8009 ImmType::None,
8010 )?;
8011 self.restore_fpcr(old_fpcr)?;
8012 for r in temps {
8013 self.release_gpr(r);
8014 }
8015 Ok(())
8016 }
8017
8018 fn f64_max(
8019 &mut self,
8020 loc_a: Location,
8021 loc_b: Location,
8022 ret: Location,
8023 ) -> Result<(), CompileError> {
8024 let mut temps = vec![];
8025 let old_fpcr = self.set_default_nan(&mut temps)?;
8026 self.emit_relaxed_binop3_neon(
8027 Assembler::emit_fmax,
8028 Size::S64,
8029 loc_a,
8030 loc_b,
8031 ret,
8032 ImmType::None,
8033 )?;
8034 self.restore_fpcr(old_fpcr)?;
8035 for r in temps {
8036 self.release_gpr(r);
8037 }
8038 Ok(())
8039 }
8040
8041 fn f64_add(
8042 &mut self,
8043 loc_a: Location,
8044 loc_b: Location,
8045 ret: Location,
8046 ) -> Result<(), CompileError> {
8047 self.emit_relaxed_binop3_neon(
8048 Assembler::emit_fadd,
8049 Size::S64,
8050 loc_a,
8051 loc_b,
8052 ret,
8053 ImmType::None,
8054 )
8055 }
8056
8057 fn f64_sub(
8058 &mut self,
8059 loc_a: Location,
8060 loc_b: Location,
8061 ret: Location,
8062 ) -> Result<(), CompileError> {
8063 self.emit_relaxed_binop3_neon(
8064 Assembler::emit_fsub,
8065 Size::S64,
8066 loc_a,
8067 loc_b,
8068 ret,
8069 ImmType::None,
8070 )
8071 }
8072
8073 fn f64_mul(
8074 &mut self,
8075 loc_a: Location,
8076 loc_b: Location,
8077 ret: Location,
8078 ) -> Result<(), CompileError> {
8079 self.emit_relaxed_binop3_neon(
8080 Assembler::emit_fmul,
8081 Size::S64,
8082 loc_a,
8083 loc_b,
8084 ret,
8085 ImmType::None,
8086 )
8087 }
8088
8089 fn f64_div(
8090 &mut self,
8091 loc_a: Location,
8092 loc_b: Location,
8093 ret: Location,
8094 ) -> Result<(), CompileError> {
8095 self.emit_relaxed_binop3_neon(
8096 Assembler::emit_fdiv,
8097 Size::S64,
8098 loc_a,
8099 loc_b,
8100 ret,
8101 ImmType::None,
8102 )
8103 }
8104
8105 fn f32_neg(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
8106 self.emit_relaxed_binop_neon(Assembler::emit_fneg, Size::S32, loc, ret, true)
8107 }
8108
8109 fn f32_abs(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
8110 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
8111 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
8112 })?;
8113 self.move_location(Size::S32, loc, Location::GPR(tmp))?;
8114 self.assembler.emit_and(
8115 Size::S32,
8116 Location::GPR(tmp),
8117 Location::Imm32(0x7fffffffu32),
8118 Location::GPR(tmp),
8119 )?;
8120 self.move_location(Size::S32, Location::GPR(tmp), ret)?;
8121 self.release_gpr(tmp);
8122 Ok(())
8123 }
8124
8125 fn emit_i32_copysign(&mut self, tmp1: GPR, tmp2: GPR) -> Result<(), CompileError> {
8126 self.assembler.emit_and(
8127 Size::S32,
8128 Location::GPR(tmp1),
8129 Location::Imm32(0x7fffffffu32),
8130 Location::GPR(tmp1),
8131 )?;
8132 self.assembler.emit_and(
8133 Size::S32,
8134 Location::GPR(tmp2),
8135 Location::Imm32(0x80000000u32),
8136 Location::GPR(tmp2),
8137 )?;
8138 self.assembler.emit_or(
8139 Size::S32,
8140 Location::GPR(tmp1),
8141 Location::GPR(tmp2),
8142 Location::GPR(tmp1),
8143 )
8144 }
8145
8146 fn f32_sqrt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
8147 self.emit_relaxed_binop_neon(Assembler::emit_fsqrt, Size::S32, loc, ret, true)
8148 }
8149
8150 fn f32_trunc(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
8151 self.emit_relaxed_binop_neon(Assembler::emit_frintz, Size::S32, loc, ret, true)
8152 }
8153
8154 fn f32_ceil(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
8155 self.emit_relaxed_binop_neon(Assembler::emit_frintp, Size::S32, loc, ret, true)
8156 }
8157
8158 fn f32_floor(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
8159 self.emit_relaxed_binop_neon(Assembler::emit_frintm, Size::S32, loc, ret, true)
8160 }
8161
8162 fn f32_nearest(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
8163 self.emit_relaxed_binop_neon(Assembler::emit_frintn, Size::S32, loc, ret, true)
8164 }
8165
8166 fn f32_cmp_ge(
8167 &mut self,
8168 loc_a: Location,
8169 loc_b: Location,
8170 ret: Location,
8171 ) -> Result<(), CompileError> {
8172 let mut temps = vec![];
8173 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
8174 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S32, loc_b, loc_a, false)?;
8175 self.assembler.emit_cset(Size::S32, dest, Condition::Ls)?;
8176 if ret != dest {
8177 self.move_location(Size::S32, dest, ret)?;
8178 }
8179 for r in temps {
8180 self.release_gpr(r);
8181 }
8182 Ok(())
8183 }
8184
8185 fn f32_cmp_gt(
8186 &mut self,
8187 loc_a: Location,
8188 loc_b: Location,
8189 ret: Location,
8190 ) -> Result<(), CompileError> {
8191 let mut temps = vec![];
8192 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
8193 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S32, loc_b, loc_a, false)?;
8194 self.assembler.emit_cset(Size::S32, dest, Condition::Cc)?;
8195 if ret != dest {
8196 self.move_location(Size::S32, dest, ret)?;
8197 }
8198 for r in temps {
8199 self.release_gpr(r);
8200 }
8201 Ok(())
8202 }
8203
8204 fn f32_cmp_le(
8205 &mut self,
8206 loc_a: Location,
8207 loc_b: Location,
8208 ret: Location,
8209 ) -> Result<(), CompileError> {
8210 let mut temps = vec![];
8211 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
8212 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S32, loc_a, loc_b, false)?;
8213 self.assembler.emit_cset(Size::S32, dest, Condition::Ls)?;
8214 if ret != dest {
8215 self.move_location(Size::S32, dest, ret)?;
8216 }
8217 for r in temps {
8218 self.release_gpr(r);
8219 }
8220 Ok(())
8221 }
8222
8223 fn f32_cmp_lt(
8224 &mut self,
8225 loc_a: Location,
8226 loc_b: Location,
8227 ret: Location,
8228 ) -> Result<(), CompileError> {
8229 let mut temps = vec![];
8230 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
8231 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S32, loc_a, loc_b, false)?;
8232 self.assembler.emit_cset(Size::S32, dest, Condition::Cc)?;
8233 if ret != dest {
8234 self.move_location(Size::S32, dest, ret)?;
8235 }
8236 for r in temps {
8237 self.release_gpr(r);
8238 }
8239 Ok(())
8240 }
8241
8242 fn f32_cmp_ne(
8243 &mut self,
8244 loc_a: Location,
8245 loc_b: Location,
8246 ret: Location,
8247 ) -> Result<(), CompileError> {
8248 let mut temps = vec![];
8249 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
8250 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S32, loc_a, loc_b, false)?;
8251 self.assembler.emit_cset(Size::S32, dest, Condition::Ne)?;
8252 if ret != dest {
8253 self.move_location(Size::S32, dest, ret)?;
8254 }
8255 for r in temps {
8256 self.release_gpr(r);
8257 }
8258 Ok(())
8259 }
8260
8261 fn f32_cmp_eq(
8262 &mut self,
8263 loc_a: Location,
8264 loc_b: Location,
8265 ret: Location,
8266 ) -> Result<(), CompileError> {
8267 let mut temps = vec![];
8268 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
8269 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S32, loc_a, loc_b, false)?;
8270 self.assembler.emit_cset(Size::S32, dest, Condition::Eq)?;
8271 if ret != dest {
8272 self.move_location(Size::S32, dest, ret)?;
8273 }
8274 for r in temps {
8275 self.release_gpr(r);
8276 }
8277 Ok(())
8278 }
8279
8280 fn f32_min(
8281 &mut self,
8282 loc_a: Location,
8283 loc_b: Location,
8284 ret: Location,
8285 ) -> Result<(), CompileError> {
8286 let mut temps = vec![];
8287 let old_fpcr = self.set_default_nan(&mut temps)?;
8288 self.emit_relaxed_binop3_neon(
8289 Assembler::emit_fmin,
8290 Size::S32,
8291 loc_a,
8292 loc_b,
8293 ret,
8294 ImmType::None,
8295 )?;
8296 self.restore_fpcr(old_fpcr)?;
8297 for r in temps {
8298 self.release_gpr(r);
8299 }
8300 Ok(())
8301 }
8302
8303 fn f32_max(
8304 &mut self,
8305 loc_a: Location,
8306 loc_b: Location,
8307 ret: Location,
8308 ) -> Result<(), CompileError> {
8309 let mut temps = vec![];
8310 let old_fpcr = self.set_default_nan(&mut temps)?;
8311 self.emit_relaxed_binop3_neon(
8312 Assembler::emit_fmax,
8313 Size::S32,
8314 loc_a,
8315 loc_b,
8316 ret,
8317 ImmType::None,
8318 )?;
8319 self.restore_fpcr(old_fpcr)?;
8320 for r in temps {
8321 self.release_gpr(r);
8322 }
8323 Ok(())
8324 }
8325
8326 fn f32_add(
8327 &mut self,
8328 loc_a: Location,
8329 loc_b: Location,
8330 ret: Location,
8331 ) -> Result<(), CompileError> {
8332 self.emit_relaxed_binop3_neon(
8333 Assembler::emit_fadd,
8334 Size::S32,
8335 loc_a,
8336 loc_b,
8337 ret,
8338 ImmType::None,
8339 )
8340 }
8341
8342 fn f32_sub(
8343 &mut self,
8344 loc_a: Location,
8345 loc_b: Location,
8346 ret: Location,
8347 ) -> Result<(), CompileError> {
8348 self.emit_relaxed_binop3_neon(
8349 Assembler::emit_fsub,
8350 Size::S32,
8351 loc_a,
8352 loc_b,
8353 ret,
8354 ImmType::None,
8355 )
8356 }
8357
8358 fn f32_mul(
8359 &mut self,
8360 loc_a: Location,
8361 loc_b: Location,
8362 ret: Location,
8363 ) -> Result<(), CompileError> {
8364 self.emit_relaxed_binop3_neon(
8365 Assembler::emit_fmul,
8366 Size::S32,
8367 loc_a,
8368 loc_b,
8369 ret,
8370 ImmType::None,
8371 )
8372 }
8373
8374 fn f32_div(
8375 &mut self,
8376 loc_a: Location,
8377 loc_b: Location,
8378 ret: Location,
8379 ) -> Result<(), CompileError> {
8380 self.emit_relaxed_binop3_neon(
8381 Assembler::emit_fdiv,
8382 Size::S32,
8383 loc_a,
8384 loc_b,
8385 ret,
8386 ImmType::None,
8387 )
8388 }
8389
8390 fn gen_std_trampoline(
8391 &self,
8392 sig: &FunctionType,
8393 calling_convention: CallingConvention,
8394 progress_callback: Option<&CompilationProgressCallback>,
8395 ) -> Result<FunctionBody, CompileError> {
8396 gen_std_trampoline_arm64(sig, calling_convention, progress_callback)
8397 }
8398 fn gen_std_dynamic_import_trampoline(
8401 &self,
8402 vmoffsets: &VMOffsets,
8403 sig: &FunctionType,
8404 calling_convention: CallingConvention,
8405 progress_callback: Option<&CompilationProgressCallback>,
8406 ) -> Result<FunctionBody, CompileError> {
8407 gen_std_dynamic_import_trampoline_arm64(
8408 vmoffsets,
8409 sig,
8410 calling_convention,
8411 progress_callback,
8412 )
8413 }
8414 fn gen_import_call_trampoline(
8417 &self,
8418 vmoffsets: &VMOffsets,
8419 index: FunctionIndex,
8420 sig: &FunctionType,
8421 calling_convention: CallingConvention,
8422 progress_callback: Option<&CompilationProgressCallback>,
8423 ) -> Result<CustomSection, CompileError> {
8424 gen_import_call_trampoline_arm64(
8425 vmoffsets,
8426 index,
8427 sig,
8428 calling_convention,
8429 progress_callback,
8430 )
8431 }
8432
8433 #[cfg(feature = "unwind")]
8434 fn gen_dwarf_unwind_info(&mut self, code_len: usize) -> Option<UnwindInstructions> {
8435 let mut instructions = vec![];
8436 for &(instruction_offset, ref inst) in &self.unwind_ops {
8437 let instruction_offset = instruction_offset as u32;
8438 match *inst {
8439 UnwindOps::PushFP { up_to_sp } => {
8440 instructions.push((
8441 instruction_offset,
8442 CallFrameInstruction::CfaOffset(up_to_sp as i32),
8443 ));
8444 instructions.push((
8445 instruction_offset,
8446 CallFrameInstruction::Offset(AArch64::X29, -(up_to_sp as i32)),
8447 ));
8448 }
8449 UnwindOps::Push2Regs {
8450 reg1,
8451 reg2,
8452 up_to_sp,
8453 } => {
8454 instructions.push((
8455 instruction_offset,
8456 CallFrameInstruction::CfaOffset(up_to_sp as i32),
8457 ));
8458 instructions.push((
8459 instruction_offset,
8460 CallFrameInstruction::Offset(reg2.dwarf_index(), -(up_to_sp as i32) + 8),
8461 ));
8462 instructions.push((
8463 instruction_offset,
8464 CallFrameInstruction::Offset(reg1.dwarf_index(), -(up_to_sp as i32)),
8465 ));
8466 }
8467 UnwindOps::DefineNewFrame => {
8468 instructions.push((
8469 instruction_offset,
8470 CallFrameInstruction::CfaRegister(AArch64::X29),
8471 ));
8472 }
8473 UnwindOps::SaveRegister { reg, bp_neg_offset } => instructions.push((
8474 instruction_offset,
8475 CallFrameInstruction::Offset(reg.dwarf_index(), -bp_neg_offset),
8476 )),
8477 UnwindOps::SubtractFP { .. } => unimplemented!(),
8478 }
8479 }
8480 Some(UnwindInstructions {
8481 instructions,
8482 len: code_len as u32,
8483 })
8484 }
8485 #[cfg(not(feature = "unwind"))]
8486
8487 fn gen_dwarf_unwind_info(&mut self, _code_len: usize) -> Option<UnwindInstructions> {
8488 None
8489 }
8490
8491 fn gen_windows_unwind_info(&mut self, _code_len: usize) -> Option<Vec<u8>> {
8492 None
8493 }
8494}
8495
8496#[cfg(test)]
8497mod test {
8498 use super::*;
8499
8500 fn test_move_location(machine: &mut MachineARM64, size: Size) -> Result<(), CompileError> {
8501 machine.move_location(size, Location::GPR(GPR::X1), Location::GPR(GPR::X2))?;
8502 machine.move_location(size, Location::GPR(GPR::X1), Location::Memory(GPR::X2, 10))?;
8503 machine.move_location(size, Location::GPR(GPR::X1), Location::Memory(GPR::X2, -10))?;
8504 machine.move_location(
8505 size,
8506 Location::GPR(GPR::X1),
8507 Location::Memory(GPR::X2, 1024),
8508 )?;
8509 machine.move_location(
8510 size,
8511 Location::GPR(GPR::X1),
8512 Location::Memory(GPR::X2, -1024),
8513 )?;
8514 machine.move_location(size, Location::Memory(GPR::X2, 10), Location::GPR(GPR::X1))?;
8515 machine.move_location(size, Location::Memory(GPR::X2, -10), Location::GPR(GPR::X1))?;
8516 machine.move_location(
8517 size,
8518 Location::Memory(GPR::X2, 1024),
8519 Location::GPR(GPR::X1),
8520 )?;
8521 machine.move_location(
8522 size,
8523 Location::Memory(GPR::X2, -1024),
8524 Location::GPR(GPR::X1),
8525 )?;
8526 machine.move_location(size, Location::GPR(GPR::X1), Location::SIMD(NEON::V0))?;
8527 machine.move_location(size, Location::SIMD(NEON::V0), Location::GPR(GPR::X1))?;
8528 machine.move_location(
8529 size,
8530 Location::SIMD(NEON::V0),
8531 Location::Memory(GPR::X2, 10),
8532 )?;
8533 machine.move_location(
8534 size,
8535 Location::SIMD(NEON::V0),
8536 Location::Memory(GPR::X2, -10),
8537 )?;
8538 machine.move_location(
8539 size,
8540 Location::SIMD(NEON::V0),
8541 Location::Memory(GPR::X2, 1024),
8542 )?;
8543 machine.move_location(
8544 size,
8545 Location::SIMD(NEON::V0),
8546 Location::Memory(GPR::X2, -1024),
8547 )?;
8548 machine.move_location(
8549 size,
8550 Location::Memory(GPR::X2, 10),
8551 Location::SIMD(NEON::V0),
8552 )?;
8553 machine.move_location(
8554 size,
8555 Location::Memory(GPR::X2, -10),
8556 Location::SIMD(NEON::V0),
8557 )?;
8558 machine.move_location(
8559 size,
8560 Location::Memory(GPR::X2, 1024),
8561 Location::SIMD(NEON::V0),
8562 )?;
8563 machine.move_location(
8564 size,
8565 Location::Memory(GPR::X2, -1024),
8566 Location::SIMD(NEON::V0),
8567 )?;
8568
8569 Ok(())
8570 }
8571
8572 fn test_move_location_extended(
8573 machine: &mut MachineARM64,
8574 signed: bool,
8575 sized: Size,
8576 ) -> Result<(), CompileError> {
8577 machine.move_location_extend(
8578 sized,
8579 signed,
8580 Location::GPR(GPR::X0),
8581 Size::S64,
8582 Location::GPR(GPR::X1),
8583 )?;
8584 machine.move_location_extend(
8585 sized,
8586 signed,
8587 Location::GPR(GPR::X0),
8588 Size::S64,
8589 Location::Memory(GPR::X1, 10),
8590 )?;
8591 machine.move_location_extend(
8592 sized,
8593 signed,
8594 Location::GPR(GPR::X0),
8595 Size::S64,
8596 Location::Memory(GPR::X1, 16),
8597 )?;
8598 machine.move_location_extend(
8599 sized,
8600 signed,
8601 Location::GPR(GPR::X0),
8602 Size::S64,
8603 Location::Memory(GPR::X1, -16),
8604 )?;
8605 machine.move_location_extend(
8606 sized,
8607 signed,
8608 Location::GPR(GPR::X0),
8609 Size::S64,
8610 Location::Memory(GPR::X1, 1024),
8611 )?;
8612 machine.move_location_extend(
8613 sized,
8614 signed,
8615 Location::GPR(GPR::X0),
8616 Size::S64,
8617 Location::Memory(GPR::X1, -1024),
8618 )?;
8619 machine.move_location_extend(
8620 sized,
8621 signed,
8622 Location::Memory(GPR::X0, 10),
8623 Size::S64,
8624 Location::GPR(GPR::X1),
8625 )?;
8626
8627 Ok(())
8628 }
8629
8630 fn test_binop_op(
8631 machine: &mut MachineARM64,
8632 op: fn(&mut MachineARM64, Location, Location, Location) -> Result<(), CompileError>,
8633 ) -> Result<(), CompileError> {
8634 op(
8635 machine,
8636 Location::GPR(GPR::X2),
8637 Location::GPR(GPR::X2),
8638 Location::GPR(GPR::X0),
8639 )?;
8640 op(
8641 machine,
8642 Location::GPR(GPR::X2),
8643 Location::Imm32(10),
8644 Location::GPR(GPR::X0),
8645 )?;
8646 op(
8647 machine,
8648 Location::GPR(GPR::X0),
8649 Location::GPR(GPR::X0),
8650 Location::GPR(GPR::X0),
8651 )?;
8652 op(
8653 machine,
8654 Location::Imm32(10),
8655 Location::GPR(GPR::X2),
8656 Location::GPR(GPR::X0),
8657 )?;
8658 op(
8659 machine,
8660 Location::GPR(GPR::X0),
8661 Location::GPR(GPR::X2),
8662 Location::Memory(GPR::X0, 10),
8663 )?;
8664 op(
8665 machine,
8666 Location::GPR(GPR::X0),
8667 Location::Memory(GPR::X2, 16),
8668 Location::Memory(GPR::X0, 10),
8669 )?;
8670 op(
8671 machine,
8672 Location::Memory(GPR::X0, 0),
8673 Location::Memory(GPR::X2, 16),
8674 Location::Memory(GPR::X0, 10),
8675 )?;
8676
8677 Ok(())
8678 }
8679
8680 fn test_float_binop_op(
8681 machine: &mut MachineARM64,
8682 op: fn(&mut MachineARM64, Location, Location, Location) -> Result<(), CompileError>,
8683 ) -> Result<(), CompileError> {
8684 op(
8685 machine,
8686 Location::SIMD(NEON::V3),
8687 Location::SIMD(NEON::V2),
8688 Location::SIMD(NEON::V0),
8689 )?;
8690 op(
8691 machine,
8692 Location::SIMD(NEON::V0),
8693 Location::SIMD(NEON::V2),
8694 Location::SIMD(NEON::V0),
8695 )?;
8696 op(
8697 machine,
8698 Location::SIMD(NEON::V0),
8699 Location::SIMD(NEON::V0),
8700 Location::SIMD(NEON::V0),
8701 )?;
8702 op(
8703 machine,
8704 Location::Memory(GPR::X0, 0),
8705 Location::SIMD(NEON::V2),
8706 Location::SIMD(NEON::V0),
8707 )?;
8708 op(
8709 machine,
8710 Location::Memory(GPR::X0, 0),
8711 Location::Memory(GPR::X1, 10),
8712 Location::SIMD(NEON::V0),
8713 )?;
8714 op(
8715 machine,
8716 Location::Memory(GPR::X0, 0),
8717 Location::Memory(GPR::X1, 16),
8718 Location::Memory(GPR::X2, 32),
8719 )?;
8720 op(
8721 machine,
8722 Location::SIMD(NEON::V0),
8723 Location::Memory(GPR::X1, 16),
8724 Location::Memory(GPR::X2, 32),
8725 )?;
8726 op(
8727 machine,
8728 Location::SIMD(NEON::V0),
8729 Location::SIMD(NEON::V1),
8730 Location::Memory(GPR::X2, 32),
8731 )?;
8732
8733 Ok(())
8734 }
8735
8736 fn test_float_cmp_op(
8737 machine: &mut MachineARM64,
8738 op: fn(&mut MachineARM64, Location, Location, Location) -> Result<(), CompileError>,
8739 ) -> Result<(), CompileError> {
8740 op(
8741 machine,
8742 Location::SIMD(NEON::V3),
8743 Location::SIMD(NEON::V2),
8744 Location::GPR(GPR::X0),
8745 )?;
8746 op(
8747 machine,
8748 Location::SIMD(NEON::V0),
8749 Location::SIMD(NEON::V0),
8750 Location::GPR(GPR::X0),
8751 )?;
8752 op(
8753 machine,
8754 Location::Memory(GPR::X1, 0),
8755 Location::SIMD(NEON::V2),
8756 Location::GPR(GPR::X0),
8757 )?;
8758 op(
8759 machine,
8760 Location::Memory(GPR::X1, 0),
8761 Location::Memory(GPR::X2, 10),
8762 Location::GPR(GPR::X0),
8763 )?;
8764 op(
8765 machine,
8766 Location::Memory(GPR::X1, 0),
8767 Location::Memory(GPR::X2, 16),
8768 Location::Memory(GPR::X0, 32),
8769 )?;
8770 op(
8771 machine,
8772 Location::SIMD(NEON::V0),
8773 Location::Memory(GPR::X2, 16),
8774 Location::Memory(GPR::X0, 32),
8775 )?;
8776 op(
8777 machine,
8778 Location::SIMD(NEON::V0),
8779 Location::SIMD(NEON::V1),
8780 Location::Memory(GPR::X0, 32),
8781 )?;
8782
8783 Ok(())
8784 }
8785
8786 #[test]
8787 fn tests_arm64() -> Result<(), CompileError> {
8788 let mut machine = MachineARM64::new(None);
8789
8790 test_move_location(&mut machine, Size::S32)?;
8791 test_move_location(&mut machine, Size::S64)?;
8792 test_move_location_extended(&mut machine, false, Size::S8)?;
8793 test_move_location_extended(&mut machine, false, Size::S16)?;
8794 test_move_location_extended(&mut machine, false, Size::S32)?;
8795 test_move_location_extended(&mut machine, true, Size::S8)?;
8796 test_move_location_extended(&mut machine, true, Size::S16)?;
8797 test_move_location_extended(&mut machine, true, Size::S32)?;
8798 test_binop_op(&mut machine, MachineARM64::emit_binop_add32)?;
8799 test_binop_op(&mut machine, MachineARM64::emit_binop_add64)?;
8800 test_binop_op(&mut machine, MachineARM64::emit_binop_sub32)?;
8801 test_binop_op(&mut machine, MachineARM64::emit_binop_sub64)?;
8802 test_binop_op(&mut machine, MachineARM64::emit_binop_and32)?;
8803 test_binop_op(&mut machine, MachineARM64::emit_binop_and64)?;
8804 test_binop_op(&mut machine, MachineARM64::emit_binop_xor32)?;
8805 test_binop_op(&mut machine, MachineARM64::emit_binop_xor64)?;
8806 test_binop_op(&mut machine, MachineARM64::emit_binop_or32)?;
8807 test_binop_op(&mut machine, MachineARM64::emit_binop_or64)?;
8808 test_binop_op(&mut machine, MachineARM64::emit_binop_mul32)?;
8809 test_binop_op(&mut machine, MachineARM64::emit_binop_mul64)?;
8810 test_float_binop_op(&mut machine, MachineARM64::f32_add)?;
8811 test_float_binop_op(&mut machine, MachineARM64::f32_sub)?;
8812 test_float_binop_op(&mut machine, MachineARM64::f32_mul)?;
8813 test_float_binop_op(&mut machine, MachineARM64::f32_div)?;
8814 test_float_cmp_op(&mut machine, MachineARM64::f32_cmp_eq)?;
8815 test_float_cmp_op(&mut machine, MachineARM64::f32_cmp_lt)?;
8816 test_float_cmp_op(&mut machine, MachineARM64::f32_cmp_le)?;
8817
8818 Ok(())
8819 }
8820}