1use std::collections::HashMap;
2
3use dynasmrt::{VecAssembler, aarch64::Aarch64Relocation};
4use fixedbitset::FixedBitSet;
5#[cfg(feature = "unwind")]
6use gimli::{AArch64, write::CallFrameInstruction};
7
8use wasmer_compiler::{
9 types::{
10 address_map::InstructionAddressMap,
11 function::FunctionBody,
12 relocation::{Relocation, RelocationKind, RelocationTarget},
13 section::CustomSection,
14 },
15 wasmparser::MemArg,
16};
17use wasmer_types::{
18 CompileError, FunctionIndex, FunctionType, SourceLoc, TrapCode, TrapInformation, VMOffsets,
19 target::{CallingConvention, CpuFeature, Target},
20};
21
22use crate::{
23 arm64_decl::{GPR, NEON},
24 codegen_error,
25 common_decl::*,
26 emitter_arm64::*,
27 location::{Location as AbstractLocation, Reg},
28 machine::*,
29 unwind::{UnwindInstructions, UnwindOps, UnwindRegister},
30};
31
32type Assembler = VecAssembler<Aarch64Relocation>;
33type Location = AbstractLocation<GPR, NEON>;
34
35pub struct MachineARM64 {
36 assembler: Assembler,
37 used_gprs: FixedBitSet,
38 used_simd: FixedBitSet,
39 trap_table: TrapTable,
40 instructions_address_map: Vec<InstructionAddressMap>,
43 src_loc: u32,
45 pushed: bool,
47 unwind_ops: Vec<(usize, UnwindOps<GPR, NEON>)>,
49 has_neon: bool,
51}
52
53pub(crate) const ARM64_RETURN_VALUE_REGISTERS: [GPR; 8] = [
56 GPR::X0,
57 GPR::X1,
58 GPR::X2,
59 GPR::X3,
60 GPR::X4,
61 GPR::X5,
62 GPR::X6,
63 GPR::X7,
64];
65
66#[allow(dead_code)]
67#[derive(PartialEq)]
68enum ImmType {
69 None,
70 NoneXzr,
71 Bits8,
72 Bits12,
73 Shift32,
74 Shift32No0,
75 Shift64,
76 Shift64No0,
77 Logical32,
78 Logical64,
79 UnscaledOffset,
80 OffsetByte,
81 OffsetHWord,
82 OffsetWord,
83 OffsetDWord,
84}
85
86const SCRATCH_REG: GPR = GPR::X17;
87
88#[allow(dead_code)]
89impl MachineARM64 {
90 pub fn new(target: Option<Target>) -> Self {
91 let has_neon = match target {
95 Some(ref target) => target.cpu_features().contains(CpuFeature::NEON),
96 None => false,
97 };
98
99 MachineARM64 {
100 assembler: Assembler::new(0),
101 used_gprs: FixedBitSet::with_capacity(32),
102 used_simd: FixedBitSet::with_capacity(32),
103 trap_table: TrapTable::default(),
104 instructions_address_map: vec![],
105 src_loc: 0,
106 pushed: false,
107 unwind_ops: vec![],
108 has_neon,
109 }
110 }
111 fn compatible_imm(&self, imm: i64, ty: ImmType) -> bool {
112 match ty {
113 ImmType::None => false,
114 ImmType::NoneXzr => false,
115 ImmType::Bits8 => (0..256).contains(&imm),
116 ImmType::Bits12 => (0..0x1000).contains(&imm),
117 ImmType::Shift32 => (0..32).contains(&imm),
118 ImmType::Shift32No0 => (1..32).contains(&imm),
119 ImmType::Shift64 => (0..64).contains(&imm),
120 ImmType::Shift64No0 => (1..64).contains(&imm),
121 ImmType::Logical32 => encode_logical_immediate_32bit(imm as u32).is_some(),
122 ImmType::Logical64 => encode_logical_immediate_64bit(imm as u64).is_some(),
123 ImmType::UnscaledOffset => (imm > -256) && (imm < 256),
124 ImmType::OffsetByte => (0..0x1000).contains(&imm),
125 ImmType::OffsetHWord => (imm & 1 == 0) && (0..0x2000).contains(&imm),
126 ImmType::OffsetWord => (imm & 3 == 0) && (0..0x4000).contains(&imm),
127 ImmType::OffsetDWord => (imm & 7 == 0) && (0..0x8000).contains(&imm),
128 }
129 }
130
131 fn location_to_reg(
132 &mut self,
133 sz: Size,
134 src: Location,
135 temps: &mut Vec<GPR>,
136 allow_imm: ImmType,
137 read_val: bool,
138 wanted: Option<GPR>,
139 ) -> Result<Location, CompileError> {
140 match src {
141 Location::GPR(_) | Location::SIMD(_) => Ok(src),
142 Location::Imm8(val) => {
143 if allow_imm == ImmType::NoneXzr && val == 0 {
144 Ok(Location::GPR(GPR::XzrSp))
145 } else if self.compatible_imm(val as i64, allow_imm) {
146 Ok(src)
147 } else {
148 let tmp = if let Some(wanted) = wanted {
149 wanted
150 } else {
151 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
152 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
153 })?;
154 temps.push(tmp);
155 tmp
156 };
157 self.assembler
158 .emit_mov_imm(Location::GPR(tmp), val as u64)?;
159 Ok(Location::GPR(tmp))
160 }
161 }
162 Location::Imm32(val) => {
163 if allow_imm == ImmType::NoneXzr && val == 0 {
164 Ok(Location::GPR(GPR::XzrSp))
165 } else if self.compatible_imm(val as i64, allow_imm) {
166 Ok(src)
167 } else {
168 let tmp = if let Some(wanted) = wanted {
169 wanted
170 } else {
171 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
172 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
173 })?;
174 temps.push(tmp);
175 tmp
176 };
177 self.assembler
178 .emit_mov_imm(Location::GPR(tmp), (val as i64) as u64)?;
179 Ok(Location::GPR(tmp))
180 }
181 }
182 Location::Imm64(val) => {
183 if allow_imm == ImmType::NoneXzr && val == 0 {
184 Ok(Location::GPR(GPR::XzrSp))
185 } else if self.compatible_imm(val as i64, allow_imm) {
186 Ok(src)
187 } else {
188 let tmp = if let Some(wanted) = wanted {
189 wanted
190 } else {
191 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
192 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
193 })?;
194 temps.push(tmp);
195 tmp
196 };
197 self.assembler
198 .emit_mov_imm(Location::GPR(tmp), val as u64)?;
199 Ok(Location::GPR(tmp))
200 }
201 }
202 Location::Memory(reg, val) => {
203 let tmp = if let Some(wanted) = wanted {
204 wanted
205 } else {
206 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
207 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
208 })?;
209 temps.push(tmp);
210 tmp
211 };
212 if read_val {
213 let offsize = match sz {
214 Size::S8 => ImmType::OffsetByte,
215 Size::S16 => ImmType::OffsetHWord,
216 Size::S32 => ImmType::OffsetWord,
217 Size::S64 => ImmType::OffsetDWord,
218 };
219 if sz == Size::S8 {
220 if self.compatible_imm(val as i64, offsize) {
221 self.assembler.emit_ldrb(
222 sz,
223 Location::GPR(tmp),
224 Location::Memory(reg, val as _),
225 )?;
226 } else {
227 if reg == tmp {
228 codegen_error!("singlepass reg==tmp unreachable");
229 }
230 self.assembler
231 .emit_mov_imm(Location::GPR(tmp), (val as i64) as u64)?;
232 self.assembler.emit_ldrb(
233 sz,
234 Location::GPR(tmp),
235 Location::Memory2(reg, tmp, Multiplier::One, 0),
236 )?;
237 }
238 } else if sz == Size::S16 {
239 if self.compatible_imm(val as i64, offsize) {
240 self.assembler.emit_ldrh(
241 sz,
242 Location::GPR(tmp),
243 Location::Memory(reg, val as _),
244 )?;
245 } else {
246 if reg == tmp {
247 codegen_error!("singlepass reg==tmp unreachable");
248 }
249 self.assembler
250 .emit_mov_imm(Location::GPR(tmp), (val as i64) as u64)?;
251 self.assembler.emit_ldrh(
252 sz,
253 Location::GPR(tmp),
254 Location::Memory2(reg, tmp, Multiplier::One, 0),
255 )?;
256 }
257 } else if self.compatible_imm(val as i64, offsize) {
258 self.assembler.emit_ldr(
259 sz,
260 Location::GPR(tmp),
261 Location::Memory(reg, val as _),
262 )?;
263 } else if self.compatible_imm(val as i64, ImmType::UnscaledOffset) {
264 self.assembler.emit_ldur(sz, Location::GPR(tmp), reg, val)?;
265 } else {
266 if reg == tmp {
267 codegen_error!("singlepass reg == tmp unreachable");
268 }
269 self.assembler
270 .emit_mov_imm(Location::GPR(tmp), (val as i64) as u64)?;
271 self.assembler.emit_ldr(
272 sz,
273 Location::GPR(tmp),
274 Location::Memory2(reg, tmp, Multiplier::One, 0),
275 )?;
276 }
277 }
278 Ok(Location::GPR(tmp))
279 }
280 _ => codegen_error!("singlepass can't emit location_to_reg {:?} {:?}", sz, src),
281 }
282 }
283 fn location_to_neon(
284 &mut self,
285 sz: Size,
286 src: Location,
287 temps: &mut Vec<NEON>,
288 allow_imm: ImmType,
289 read_val: bool,
290 ) -> Result<Location, CompileError> {
291 match src {
292 Location::SIMD(_) => Ok(src),
293 Location::GPR(_) => {
294 let tmp = self.acquire_temp_simd().ok_or_else(|| {
295 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
296 })?;
297 temps.push(tmp);
298 if read_val {
299 self.assembler.emit_mov(sz, src, Location::SIMD(tmp))?;
300 }
301 Ok(Location::SIMD(tmp))
302 }
303 Location::Imm8(val) => {
304 if self.compatible_imm(val as i64, allow_imm) {
305 Ok(src)
306 } else {
307 let gpr = self.acquire_temp_gpr().ok_or_else(|| {
308 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
309 })?;
310 let tmp = self.acquire_temp_simd().ok_or_else(|| {
311 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
312 })?;
313 temps.push(tmp);
314 self.assembler
315 .emit_mov_imm(Location::GPR(gpr), val as u64)?;
316 self.assembler
317 .emit_mov(sz, Location::GPR(gpr), Location::SIMD(tmp))?;
318 self.release_gpr(gpr);
319 Ok(Location::SIMD(tmp))
320 }
321 }
322 Location::Imm32(val) => {
323 if self.compatible_imm(val as i64, allow_imm) {
324 Ok(src)
325 } else {
326 let gpr = self.acquire_temp_gpr().ok_or_else(|| {
327 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
328 })?;
329 let tmp = self.acquire_temp_simd().ok_or_else(|| {
330 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
331 })?;
332 temps.push(tmp);
333 self.assembler
334 .emit_mov_imm(Location::GPR(gpr), (val as i64) as u64)?;
335 self.assembler
336 .emit_mov(sz, Location::GPR(gpr), Location::SIMD(tmp))?;
337 self.release_gpr(gpr);
338 Ok(Location::SIMD(tmp))
339 }
340 }
341 Location::Imm64(val) => {
342 if self.compatible_imm(val as i64, allow_imm) {
343 Ok(src)
344 } else {
345 let gpr = self.acquire_temp_gpr().ok_or_else(|| {
346 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
347 })?;
348 let tmp = self.acquire_temp_simd().ok_or_else(|| {
349 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
350 })?;
351 temps.push(tmp);
352 self.assembler
353 .emit_mov_imm(Location::GPR(gpr), val as u64)?;
354 self.assembler
355 .emit_mov(sz, Location::GPR(gpr), Location::SIMD(tmp))?;
356 self.release_gpr(gpr);
357 Ok(Location::SIMD(tmp))
358 }
359 }
360 Location::Memory(reg, val) => {
361 let tmp = self.acquire_temp_simd().ok_or_else(|| {
362 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
363 })?;
364 temps.push(tmp);
365 if read_val {
366 let offsize = if sz == Size::S32 {
367 ImmType::OffsetWord
368 } else {
369 ImmType::OffsetDWord
370 };
371 if self.compatible_imm(val as i64, offsize) {
372 self.assembler.emit_ldr(
373 sz,
374 Location::SIMD(tmp),
375 Location::Memory(reg, val as _),
376 )?;
377 } else if self.compatible_imm(val as i64, ImmType::UnscaledOffset) {
378 self.assembler
379 .emit_ldur(sz, Location::SIMD(tmp), reg, val)?;
380 } else {
381 let gpr = self.acquire_temp_gpr().ok_or_else(|| {
382 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
383 })?;
384 self.assembler
385 .emit_mov_imm(Location::GPR(gpr), (val as i64) as u64)?;
386 self.assembler.emit_ldr(
387 sz,
388 Location::SIMD(tmp),
389 Location::Memory2(reg, gpr, Multiplier::One, 0),
390 )?;
391 self.release_gpr(gpr);
392 }
393 }
394 Ok(Location::SIMD(tmp))
395 }
396 _ => codegen_error!("singlepass can't emit location_to_neon {:?} {:?}", sz, src),
397 }
398 }
399
400 fn emit_relaxed_binop(
401 &mut self,
402 op: fn(&mut Assembler, Size, Location, Location) -> Result<(), CompileError>,
403 sz: Size,
404 src: Location,
405 dst: Location,
406 putback: bool,
407 ) -> Result<(), CompileError> {
408 let mut temps = vec![];
409 let src_imm = if putback {
410 ImmType::None
411 } else {
412 ImmType::Bits12
413 };
414 let src = self.location_to_reg(sz, src, &mut temps, src_imm, true, None)?;
415 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, !putback, None)?;
416 op(&mut self.assembler, sz, src, dest)?;
417 if dst != dest && putback {
418 self.move_location(sz, dest, dst)?;
419 }
420 for r in temps {
421 self.release_gpr(r);
422 }
423 Ok(())
424 }
425 fn emit_relaxed_binop_neon(
426 &mut self,
427 op: fn(&mut Assembler, Size, Location, Location) -> Result<(), CompileError>,
428 sz: Size,
429 src: Location,
430 dst: Location,
431 putback: bool,
432 ) -> Result<(), CompileError> {
433 let mut temps = vec![];
434 let src = self.location_to_neon(sz, src, &mut temps, ImmType::None, true)?;
435 let dest = self.location_to_neon(sz, dst, &mut temps, ImmType::None, !putback)?;
436 op(&mut self.assembler, sz, src, dest)?;
437 if dst != dest && putback {
438 self.move_location(sz, dest, dst)?;
439 }
440 for r in temps {
441 self.release_simd(r);
442 }
443 Ok(())
444 }
445 fn emit_relaxed_binop3(
446 &mut self,
447 op: fn(&mut Assembler, Size, Location, Location, Location) -> Result<(), CompileError>,
448 sz: Size,
449 src1: Location,
450 src2: Location,
451 dst: Location,
452 allow_imm: ImmType,
453 ) -> Result<(), CompileError> {
454 let mut temps = vec![];
455 let src1 = self.location_to_reg(sz, src1, &mut temps, ImmType::None, true, None)?;
456 let src2 = self.location_to_reg(sz, src2, &mut temps, allow_imm, true, None)?;
457 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
458 op(&mut self.assembler, sz, src1, src2, dest)?;
459 if dst != dest {
460 self.move_location(sz, dest, dst)?;
461 }
462 for r in temps {
463 self.release_gpr(r);
464 }
465 Ok(())
466 }
467 fn emit_relaxed_binop3_neon(
468 &mut self,
469 op: fn(&mut Assembler, Size, Location, Location, Location) -> Result<(), CompileError>,
470 sz: Size,
471 src1: Location,
472 src2: Location,
473 dst: Location,
474 allow_imm: ImmType,
475 ) -> Result<(), CompileError> {
476 let mut temps = vec![];
477 let src1 = self.location_to_neon(sz, src1, &mut temps, ImmType::None, true)?;
478 let src2 = self.location_to_neon(sz, src2, &mut temps, allow_imm, true)?;
479 let dest = self.location_to_neon(sz, dst, &mut temps, ImmType::None, false)?;
480 op(&mut self.assembler, sz, src1, src2, dest)?;
481 if dst != dest {
482 self.move_location(sz, dest, dst)?;
483 }
484 for r in temps {
485 self.release_simd(r);
486 }
487 Ok(())
488 }
489 fn emit_relaxed_ldr64(
490 &mut self,
491 sz: Size,
492 dst: Location,
493 src: Location,
494 ) -> Result<(), CompileError> {
495 let mut temps = vec![];
496 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
497 match src {
498 Location::Memory(addr, offset) => {
499 if self.compatible_imm(offset as i64, ImmType::OffsetDWord) {
500 self.assembler.emit_ldr(Size::S64, dest, src)?;
501 } else if self.compatible_imm(offset as i64, ImmType::UnscaledOffset) {
502 self.assembler.emit_ldur(Size::S64, dest, addr, offset)?;
503 } else {
504 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
505 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
506 })?;
507 self.assembler
508 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
509 self.assembler.emit_ldr(
510 Size::S64,
511 dest,
512 Location::Memory2(addr, tmp, Multiplier::One, 0),
513 )?;
514 temps.push(tmp);
515 }
516 }
517 _ => codegen_error!("singlepass emit_relaxed_ldr64 unreachable"),
518 }
519 if dst != dest {
520 self.move_location(sz, dest, dst)?;
521 }
522 for r in temps {
523 self.release_gpr(r);
524 }
525 Ok(())
526 }
527 fn emit_relaxed_ldr32(
528 &mut self,
529 sz: Size,
530 dst: Location,
531 src: Location,
532 ) -> Result<(), CompileError> {
533 let mut temps = vec![];
534 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
535 match src {
536 Location::Memory(addr, offset) => {
537 if self.compatible_imm(offset as i64, ImmType::OffsetWord) {
538 self.assembler.emit_ldr(Size::S32, dest, src)?;
539 } else if self.compatible_imm(offset as i64, ImmType::UnscaledOffset) {
540 self.assembler.emit_ldur(Size::S32, dest, addr, offset)?;
541 } else {
542 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
543 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
544 })?;
545 self.assembler
546 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
547 self.assembler.emit_ldr(
548 Size::S32,
549 dest,
550 Location::Memory2(addr, tmp, Multiplier::One, 0),
551 )?;
552 temps.push(tmp);
553 }
554 }
555 _ => codegen_error!("singlepass emit_relaxed_ldr32 unreachable"),
556 }
557 if dst != dest {
558 self.move_location(sz, dest, dst)?;
559 }
560 for r in temps {
561 self.release_gpr(r);
562 }
563 Ok(())
564 }
565 fn emit_relaxed_ldr32s(
566 &mut self,
567 sz: Size,
568 dst: Location,
569 src: Location,
570 ) -> Result<(), CompileError> {
571 let mut temps = vec![];
572 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
573 match src {
574 Location::Memory(addr, offset) => {
575 if self.compatible_imm(offset as i64, ImmType::OffsetWord) {
576 self.assembler.emit_ldrsw(Size::S64, dest, src)?;
577 } else {
578 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
579 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
580 })?;
581 self.assembler
582 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
583 self.assembler.emit_ldrsw(
584 Size::S64,
585 dest,
586 Location::Memory2(addr, tmp, Multiplier::One, 0),
587 )?;
588 temps.push(tmp);
589 }
590 }
591 _ => codegen_error!("singplepass emit_relaxed_ldr32s unreachable"),
592 }
593 if dst != dest {
594 self.move_location(sz, dest, dst)?;
595 }
596 for r in temps {
597 self.release_gpr(r);
598 }
599 Ok(())
600 }
601 fn emit_relaxed_ldr16(
602 &mut self,
603 sz: Size,
604 dst: Location,
605 src: Location,
606 ) -> Result<(), CompileError> {
607 let mut temps = vec![];
608 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
609 match src {
610 Location::Memory(addr, offset) => {
611 if self.compatible_imm(offset as i64, ImmType::OffsetHWord) {
612 self.assembler.emit_ldrh(Size::S32, dest, src)?;
613 } else {
614 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
615 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
616 })?;
617 self.assembler
618 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
619 self.assembler.emit_ldrh(
620 Size::S32,
621 dest,
622 Location::Memory2(addr, tmp, Multiplier::One, 0),
623 )?;
624 temps.push(tmp);
625 }
626 }
627 _ => codegen_error!("singlpass emit_relaxed_ldr16 unreachable"),
628 }
629 if dst != dest {
630 self.move_location(sz, dest, dst)?;
631 }
632 for r in temps {
633 self.release_gpr(r);
634 }
635 Ok(())
636 }
637 fn emit_relaxed_ldr16s(
638 &mut self,
639 sz: Size,
640 dst: Location,
641 src: Location,
642 ) -> Result<(), CompileError> {
643 let mut temps = vec![];
644 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
645 match src {
646 Location::Memory(addr, offset) => {
647 if self.compatible_imm(offset as i64, ImmType::OffsetHWord) {
648 self.assembler.emit_ldrsh(sz, dest, src)?;
649 } else {
650 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
651 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
652 })?;
653 self.assembler
654 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
655 self.assembler.emit_ldrsh(
656 sz,
657 dest,
658 Location::Memory2(addr, tmp, Multiplier::One, 0),
659 )?;
660 temps.push(tmp);
661 }
662 }
663 _ => codegen_error!("singlepass emit_relaxed_ldr16s unreachable"),
664 }
665 if dst != dest {
666 self.move_location(sz, dest, dst)?;
667 }
668 for r in temps {
669 self.release_gpr(r);
670 }
671 Ok(())
672 }
673 fn emit_relaxed_ldr8(
674 &mut self,
675 sz: Size,
676 dst: Location,
677 src: Location,
678 ) -> Result<(), CompileError> {
679 let mut temps = vec![];
680 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
681 match src {
682 Location::Memory(addr, offset) => {
683 if self.compatible_imm(offset as i64, ImmType::OffsetByte) {
684 self.assembler.emit_ldrb(Size::S32, dest, src)?;
685 } else {
686 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
687 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
688 })?;
689 self.assembler
690 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
691 self.assembler.emit_ldrb(
692 Size::S32,
693 dest,
694 Location::Memory2(addr, tmp, Multiplier::One, 0),
695 )?;
696 temps.push(tmp);
697 }
698 }
699 _ => codegen_error!("singplepass emit_relaxed_ldr8 unreachable"),
700 }
701 if dst != dest {
702 self.move_location(sz, dest, dst)?;
703 }
704 for r in temps {
705 self.release_gpr(r);
706 }
707 Ok(())
708 }
709 fn emit_relaxed_ldr8s(
710 &mut self,
711 sz: Size,
712 dst: Location,
713 src: Location,
714 ) -> Result<(), CompileError> {
715 let mut temps = vec![];
716 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
717 match src {
718 Location::Memory(addr, offset) => {
719 if self.compatible_imm(offset as i64, ImmType::OffsetByte) {
720 self.assembler.emit_ldrsb(sz, dest, src)?;
721 } else {
722 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
723 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
724 })?;
725 self.assembler
726 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
727 self.assembler.emit_ldrsb(
728 sz,
729 dest,
730 Location::Memory2(addr, tmp, Multiplier::One, 0),
731 )?;
732 temps.push(tmp);
733 }
734 }
735 _ => codegen_error!("singlepass emit_relaxed_ldr8s unreachable"),
736 }
737 if dst != dest {
738 self.move_location(sz, dest, dst)?;
739 }
740 for r in temps {
741 self.release_gpr(r);
742 }
743 Ok(())
744 }
745 fn emit_relaxed_str64(&mut self, dst: Location, src: Location) -> Result<(), CompileError> {
746 let mut temps = vec![];
747 let dst = self.location_to_reg(Size::S64, dst, &mut temps, ImmType::NoneXzr, true, None)?;
748 match src {
749 Location::Memory(addr, offset) => {
750 if self.compatible_imm(offset as i64, ImmType::OffsetDWord) {
751 self.assembler.emit_str(Size::S64, dst, src)?;
752 } else if self.compatible_imm(offset as i64, ImmType::UnscaledOffset) {
753 self.assembler.emit_stur(Size::S64, dst, addr, offset)?;
754 } else {
755 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
756 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
757 })?;
758 self.assembler
759 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
760 self.assembler.emit_str(
761 Size::S64,
762 dst,
763 Location::Memory2(addr, tmp, Multiplier::One, 0),
764 )?;
765 temps.push(tmp);
766 }
767 }
768 _ => codegen_error!("singlepass can't emit str64 {:?} {:?}", dst, src),
769 }
770 for r in temps {
771 self.release_gpr(r);
772 }
773 Ok(())
774 }
775 fn emit_relaxed_str32(&mut self, dst: Location, src: Location) -> Result<(), CompileError> {
776 let mut temps = vec![];
777 let dst = self.location_to_reg(Size::S64, dst, &mut temps, ImmType::NoneXzr, true, None)?;
778 match src {
779 Location::Memory(addr, offset) => {
780 if self.compatible_imm(offset as i64, ImmType::OffsetWord) {
781 self.assembler.emit_str(Size::S32, dst, src)?;
782 } else if self.compatible_imm(offset as i64, ImmType::UnscaledOffset) {
783 self.assembler.emit_stur(Size::S32, dst, addr, offset)?;
784 } else {
785 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
786 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
787 })?;
788 self.assembler
789 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
790 self.assembler.emit_str(
791 Size::S32,
792 dst,
793 Location::Memory2(addr, tmp, Multiplier::One, 0),
794 )?;
795 temps.push(tmp);
796 }
797 }
798 _ => codegen_error!("singplepass emit_relaxed_str32 unreachable"),
799 }
800 for r in temps {
801 self.release_gpr(r);
802 }
803 Ok(())
804 }
805 fn emit_relaxed_str16(&mut self, dst: Location, src: Location) -> Result<(), CompileError> {
806 let mut temps = vec![];
807 let dst = self.location_to_reg(Size::S64, dst, &mut temps, ImmType::NoneXzr, true, None)?;
808 match src {
809 Location::Memory(addr, offset) => {
810 if self.compatible_imm(offset as i64, ImmType::OffsetHWord) {
811 self.assembler.emit_strh(Size::S32, dst, src)?;
812 } else {
813 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
814 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
815 })?;
816 self.assembler
817 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
818 self.assembler.emit_strh(
819 Size::S32,
820 dst,
821 Location::Memory2(addr, tmp, Multiplier::One, 0),
822 )?;
823 temps.push(tmp);
824 }
825 }
826 _ => codegen_error!("singlepass emit_relaxed_str16 unreachable"),
827 }
828 for r in temps {
829 self.release_gpr(r);
830 }
831 Ok(())
832 }
833 fn emit_relaxed_str8(&mut self, dst: Location, src: Location) -> Result<(), CompileError> {
834 let mut temps = vec![];
835 let dst = self.location_to_reg(Size::S64, dst, &mut temps, ImmType::NoneXzr, true, None)?;
836 match src {
837 Location::Memory(addr, offset) => {
838 if self.compatible_imm(offset as i64, ImmType::OffsetByte) {
839 self.assembler
840 .emit_strb(Size::S32, dst, Location::Memory(addr, offset))?;
841 } else {
842 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
843 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
844 })?;
845 self.assembler
846 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
847 self.assembler.emit_strb(
848 Size::S32,
849 dst,
850 Location::Memory2(addr, tmp, Multiplier::One, 0),
851 )?;
852 temps.push(tmp);
853 }
854 }
855 _ => codegen_error!("singlepass emit_relaxed_str8 unreachable"),
856 }
857 for r in temps {
858 self.release_gpr(r);
859 }
860 Ok(())
861 }
862 fn emit_cmpop_i64_dynamic_b(
864 &mut self,
865 c: Condition,
866 loc_a: Location,
867 loc_b: Location,
868 ret: Location,
869 ) -> Result<(), CompileError> {
870 match ret {
871 Location::GPR(_) => {
872 self.emit_relaxed_cmp(Size::S64, loc_b, loc_a)?;
873 self.assembler.emit_cset(Size::S32, ret, c)?;
874 }
875 Location::Memory(_, _) => {
876 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
877 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
878 })?;
879 self.emit_relaxed_cmp(Size::S64, loc_b, loc_a)?;
880 self.assembler.emit_cset(Size::S32, Location::GPR(tmp), c)?;
881 self.move_location(Size::S32, Location::GPR(tmp), ret)?;
882 self.release_gpr(tmp);
883 }
884 _ => {
885 codegen_error!("singlepass emit_compop_i64_dynamic_b unreachable");
886 }
887 }
888 Ok(())
889 }
890 fn emit_cmpop_i32_dynamic_b(
892 &mut self,
893 c: Condition,
894 loc_a: Location,
895 loc_b: Location,
896 ret: Location,
897 ) -> Result<(), CompileError> {
898 match ret {
899 Location::GPR(_) => {
900 self.emit_relaxed_cmp(Size::S32, loc_b, loc_a)?;
901 self.assembler.emit_cset(Size::S32, ret, c)?;
902 }
903 Location::Memory(_, _) => {
904 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
905 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
906 })?;
907 self.emit_relaxed_cmp(Size::S32, loc_b, loc_a)?;
908 self.assembler.emit_cset(Size::S32, Location::GPR(tmp), c)?;
909 self.move_location(Size::S32, Location::GPR(tmp), ret)?;
910 self.release_gpr(tmp);
911 }
912 _ => {
913 codegen_error!("singlepass emit_cmpop_i32_dynamic_b unreachable");
914 }
915 }
916 Ok(())
917 }
918
919 #[allow(clippy::too_many_arguments)]
920 fn memory_op<F: FnOnce(&mut Self, GPR) -> Result<(), CompileError>>(
921 &mut self,
922 addr: Location,
923 memarg: &MemArg,
924 check_alignment: bool,
925 value_size: usize,
926 need_check: bool,
927 imported_memories: bool,
928 offset: i32,
929 heap_access_oob: Label,
930 unaligned_atomic: Label,
931 cb: F,
932 ) -> Result<(), CompileError> {
933 let tmp_addr = self.acquire_temp_gpr().ok_or_else(|| {
934 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
935 })?;
936
937 let (base_loc, bound_loc) = if imported_memories {
939 self.emit_relaxed_binop(
941 Assembler::emit_mov,
942 Size::S64,
943 Location::Memory(self.get_vmctx_reg(), offset),
944 Location::GPR(tmp_addr),
945 true,
946 )?;
947 (Location::Memory(tmp_addr, 0), Location::Memory(tmp_addr, 8))
948 } else {
949 (
950 Location::Memory(self.get_vmctx_reg(), offset),
951 Location::Memory(self.get_vmctx_reg(), offset + 8),
952 )
953 };
954
955 let tmp_base = self.acquire_temp_gpr().ok_or_else(|| {
956 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
957 })?;
958 let tmp_bound = self.acquire_temp_gpr().ok_or_else(|| {
959 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
960 })?;
961
962 self.emit_relaxed_ldr64(Size::S64, Location::GPR(tmp_base), base_loc)?;
964
965 if need_check {
967 self.emit_relaxed_ldr64(Size::S64, Location::GPR(tmp_bound), bound_loc)?;
968
969 self.assembler.emit_add(
973 Size::S64,
974 Location::GPR(tmp_bound),
975 Location::GPR(tmp_base),
976 Location::GPR(tmp_bound),
977 )?;
978 if self.compatible_imm(value_size as _, ImmType::Bits12) {
979 self.assembler.emit_sub(
980 Size::S64,
981 Location::GPR(tmp_bound),
982 Location::Imm32(value_size as _),
983 Location::GPR(tmp_bound),
984 )?;
985 } else {
986 let tmp2 = self.acquire_temp_gpr().ok_or_else(|| {
987 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
988 })?;
989 self.assembler
990 .emit_mov_imm(Location::GPR(tmp2), value_size as u64)?;
991 self.assembler.emit_sub(
992 Size::S64,
993 Location::GPR(tmp_bound),
994 Location::GPR(tmp2),
995 Location::GPR(tmp_bound),
996 )?;
997 self.release_gpr(tmp2);
998 }
999 }
1000
1001 self.move_location(Size::S32, addr, Location::GPR(tmp_addr))?;
1005
1006 if memarg.offset != 0 {
1008 if self.compatible_imm(memarg.offset as _, ImmType::Bits12) {
1009 self.assembler.emit_adds(
1010 Size::S32,
1011 Location::Imm32(memarg.offset as u32),
1012 Location::GPR(tmp_addr),
1013 Location::GPR(tmp_addr),
1014 )?;
1015 } else {
1016 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
1017 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1018 })?;
1019 self.assembler
1020 .emit_mov_imm(Location::GPR(tmp), memarg.offset as _)?;
1021 self.assembler.emit_adds(
1022 Size::S32,
1023 Location::GPR(tmp_addr),
1024 Location::GPR(tmp),
1025 Location::GPR(tmp_addr),
1026 )?;
1027 self.release_gpr(tmp);
1028 }
1029
1030 self.assembler
1032 .emit_bcond_label_far(Condition::Cs, heap_access_oob)?;
1033 }
1034
1035 self.assembler.emit_add(
1037 Size::S64,
1038 Location::GPR(tmp_base),
1039 Location::GPR(tmp_addr),
1040 Location::GPR(tmp_addr),
1041 )?;
1042
1043 if need_check {
1044 self.assembler.emit_cmp(
1046 Size::S64,
1047 Location::GPR(tmp_bound),
1048 Location::GPR(tmp_addr),
1049 )?;
1050
1051 self.assembler
1053 .emit_bcond_label_far(Condition::Hi, heap_access_oob)?;
1054 }
1055
1056 self.release_gpr(tmp_bound);
1057 self.release_gpr(tmp_base);
1058
1059 let align = value_size as u32;
1060 if check_alignment && align != 1 {
1061 self.assembler.emit_tst(
1062 Size::S64,
1063 Location::Imm32(align - 1),
1064 Location::GPR(tmp_addr),
1065 )?;
1066 self.assembler
1067 .emit_bcond_label_far(Condition::Ne, unaligned_atomic)?;
1068 }
1069 let begin = self.assembler.get_offset().0;
1070 cb(self, tmp_addr)?;
1071 let end = self.assembler.get_offset().0;
1072 self.mark_address_range_with_trap_code(TrapCode::HeapAccessOutOfBounds, begin, end);
1073
1074 self.release_gpr(tmp_addr);
1075 Ok(())
1076 }
1077
1078 fn offset_is_ok(&self, size: Size, offset: i32) -> bool {
1079 if offset < 0 {
1080 return false;
1081 }
1082 let shift = size.bytes().trailing_zeros() as i32;
1083 if offset >= 0x1000 << shift {
1084 return false;
1085 }
1086 if (offset & ((1 << shift) - 1)) != 0 {
1087 return false;
1088 }
1089 true
1090 }
1091
1092 fn emit_push(&mut self, sz: Size, src: Location) -> Result<(), CompileError> {
1093 match (sz, src) {
1094 (Size::S64, Location::GPR(_)) | (Size::S64, Location::SIMD(_)) => {
1095 let offset = if self.pushed {
1096 0
1097 } else {
1098 self.assembler.emit_sub(
1099 Size::S64,
1100 Location::GPR(GPR::XzrSp),
1101 Location::Imm8(16),
1102 Location::GPR(GPR::XzrSp),
1103 )?;
1104 8
1105 };
1106 self.assembler
1107 .emit_stur(Size::S64, src, GPR::XzrSp, offset)?;
1108 self.pushed = !self.pushed;
1109 }
1110 (Size::S64, _) => {
1111 let mut temps = vec![];
1112 let src = self.location_to_reg(sz, src, &mut temps, ImmType::None, true, None)?;
1113 let offset = if self.pushed {
1114 0
1115 } else {
1116 self.assembler.emit_sub(
1117 Size::S64,
1118 Location::GPR(GPR::XzrSp),
1119 Location::Imm8(16),
1120 Location::GPR(GPR::XzrSp),
1121 )?;
1122 8
1123 };
1124 self.assembler
1125 .emit_stur(Size::S64, src, GPR::XzrSp, offset)?;
1126 self.pushed = !self.pushed;
1127 for r in temps {
1128 self.release_gpr(r);
1129 }
1130 }
1131 _ => codegen_error!("singlepass can't emit PUSH {:?} {:?}", sz, src),
1132 }
1133 Ok(())
1134 }
1135 fn emit_double_push(
1136 &mut self,
1137 sz: Size,
1138 src1: Location,
1139 src2: Location,
1140 ) -> Result<(), CompileError> {
1141 if !self.pushed {
1142 match (sz, src1, src2) {
1143 (Size::S64, Location::GPR(_), Location::GPR(_)) => {
1144 self.assembler
1145 .emit_stpdb(Size::S64, src1, src2, GPR::XzrSp, 16)?;
1146 }
1147 _ => {
1148 self.emit_push(sz, src1)?;
1149 self.emit_push(sz, src2)?;
1150 }
1151 }
1152 } else {
1153 self.emit_push(sz, src1)?;
1154 self.emit_push(sz, src2)?;
1155 }
1156 Ok(())
1157 }
1158 fn emit_pop(&mut self, sz: Size, dst: Location) -> Result<(), CompileError> {
1159 match (sz, dst) {
1160 (Size::S64, Location::GPR(_)) | (Size::S64, Location::SIMD(_)) => {
1161 let offset = if self.pushed { 8 } else { 0 };
1162 self.assembler
1163 .emit_ldur(Size::S64, dst, GPR::XzrSp, offset)?;
1164 if self.pushed {
1165 self.assembler.emit_add(
1166 Size::S64,
1167 Location::GPR(GPR::XzrSp),
1168 Location::Imm8(16),
1169 Location::GPR(GPR::XzrSp),
1170 )?;
1171 }
1172 self.pushed = !self.pushed;
1173 }
1174 _ => codegen_error!("singlepass can't emit POP {:?} {:?}", sz, dst),
1175 }
1176 Ok(())
1177 }
1178 fn emit_double_pop(
1179 &mut self,
1180 sz: Size,
1181 dst1: Location,
1182 dst2: Location,
1183 ) -> Result<(), CompileError> {
1184 if !self.pushed {
1185 match (sz, dst1, dst2) {
1186 (Size::S64, Location::GPR(_), Location::GPR(_)) => {
1187 self.assembler
1188 .emit_ldpia(Size::S64, dst1, dst2, GPR::XzrSp, 16)?;
1189 }
1190 _ => {
1191 self.emit_pop(sz, dst2)?;
1192 self.emit_pop(sz, dst1)?;
1193 }
1194 }
1195 } else {
1196 self.emit_pop(sz, dst2)?;
1197 self.emit_pop(sz, dst1)?;
1198 }
1199 Ok(())
1200 }
1201
1202 fn set_default_nan(&mut self, temps: &mut Vec<GPR>) -> Result<GPR, CompileError> {
1203 let old_fpcr = self.acquire_temp_gpr().ok_or_else(|| {
1205 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1206 })?;
1207 temps.push(old_fpcr);
1208 self.assembler.emit_read_fpcr(old_fpcr)?;
1209 let new_fpcr = self.acquire_temp_gpr().ok_or_else(|| {
1210 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1211 })?;
1212 temps.push(new_fpcr);
1213 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
1214 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1215 })?;
1216 temps.push(tmp);
1217 self.assembler
1218 .emit_mov(Size::S32, Location::Imm32(1), Location::GPR(tmp))?;
1219 self.assembler
1220 .emit_mov(Size::S64, Location::GPR(old_fpcr), Location::GPR(new_fpcr))?;
1221 self.assembler.emit_bfi(
1223 Size::S64,
1224 Location::GPR(tmp),
1225 25,
1226 1,
1227 Location::GPR(new_fpcr),
1228 )?;
1229 self.assembler.emit_write_fpcr(new_fpcr)?;
1230 Ok(old_fpcr)
1231 }
1232 fn set_trap_enabled(&mut self, temps: &mut Vec<GPR>) -> Result<GPR, CompileError> {
1233 let old_fpcr = self.acquire_temp_gpr().ok_or_else(|| {
1235 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1236 })?;
1237 temps.push(old_fpcr);
1238 self.assembler.emit_read_fpcr(old_fpcr)?;
1239 let new_fpcr = self.acquire_temp_gpr().ok_or_else(|| {
1240 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1241 })?;
1242 temps.push(new_fpcr);
1243 self.assembler
1244 .emit_mov(Size::S64, Location::GPR(old_fpcr), Location::GPR(new_fpcr))?;
1245 self.assembler
1247 .emit_bfc(Size::S64, 8, 1, Location::GPR(new_fpcr))?;
1248 self.assembler.emit_write_fpcr(new_fpcr)?;
1249 Ok(old_fpcr)
1250 }
1251 fn restore_fpcr(&mut self, old_fpcr: GPR) -> Result<(), CompileError> {
1252 self.assembler.emit_write_fpcr(old_fpcr)
1253 }
1254
1255 fn reset_exception_fpsr(&mut self) -> Result<(), CompileError> {
1256 let fpsr = self.acquire_temp_gpr().ok_or_else(|| {
1258 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1259 })?;
1260 self.assembler.emit_read_fpsr(fpsr)?;
1261 self.assembler
1263 .emit_bfc(Size::S64, 0, 1, Location::GPR(fpsr))?;
1264 self.assembler.emit_write_fpsr(fpsr)?;
1265 self.release_gpr(fpsr);
1266 Ok(())
1267 }
1268 fn read_fpsr(&mut self) -> Result<GPR, CompileError> {
1269 let fpsr = self.acquire_temp_gpr().ok_or_else(|| {
1270 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1271 })?;
1272 self.assembler.emit_read_fpsr(fpsr)?;
1273 Ok(fpsr)
1274 }
1275
1276 fn trap_float_convertion_errors(
1277 &mut self,
1278 old_fpcr: GPR,
1279 sz: Size,
1280 f: Location,
1281 temps: &mut Vec<GPR>,
1282 ) -> Result<(), CompileError> {
1283 let trap_badconv = self.assembler.get_label();
1284 let end = self.assembler.get_label();
1285
1286 let fpsr = self.read_fpsr()?;
1287 temps.push(fpsr);
1288 self.assembler
1290 .emit_tbz_label(Size::S32, Location::GPR(fpsr), 0, end)?;
1291 self.assembler
1293 .emit_bfc(Size::S64, 0, 4, Location::GPR(fpsr))?;
1294 self.restore_fpcr(old_fpcr)?;
1295 self.assembler.emit_fcmp(sz, f, f)?;
1296 self.assembler
1297 .emit_bcond_label(Condition::Vs, trap_badconv)?;
1298 self.emit_illegal_op_internal(TrapCode::IntegerOverflow)?;
1300
1301 self.emit_label(trap_badconv)?;
1302 self.emit_illegal_op_internal(TrapCode::BadConversionToInteger)?;
1303
1304 self.emit_label(end)?;
1305 self.restore_fpcr(old_fpcr)
1306 }
1307
1308 fn used_gprs_contains(&self, r: &GPR) -> bool {
1309 self.used_gprs.contains(r.into_index())
1310 }
1311 fn used_simd_contains(&self, r: &NEON) -> bool {
1312 self.used_simd.contains(r.into_index())
1313 }
1314 fn used_gprs_insert(&mut self, r: GPR) {
1315 self.used_gprs.insert(r.into_index());
1316 }
1317 fn used_simd_insert(&mut self, r: NEON) {
1318 self.used_simd.insert(r.into_index());
1319 }
1320 fn used_gprs_remove(&mut self, r: &GPR) -> bool {
1321 let ret = self.used_gprs_contains(r);
1322 self.used_gprs.set(r.into_index(), false);
1323 ret
1324 }
1325 fn used_simd_remove(&mut self, r: &NEON) -> bool {
1326 let ret = self.used_simd_contains(r);
1327 self.used_simd.set(r.into_index(), false);
1328 ret
1329 }
1330 fn emit_unwind_op(&mut self, op: UnwindOps<GPR, NEON>) {
1331 self.unwind_ops.push((self.get_offset().0, op));
1332 }
1333 fn emit_illegal_op_internal(&mut self, trap: TrapCode) -> Result<(), CompileError> {
1334 self.assembler.emit_udf(0xc0 | (trap as u8) as u16)
1335 }
1336}
1337
1338impl Machine for MachineARM64 {
1339 type GPR = GPR;
1340 type SIMD = NEON;
1341
1342 fn assembler_get_offset(&self) -> Offset {
1343 self.assembler.get_offset()
1344 }
1345
1346 fn get_vmctx_reg(&self) -> GPR {
1347 GPR::X28
1348 }
1349
1350 fn get_used_gprs(&self) -> Vec<GPR> {
1351 GPR::iterator()
1352 .filter(|x| self.used_gprs.contains(x.into_index()))
1353 .cloned()
1354 .collect()
1355 }
1356
1357 fn get_used_simd(&self) -> Vec<NEON> {
1358 NEON::iterator()
1359 .filter(|x| self.used_simd.contains(x.into_index()))
1360 .cloned()
1361 .collect()
1362 }
1363
1364 fn pick_gpr(&self) -> Option<GPR> {
1365 use GPR::*;
1366 static REGS: &[GPR] = &[X9, X10, X11, X12, X13, X14, X15];
1367 for r in REGS {
1368 if !self.used_gprs_contains(r) {
1369 return Some(*r);
1370 }
1371 }
1372 None
1373 }
1374
1375 fn pick_temp_gpr(&self) -> Option<GPR> {
1376 use GPR::*;
1377 static REGS: &[GPR] = &[X8, X7, X6, X5, X4, X3, X2, X1];
1378 for r in REGS {
1379 if !self.used_gprs_contains(r) {
1380 return Some(*r);
1381 }
1382 }
1383 None
1384 }
1385
1386 fn acquire_temp_gpr(&mut self) -> Option<GPR> {
1387 let gpr = self.pick_temp_gpr();
1388 if let Some(x) = gpr {
1389 self.used_gprs_insert(x);
1390 }
1391 gpr
1392 }
1393
1394 fn release_gpr(&mut self, gpr: GPR) {
1395 assert!(self.used_gprs_remove(&gpr));
1396 }
1397
1398 fn reserve_unused_temp_gpr(&mut self, gpr: GPR) -> GPR {
1399 assert!(!self.used_gprs_contains(&gpr));
1400 self.used_gprs_insert(gpr);
1401 gpr
1402 }
1403
1404 fn reserve_gpr(&mut self, gpr: GPR) {
1405 self.used_gprs_insert(gpr);
1406 }
1407
1408 fn push_used_gpr(&mut self, used_gprs: &[GPR]) -> Result<usize, CompileError> {
1409 if used_gprs.len() % 2 == 1 {
1410 self.emit_push(Size::S64, Location::GPR(GPR::XzrSp))?;
1411 }
1412 for r in used_gprs.iter() {
1413 self.emit_push(Size::S64, Location::GPR(*r))?;
1414 }
1415 Ok(used_gprs.len().div_ceil(2) * 16)
1416 }
1417
1418 fn pop_used_gpr(&mut self, used_gprs: &[GPR]) -> Result<(), CompileError> {
1419 for r in used_gprs.iter().rev() {
1420 self.emit_pop(Size::S64, Location::GPR(*r))?;
1421 }
1422 if used_gprs.len() % 2 == 1 {
1423 self.emit_pop(Size::S64, Location::GPR(GPR::XzrSp))?;
1424 }
1425 Ok(())
1426 }
1427
1428 fn pick_simd(&self) -> Option<NEON> {
1429 use NEON::*;
1430 static REGS: &[NEON] = &[V8, V9, V10, V11, V12];
1431 for r in REGS {
1432 if !self.used_simd_contains(r) {
1433 return Some(*r);
1434 }
1435 }
1436 None
1437 }
1438
1439 fn pick_temp_simd(&self) -> Option<NEON> {
1440 use NEON::*;
1441 static REGS: &[NEON] = &[V0, V1, V2, V3, V4, V5, V6, V7];
1442 for r in REGS {
1443 if !self.used_simd_contains(r) {
1444 return Some(*r);
1445 }
1446 }
1447 None
1448 }
1449
1450 fn acquire_temp_simd(&mut self) -> Option<NEON> {
1451 let simd = self.pick_temp_simd();
1452 if let Some(x) = simd {
1453 self.used_simd_insert(x);
1454 }
1455 simd
1456 }
1457
1458 fn reserve_simd(&mut self, simd: NEON) {
1459 self.used_simd_insert(simd);
1460 }
1461
1462 fn release_simd(&mut self, simd: NEON) {
1463 assert!(self.used_simd_remove(&simd));
1464 }
1465
1466 fn push_used_simd(&mut self, used_neons: &[NEON]) -> Result<usize, CompileError> {
1467 let stack_adjust = if used_neons.len() % 2 == 1 {
1468 (used_neons.len() * 8) as u32 + 8
1469 } else {
1470 (used_neons.len() * 8) as u32
1471 };
1472 self.extend_stack(stack_adjust)?;
1473
1474 for (i, r) in used_neons.iter().enumerate() {
1475 self.assembler.emit_str(
1476 Size::S64,
1477 Location::SIMD(*r),
1478 Location::Memory(GPR::XzrSp, (i * 8) as i32),
1479 )?;
1480 }
1481 Ok(stack_adjust as usize)
1482 }
1483
1484 fn pop_used_simd(&mut self, used_neons: &[NEON]) -> Result<(), CompileError> {
1485 for (i, r) in used_neons.iter().enumerate() {
1486 self.assembler.emit_ldr(
1487 Size::S64,
1488 Location::SIMD(*r),
1489 Location::Memory(GPR::XzrSp, (i * 8) as i32),
1490 )?;
1491 }
1492 let stack_adjust = if used_neons.len() % 2 == 1 {
1493 (used_neons.len() * 8) as u32 + 8
1494 } else {
1495 (used_neons.len() * 8) as u32
1496 };
1497 self.assembler.emit_add(
1498 Size::S64,
1499 Location::GPR(GPR::XzrSp),
1500 Location::Imm32(stack_adjust as _),
1501 Location::GPR(GPR::XzrSp),
1502 )
1503 }
1504
1505 fn set_srcloc(&mut self, offset: u32) {
1506 self.src_loc = offset;
1507 }
1508
1509 fn mark_address_range_with_trap_code(&mut self, code: TrapCode, begin: usize, end: usize) {
1510 for i in begin..end {
1511 self.trap_table.offset_to_code.insert(i, code);
1512 }
1513 self.mark_instruction_address_end(begin);
1514 }
1515
1516 fn mark_address_with_trap_code(&mut self, code: TrapCode) {
1517 let offset = self.assembler.get_offset().0;
1518 self.trap_table.offset_to_code.insert(offset, code);
1519 self.mark_instruction_address_end(offset);
1520 }
1521
1522 fn mark_instruction_with_trap_code(&mut self, code: TrapCode) -> usize {
1523 let offset = self.assembler.get_offset().0;
1524 self.trap_table.offset_to_code.insert(offset, code);
1525 offset
1526 }
1527
1528 fn mark_instruction_address_end(&mut self, begin: usize) {
1529 self.instructions_address_map.push(InstructionAddressMap {
1530 srcloc: SourceLoc::new(self.src_loc),
1531 code_offset: begin,
1532 code_len: self.assembler.get_offset().0 - begin,
1533 });
1534 }
1535
1536 fn insert_stackoverflow(&mut self) {
1537 let offset = 0;
1538 self.trap_table
1539 .offset_to_code
1540 .insert(offset, TrapCode::StackOverflow);
1541 self.mark_instruction_address_end(offset);
1542 }
1543
1544 fn collect_trap_information(&self) -> Vec<TrapInformation> {
1545 self.trap_table
1546 .offset_to_code
1547 .clone()
1548 .into_iter()
1549 .map(|(offset, code)| TrapInformation {
1550 code_offset: offset as u32,
1551 trap_code: code,
1552 })
1553 .collect()
1554 }
1555
1556 fn instructions_address_map(&self) -> Vec<InstructionAddressMap> {
1557 self.instructions_address_map.clone()
1558 }
1559
1560 fn round_stack_adjust(&self, value: usize) -> usize {
1561 value.next_multiple_of(16)
1562 }
1563
1564 fn local_on_stack(&mut self, stack_offset: i32) -> Location {
1565 Location::Memory(GPR::X29, -stack_offset)
1566 }
1567
1568 fn extend_stack(&mut self, delta_stack_offset: u32) -> Result<(), CompileError> {
1569 let delta = if self.compatible_imm(delta_stack_offset as _, ImmType::Bits12) {
1570 Location::Imm32(delta_stack_offset as _)
1571 } else {
1572 let tmp = SCRATCH_REG;
1573 self.assembler
1574 .emit_mov_imm(Location::GPR(tmp), delta_stack_offset as u64)?;
1575 Location::GPR(tmp)
1576 };
1577 self.assembler.emit_sub(
1578 Size::S64,
1579 Location::GPR(GPR::XzrSp),
1580 delta,
1581 Location::GPR(GPR::XzrSp),
1582 )
1583 }
1584
1585 fn truncate_stack(&mut self, delta_stack_offset: u32) -> Result<(), CompileError> {
1586 let delta = if self.compatible_imm(delta_stack_offset as _, ImmType::Bits12) {
1587 Location::Imm32(delta_stack_offset as _)
1588 } else {
1589 let tmp = SCRATCH_REG;
1590 self.assembler
1591 .emit_mov_imm(Location::GPR(tmp), delta_stack_offset as u64)?;
1592 Location::GPR(tmp)
1593 };
1594 self.assembler.emit_add(
1595 Size::S64,
1596 Location::GPR(GPR::XzrSp),
1597 delta,
1598 Location::GPR(GPR::XzrSp),
1599 )
1600 }
1601
1602 fn move_location_for_native(
1603 &mut self,
1604 size: Size,
1605 loc: Location,
1606 dest: Location,
1607 ) -> Result<(), CompileError> {
1608 match loc {
1609 Location::Imm64(_)
1610 | Location::Imm32(_)
1611 | Location::Imm8(_)
1612 | Location::Memory(_, _)
1613 | Location::Memory2(_, _, _, _) => {
1614 self.move_location(size, loc, Location::GPR(SCRATCH_REG))?;
1615 self.move_location(size, Location::GPR(SCRATCH_REG), dest)
1616 }
1617 _ => self.move_location(size, loc, dest),
1618 }
1619 }
1620
1621 fn zero_location(&mut self, size: Size, location: Location) -> Result<(), CompileError> {
1622 self.move_location(size, Location::GPR(GPR::XzrSp), location)
1623 }
1624
1625 fn local_pointer(&self) -> GPR {
1626 GPR::X29
1627 }
1628
1629 fn is_local_on_stack(&self, idx: usize) -> bool {
1630 idx > 7
1631 }
1632
1633 fn get_local_location(&self, idx: usize, callee_saved_regs_size: usize) -> Location {
1634 match idx {
1636 0 => Location::GPR(GPR::X19),
1637 1 => Location::GPR(GPR::X20),
1638 2 => Location::GPR(GPR::X21),
1639 3 => Location::GPR(GPR::X22),
1640 4 => Location::GPR(GPR::X23),
1641 5 => Location::GPR(GPR::X24),
1642 6 => Location::GPR(GPR::X25),
1643 7 => Location::GPR(GPR::X26),
1644 _ => Location::Memory(GPR::X29, -(((idx - 7) * 8 + callee_saved_regs_size) as i32)),
1645 }
1646 }
1647
1648 fn move_local(&mut self, stack_offset: i32, location: Location) -> Result<(), CompileError> {
1649 if stack_offset < 256 {
1650 self.assembler
1651 .emit_stur(Size::S64, location, GPR::X29, -stack_offset)?;
1652 } else {
1653 let tmp = SCRATCH_REG;
1654 if stack_offset < 0x1_0000 {
1655 self.assembler
1656 .emit_mov_imm(Location::GPR(tmp), (-stack_offset as i64) as u64)?;
1657 self.assembler.emit_str(
1658 Size::S64,
1659 location,
1660 Location::Memory2(GPR::X29, tmp, Multiplier::One, 0),
1661 )?;
1662 } else {
1663 self.assembler
1664 .emit_mov_imm(Location::GPR(tmp), (stack_offset as i64) as u64)?;
1665 self.assembler.emit_sub(
1666 Size::S64,
1667 Location::GPR(GPR::X29),
1668 Location::GPR(tmp),
1669 Location::GPR(tmp),
1670 )?;
1671 self.assembler
1672 .emit_str(Size::S64, location, Location::GPR(tmp))?;
1673 }
1674 }
1675 match location {
1676 Location::GPR(x) => self.emit_unwind_op(UnwindOps::SaveRegister {
1677 reg: UnwindRegister::<GPR, NEON>::GPR(x),
1678 bp_neg_offset: stack_offset,
1679 }),
1680 Location::SIMD(x) => self.emit_unwind_op(UnwindOps::SaveRegister {
1681 reg: UnwindRegister::FPR(x),
1682 bp_neg_offset: stack_offset,
1683 }),
1684 _ => (),
1685 }
1686 Ok(())
1687 }
1688
1689 fn list_to_save(&self, _calling_convention: CallingConvention) -> Vec<Location> {
1690 vec![]
1691 }
1692
1693 fn get_param_registers(&self, _calling_convention: CallingConvention) -> &'static [Self::GPR] {
1694 &[
1695 GPR::X0,
1696 GPR::X1,
1697 GPR::X2,
1698 GPR::X3,
1699 GPR::X4,
1700 GPR::X5,
1701 GPR::X6,
1702 GPR::X7,
1703 ]
1704 }
1705
1706 fn get_param_location(
1707 &self,
1708 idx: usize,
1709 sz: Size,
1710 stack_args: &mut usize,
1711 calling_convention: CallingConvention,
1712 ) -> Location {
1713 let register_params = self.get_param_registers(calling_convention);
1714 match calling_convention {
1715 CallingConvention::AppleAarch64 => register_params.get(idx).map_or_else(
1716 || {
1717 let sz = sz.bytes() as usize;
1719 *stack_args = (*stack_args).next_multiple_of(sz);
1720 let loc = Location::Memory(GPR::XzrSp, *stack_args as i32);
1721 *stack_args += sz;
1722 loc
1723 },
1724 |reg| Location::GPR(*reg),
1725 ),
1726 _ => {
1727 if let Some(reg) = register_params.get(idx) {
1728 Location::GPR(*reg)
1729 } else {
1730 let loc = Location::Memory(GPR::XzrSp, *stack_args as i32);
1731 *stack_args += 8;
1732 loc
1733 }
1734 }
1735 }
1736 }
1737
1738 fn get_call_param_location(
1739 &self,
1740 return_slots: usize,
1741 idx: usize,
1742 sz: Size,
1743 stack_args: &mut usize,
1744 calling_convention: CallingConvention,
1745 ) -> Location {
1746 let register_params = self.get_param_registers(calling_convention);
1747 let return_values_memory_size =
1748 8 * return_slots.saturating_sub(ARM64_RETURN_VALUE_REGISTERS.len()) as i32;
1749
1750 match calling_convention {
1751 CallingConvention::AppleAarch64 => register_params.get(idx).map_or_else(
1752 || {
1753 let sz = sz.bytes() as usize;
1754 *stack_args = (*stack_args).next_multiple_of(sz);
1756 let loc = Location::Memory(
1757 GPR::X29,
1758 16 * 2 + return_values_memory_size + *stack_args as i32,
1759 );
1760 *stack_args += sz;
1761 loc
1762 },
1763 |reg| Location::GPR(*reg),
1764 ),
1765 _ => register_params.get(idx).map_or_else(
1766 || {
1767 let loc = Location::Memory(
1768 GPR::X29,
1769 16 * 2 + return_values_memory_size + *stack_args as i32,
1770 );
1771 *stack_args += 8;
1772 loc
1773 },
1774 |reg| Location::GPR(*reg),
1775 ),
1776 }
1777 }
1778
1779 fn get_simple_param_location(
1780 &self,
1781 idx: usize,
1782 calling_convention: CallingConvention,
1783 ) -> Self::GPR {
1784 self.get_param_registers(calling_convention)[idx]
1785 }
1786
1787 fn adjust_gpr_param_location(
1788 &mut self,
1789 _register: Self::GPR,
1790 _size: Size,
1791 ) -> Result<(), CompileError> {
1792 Ok(())
1793 }
1794
1795 fn get_return_value_location(
1796 &self,
1797 idx: usize,
1798 stack_location: &mut usize,
1799 _calling_convention: CallingConvention,
1800 ) -> AbstractLocation<Self::GPR, Self::SIMD> {
1801 ARM64_RETURN_VALUE_REGISTERS.get(idx).map_or_else(
1802 || {
1803 let loc = Location::Memory(GPR::XzrSp, *stack_location as i32);
1804 *stack_location += 8;
1805 loc
1806 },
1807 |reg| Location::GPR(*reg),
1808 )
1809 }
1810
1811 fn get_call_return_value_location(
1812 &self,
1813 idx: usize,
1814 _calling_convention: CallingConvention,
1815 ) -> AbstractLocation<Self::GPR, Self::SIMD> {
1816 ARM64_RETURN_VALUE_REGISTERS.get(idx).map_or_else(
1817 || {
1818 Location::Memory(
1819 GPR::X29,
1820 (16 * 2 + (idx - ARM64_RETURN_VALUE_REGISTERS.len()) * 8) as i32,
1821 )
1822 },
1823 |reg| Location::GPR(*reg),
1824 )
1825 }
1826
1827 fn move_location(
1828 &mut self,
1829 size: Size,
1830 source: Location,
1831 dest: Location,
1832 ) -> Result<(), CompileError> {
1833 match source {
1834 Location::GPR(_) | Location::SIMD(_) => match dest {
1835 Location::GPR(_) | Location::SIMD(_) => self.assembler.emit_mov(size, source, dest),
1836 Location::Memory(addr, offs) => {
1837 if self.offset_is_ok(size, offs) {
1838 self.assembler.emit_str(size, source, dest)
1839 } else if self.compatible_imm(offs as i64, ImmType::UnscaledOffset) {
1840 self.assembler.emit_stur(size, source, addr, offs)
1841 } else {
1842 let tmp = SCRATCH_REG;
1843 if offs < 0 {
1844 self.assembler
1845 .emit_mov_imm(Location::GPR(tmp), (-offs) as u64)?;
1846 self.assembler.emit_sub(
1847 Size::S64,
1848 Location::GPR(addr),
1849 Location::GPR(tmp),
1850 Location::GPR(tmp),
1851 )?;
1852 } else {
1853 self.assembler
1854 .emit_mov_imm(Location::GPR(tmp), offs as u64)?;
1855 self.assembler.emit_add(
1856 Size::S64,
1857 Location::GPR(addr),
1858 Location::GPR(tmp),
1859 Location::GPR(tmp),
1860 )?;
1861 }
1862 self.assembler
1863 .emit_str(size, source, Location::Memory(tmp, 0))
1864 }
1865 }
1866 _ => codegen_error!(
1867 "singlepass can't emit move_location {:?} {:?} => {:?}",
1868 size,
1869 source,
1870 dest
1871 ),
1872 },
1873 Location::Imm8(_) => match dest {
1874 Location::GPR(_) => self.assembler.emit_mov(size, source, dest),
1875 Location::Memory(_, _) => match size {
1876 Size::S64 => self.emit_relaxed_str64(source, dest),
1877 Size::S32 => self.emit_relaxed_str32(source, dest),
1878 Size::S16 => self.emit_relaxed_str16(source, dest),
1879 Size::S8 => self.emit_relaxed_str8(source, dest),
1880 },
1881 _ => codegen_error!(
1882 "singlepass can't emit move_location {:?} {:?} => {:?}",
1883 size,
1884 source,
1885 dest
1886 ),
1887 },
1888 Location::Imm32(val) => match dest {
1889 Location::GPR(_) => self.assembler.emit_mov_imm(dest, val as u64),
1890 Location::Memory(_, _) => match size {
1891 Size::S64 => self.emit_relaxed_str64(source, dest),
1892 Size::S32 => self.emit_relaxed_str32(source, dest),
1893 Size::S16 => self.emit_relaxed_str16(source, dest),
1894 Size::S8 => self.emit_relaxed_str8(source, dest),
1895 },
1896 _ => codegen_error!(
1897 "singlepass can't emit move_location {:?} {:?} => {:?}",
1898 size,
1899 source,
1900 dest
1901 ),
1902 },
1903 Location::Imm64(val) => match dest {
1904 Location::GPR(_) => self.assembler.emit_mov_imm(dest, val),
1905 Location::Memory(_, _) => match size {
1906 Size::S64 => self.emit_relaxed_str64(source, dest),
1907 Size::S32 => self.emit_relaxed_str32(source, dest),
1908 Size::S16 => self.emit_relaxed_str16(source, dest),
1909 Size::S8 => self.emit_relaxed_str8(source, dest),
1910 },
1911 _ => codegen_error!(
1912 "singlepass can't emit move_location {:?} {:?} => {:?}",
1913 size,
1914 source,
1915 dest
1916 ),
1917 },
1918 Location::Memory(addr, offs) => match dest {
1919 Location::GPR(_) | Location::SIMD(_) => {
1920 if self.offset_is_ok(size, offs) {
1921 self.assembler.emit_ldr(size, dest, source)
1922 } else if offs > -256 && offs < 256 {
1923 self.assembler.emit_ldur(size, dest, addr, offs)
1924 } else {
1925 let tmp = SCRATCH_REG;
1926 if offs < 0 {
1927 self.assembler
1928 .emit_mov_imm(Location::GPR(tmp), (-offs) as u64)?;
1929 self.assembler.emit_sub(
1930 Size::S64,
1931 Location::GPR(addr),
1932 Location::GPR(tmp),
1933 Location::GPR(tmp),
1934 )?;
1935 } else {
1936 self.assembler
1937 .emit_mov_imm(Location::GPR(tmp), offs as u64)?;
1938 self.assembler.emit_add(
1939 Size::S64,
1940 Location::GPR(addr),
1941 Location::GPR(tmp),
1942 Location::GPR(tmp),
1943 )?;
1944 }
1945 self.assembler
1946 .emit_ldr(size, dest, Location::Memory(tmp, 0))
1947 }
1948 }
1949 _ => {
1950 let mut temps = vec![];
1951 let src =
1952 self.location_to_reg(size, source, &mut temps, ImmType::None, true, None)?;
1953 self.move_location(size, src, dest)?;
1954 for r in temps {
1955 self.release_gpr(r);
1956 }
1957 Ok(())
1958 }
1959 },
1960 _ => codegen_error!(
1961 "singlepass can't emit move_location {:?} {:?} => {:?}",
1962 size,
1963 source,
1964 dest
1965 ),
1966 }
1967 }
1968
1969 fn move_location_extend(
1970 &mut self,
1971 size_val: Size,
1972 signed: bool,
1973 source: Location,
1974 size_op: Size,
1975 dest: Location,
1976 ) -> Result<(), CompileError> {
1977 if size_op != Size::S64 {
1978 codegen_error!("singlepass move_location_extend unreachable");
1979 }
1980 let mut temps = vec![];
1981 let dst = self.location_to_reg(size_op, dest, &mut temps, ImmType::None, false, None)?;
1982 let src = match (size_val, signed, source) {
1983 (Size::S64, _, _) => source,
1984 (Size::S32, false, Location::GPR(_)) => {
1985 self.assembler.emit_mov(size_val, source, dst)?;
1986 dst
1987 }
1988 (Size::S8, false, Location::GPR(_)) => {
1989 self.assembler.emit_uxtb(size_op, source, dst)?;
1990 dst
1991 }
1992 (Size::S16, false, Location::GPR(_)) => {
1993 self.assembler.emit_uxth(size_op, source, dst)?;
1994 dst
1995 }
1996 (Size::S8, true, Location::GPR(_)) => {
1997 self.assembler.emit_sxtb(size_op, source, dst)?;
1998 dst
1999 }
2000 (Size::S16, true, Location::GPR(_)) => {
2001 self.assembler.emit_sxth(size_op, source, dst)?;
2002 dst
2003 }
2004 (Size::S32, true, Location::GPR(_)) => {
2005 self.assembler.emit_sxtw(size_op, source, dst)?;
2006 dst
2007 }
2008 (Size::S32, false, Location::Memory(_, _)) => {
2009 self.emit_relaxed_ldr32(size_op, dst, source)?;
2010 dst
2011 }
2012 (Size::S32, true, Location::Memory(_, _)) => {
2013 self.emit_relaxed_ldr32s(size_op, dst, source)?;
2014 dst
2015 }
2016 (Size::S16, false, Location::Memory(_, _)) => {
2017 self.emit_relaxed_ldr16(size_op, dst, source)?;
2018 dst
2019 }
2020 (Size::S16, true, Location::Memory(_, _)) => {
2021 self.emit_relaxed_ldr16s(size_op, dst, source)?;
2022 dst
2023 }
2024 (Size::S8, false, Location::Memory(_, _)) => {
2025 self.emit_relaxed_ldr8(size_op, dst, source)?;
2026 dst
2027 }
2028 (Size::S8, true, Location::Memory(_, _)) => {
2029 self.emit_relaxed_ldr8s(size_op, dst, source)?;
2030 dst
2031 }
2032 _ => codegen_error!(
2033 "singlepass can't emit move_location_extend {:?} {:?} {:?} => {:?} {:?}",
2034 size_val,
2035 signed,
2036 source,
2037 size_op,
2038 dest
2039 ),
2040 };
2041 if src != dst {
2042 self.move_location(size_op, src, dst)?;
2043 }
2044 if dst != dest {
2045 self.move_location(size_op, dst, dest)?;
2046 }
2047 for r in temps {
2048 self.release_gpr(r);
2049 }
2050 Ok(())
2051 }
2052
2053 fn init_stack_loc(
2054 &mut self,
2055 init_stack_loc_cnt: u64,
2056 last_stack_loc: Location,
2057 ) -> Result<(), CompileError> {
2058 let label = self.assembler.get_label();
2059 let mut temps = vec![];
2060 let dest = self.acquire_temp_gpr().ok_or_else(|| {
2061 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2062 })?;
2063 temps.push(dest);
2064 let cnt = self.location_to_reg(
2065 Size::S64,
2066 Location::Imm64(init_stack_loc_cnt),
2067 &mut temps,
2068 ImmType::None,
2069 true,
2070 None,
2071 )?;
2072 let dest = match last_stack_loc {
2073 Location::GPR(_) => codegen_error!("singlepass init_stack_loc unreachable"),
2074 Location::SIMD(_) => codegen_error!("singlepass init_stack_loc unreachable"),
2075 Location::Memory(reg, offset) => {
2076 if offset < 0 {
2077 let offset = (-offset) as u32;
2078 if self.compatible_imm(offset as i64, ImmType::Bits12) {
2079 self.assembler.emit_sub(
2080 Size::S64,
2081 Location::GPR(reg),
2082 Location::Imm32(offset),
2083 Location::GPR(dest),
2084 )?;
2085 } else {
2086 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2087 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2088 })?;
2089 self.assembler
2090 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
2091 self.assembler.emit_sub(
2092 Size::S64,
2093 Location::GPR(reg),
2094 Location::GPR(tmp),
2095 Location::GPR(dest),
2096 )?;
2097 temps.push(tmp);
2098 }
2099 dest
2100 } else {
2101 let offset = offset as u32;
2102 if self.compatible_imm(offset as i64, ImmType::Bits12) {
2103 self.assembler.emit_add(
2104 Size::S64,
2105 Location::GPR(reg),
2106 Location::Imm32(offset),
2107 Location::GPR(dest),
2108 )?;
2109 } else {
2110 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2111 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2112 })?;
2113 self.assembler
2114 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
2115 self.assembler.emit_add(
2116 Size::S64,
2117 Location::GPR(reg),
2118 Location::GPR(tmp),
2119 Location::GPR(dest),
2120 )?;
2121 temps.push(tmp);
2122 }
2123 dest
2124 }
2125 }
2126 _ => codegen_error!("singlepass can't emit init_stack_loc {:?}", last_stack_loc),
2127 };
2128 self.assembler.emit_label(label)?;
2129 self.assembler
2130 .emit_stria(Size::S64, Location::GPR(GPR::XzrSp), dest, 8)?;
2131 self.assembler
2132 .emit_sub(Size::S64, cnt, Location::Imm8(1), cnt)?;
2133 self.assembler.emit_cbnz_label(Size::S64, cnt, label)?;
2134 for r in temps {
2135 self.release_gpr(r);
2136 }
2137 Ok(())
2138 }
2139
2140 fn restore_saved_area(&mut self, saved_area_offset: i32) -> Result<(), CompileError> {
2141 let real_delta = if saved_area_offset & 15 != 0 {
2142 self.pushed = true;
2143 saved_area_offset + 8
2144 } else {
2145 self.pushed = false;
2146 saved_area_offset
2147 };
2148 if self.compatible_imm(real_delta as _, ImmType::Bits12) {
2149 self.assembler.emit_sub(
2150 Size::S64,
2151 Location::GPR(GPR::X29),
2152 Location::Imm32(real_delta as _),
2153 Location::GPR(GPR::XzrSp),
2154 )?;
2155 } else {
2156 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2157 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2158 })?;
2159 self.assembler
2160 .emit_mov_imm(Location::GPR(tmp), real_delta as u64)?;
2161 self.assembler.emit_sub(
2162 Size::S64,
2163 Location::GPR(GPR::X29),
2164 Location::GPR(tmp),
2165 Location::GPR(GPR::XzrSp),
2166 )?;
2167 self.release_gpr(tmp);
2168 }
2169 Ok(())
2170 }
2171
2172 fn pop_location(&mut self, location: Location) -> Result<(), CompileError> {
2173 self.emit_pop(Size::S64, location)
2174 }
2175
2176 fn assembler_finalize(
2177 self,
2178 assembly_comments: HashMap<usize, AssemblyComment>,
2179 ) -> Result<FinalizedAssembly, CompileError> {
2180 Ok(FinalizedAssembly {
2181 body: self.assembler.finalize().map_err(|e| {
2182 CompileError::Codegen(format!("Assembler failed finalization with: {e:?}"))
2183 })?,
2184 assembly_comments,
2185 })
2186 }
2187
2188 fn get_offset(&self) -> Offset {
2189 self.assembler.get_offset()
2190 }
2191
2192 fn finalize_function(&mut self) -> Result<(), CompileError> {
2193 self.assembler.finalize_function();
2194 Ok(())
2195 }
2196
2197 fn emit_function_prolog(&mut self) -> Result<(), CompileError> {
2198 self.emit_double_push(Size::S64, Location::GPR(GPR::X29), Location::GPR(GPR::X30))?; self.emit_unwind_op(UnwindOps::Push2Regs {
2200 reg1: UnwindRegister::GPR(GPR::X29),
2201 reg2: UnwindRegister::GPR(GPR::X30),
2202 up_to_sp: 16,
2203 });
2204 self.emit_double_push(Size::S64, Location::GPR(GPR::X27), Location::GPR(GPR::X28))?;
2205 self.emit_unwind_op(UnwindOps::Push2Regs {
2206 reg1: UnwindRegister::GPR(GPR::X27),
2207 reg2: UnwindRegister::GPR(GPR::X28),
2208 up_to_sp: 32,
2209 });
2210 self.assembler.emit_add(
2212 Size::S64,
2213 Location::GPR(GPR::XzrSp),
2214 Location::Imm8(0),
2215 Location::GPR(GPR::X29),
2216 )?;
2217 self.emit_unwind_op(UnwindOps::DefineNewFrame);
2218 Ok(())
2219 }
2220
2221 fn emit_function_epilog(&mut self) -> Result<(), CompileError> {
2222 self.assembler.emit_add(
2224 Size::S64,
2225 Location::GPR(GPR::X29),
2226 Location::Imm8(0),
2227 Location::GPR(GPR::XzrSp),
2228 )?;
2229 self.pushed = false; self.emit_double_pop(Size::S64, Location::GPR(GPR::X27), Location::GPR(GPR::X28))?;
2231 self.emit_double_pop(Size::S64, Location::GPR(GPR::X29), Location::GPR(GPR::X30))?;
2232 Ok(())
2233 }
2234
2235 fn emit_function_return_float(&mut self) -> Result<(), CompileError> {
2236 self.assembler
2237 .emit_mov(Size::S64, Location::GPR(GPR::X0), Location::SIMD(NEON::V0))
2238 }
2239
2240 fn canonicalize_nan(
2241 &mut self,
2242 sz: Size,
2243 input: Location,
2244 output: Location,
2245 ) -> Result<(), CompileError> {
2246 let mut tempn = vec![];
2247 let mut temps = vec![];
2248 let old_fpcr = self.set_default_nan(&mut temps)?;
2249 match (sz, input, output) {
2251 (Size::S32, Location::SIMD(_), Location::SIMD(_)) => {
2252 self.assembler.emit_fmax(sz, input, input, output)?;
2253 }
2254 (Size::S64, Location::SIMD(_), Location::SIMD(_)) => {
2255 self.assembler.emit_fmax(sz, input, input, output)?;
2256 }
2257 (Size::S32, Location::SIMD(_), _) | (Size::S64, Location::SIMD(_), _) => {
2258 let tmp = self.location_to_neon(sz, output, &mut tempn, ImmType::None, false)?;
2259 self.assembler.emit_fmax(sz, input, input, tmp)?;
2260 self.move_location(sz, tmp, output)?;
2261 }
2262 (Size::S32, Location::Memory(_, _), _) | (Size::S64, Location::Memory(_, _), _) => {
2263 let src = self.location_to_neon(sz, input, &mut tempn, ImmType::None, true)?;
2264 let tmp = self.location_to_neon(sz, output, &mut tempn, ImmType::None, false)?;
2265 self.assembler.emit_fmax(sz, src, src, tmp)?;
2266 if tmp != output {
2267 self.move_location(sz, tmp, output)?;
2268 }
2269 }
2270 _ => codegen_error!(
2271 "singlepass can't emit canonicalize_nan {:?} {:?} {:?}",
2272 sz,
2273 input,
2274 output
2275 ),
2276 }
2277
2278 self.restore_fpcr(old_fpcr)?;
2279 for r in temps {
2280 self.release_gpr(r);
2281 }
2282 for r in tempn {
2283 self.release_simd(r);
2284 }
2285 Ok(())
2286 }
2287
2288 fn emit_illegal_op(&mut self, trap: TrapCode) -> Result<(), CompileError> {
2289 let offset = self.assembler.get_offset().0;
2290 self.assembler.emit_udf(0xc0 | (trap as u8) as u16)?;
2291 self.mark_instruction_address_end(offset);
2292 Ok(())
2293 }
2294
2295 fn get_label(&mut self) -> Label {
2296 self.assembler.new_dynamic_label()
2297 }
2298
2299 fn emit_label(&mut self, label: Label) -> Result<(), CompileError> {
2300 self.assembler.emit_label(label)
2301 }
2302
2303 fn get_gpr_for_call(&self) -> GPR {
2304 GPR::X27
2305 }
2306
2307 fn emit_call_register(&mut self, reg: GPR) -> Result<(), CompileError> {
2308 self.assembler.emit_call_register(reg)
2309 }
2310
2311 fn emit_call_label(&mut self, label: Label) -> Result<(), CompileError> {
2312 self.assembler.emit_call_label(label)
2313 }
2314
2315 fn arch_emit_indirect_call_with_trampoline(
2316 &mut self,
2317 location: Location,
2318 ) -> Result<(), CompileError> {
2319 self.assembler
2320 .arch_emit_indirect_call_with_trampoline(location)
2321 }
2322
2323 fn emit_debug_breakpoint(&mut self) -> Result<(), CompileError> {
2324 self.assembler.emit_brk()
2325 }
2326
2327 fn emit_call_location(&mut self, location: Location) -> Result<(), CompileError> {
2328 let mut temps = vec![];
2329 let loc = self.location_to_reg(
2330 Size::S64,
2331 location,
2332 &mut temps,
2333 ImmType::None,
2334 true,
2335 Some(self.get_gpr_for_call()),
2336 )?;
2337 match loc {
2338 Location::GPR(reg) => self.assembler.emit_call_register(reg),
2339 _ => codegen_error!("singlepass can't emit CALL Location"),
2340 }?;
2341 for r in temps {
2342 self.release_gpr(r);
2343 }
2344 Ok(())
2345 }
2346
2347 fn location_add(
2348 &mut self,
2349 size: Size,
2350 source: Location,
2351 dest: Location,
2352 flags: bool,
2353 ) -> Result<(), CompileError> {
2354 let mut temps = vec![];
2355 let src = self.location_to_reg(size, source, &mut temps, ImmType::Bits12, true, None)?;
2356 let dst = self.location_to_reg(size, dest, &mut temps, ImmType::None, true, None)?;
2357 if flags {
2358 self.assembler.emit_adds(size, dst, src, dst)?;
2359 } else {
2360 self.assembler.emit_add(size, dst, src, dst)?;
2361 }
2362 if dst != dest {
2363 self.move_location(size, dst, dest)?;
2364 }
2365 for r in temps {
2366 self.release_gpr(r);
2367 }
2368 Ok(())
2369 }
2370
2371 fn location_cmp(
2372 &mut self,
2373 size: Size,
2374 source: Location,
2375 dest: Location,
2376 ) -> Result<(), CompileError> {
2377 self.emit_relaxed_binop(Assembler::emit_cmp, size, source, dest, false)
2378 }
2379
2380 fn jmp_unconditional(&mut self, label: Label) -> Result<(), CompileError> {
2381 self.assembler.emit_b_label(label)
2382 }
2383
2384 fn jmp_on_condition(
2385 &mut self,
2386 cond: UnsignedCondition,
2387 size: Size,
2388 loc_a: AbstractLocation<Self::GPR, Self::SIMD>,
2389 loc_b: AbstractLocation<Self::GPR, Self::SIMD>,
2390 label: Label,
2391 ) -> Result<(), CompileError> {
2392 self.emit_relaxed_binop(Assembler::emit_cmp, size, loc_b, loc_a, false)?;
2393 let cond = match cond {
2394 UnsignedCondition::Equal => Condition::Eq,
2395 UnsignedCondition::NotEqual => Condition::Ne,
2396 UnsignedCondition::Above => Condition::Hi,
2397 UnsignedCondition::AboveEqual => Condition::Cs,
2398 UnsignedCondition::Below => Condition::Cc,
2399 UnsignedCondition::BelowEqual => Condition::Ls,
2400 };
2401 self.assembler.emit_bcond_label_far(cond, label)
2402 }
2403
2404 fn emit_jmp_to_jumptable(&mut self, label: Label, cond: Location) -> Result<(), CompileError> {
2405 let tmp1 = self.acquire_temp_gpr().ok_or_else(|| {
2406 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2407 })?;
2408 let tmp2 = self.acquire_temp_gpr().ok_or_else(|| {
2409 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2410 })?;
2411
2412 self.assembler.emit_load_label(tmp1, label)?;
2413 self.move_location(Size::S32, cond, Location::GPR(tmp2))?;
2414
2415 self.assembler.emit_add_lsl(
2416 Size::S64,
2417 Location::GPR(tmp1),
2418 Location::GPR(tmp2),
2419 2,
2420 Location::GPR(tmp2),
2421 )?;
2422 self.assembler.emit_b_register(tmp2)?;
2423 self.release_gpr(tmp2);
2424 self.release_gpr(tmp1);
2425 Ok(())
2426 }
2427
2428 fn align_for_loop(&mut self) -> Result<(), CompileError> {
2429 Ok(())
2431 }
2432
2433 fn emit_ret(&mut self) -> Result<(), CompileError> {
2434 self.assembler.emit_ret()
2435 }
2436
2437 fn emit_push(&mut self, size: Size, loc: Location) -> Result<(), CompileError> {
2438 self.emit_push(size, loc)
2439 }
2440
2441 fn emit_pop(&mut self, size: Size, loc: Location) -> Result<(), CompileError> {
2442 self.emit_pop(size, loc)
2443 }
2444
2445 fn emit_memory_fence(&mut self) -> Result<(), CompileError> {
2446 self.assembler.emit_dmb()
2447 }
2448
2449 fn emit_imul_imm32(&mut self, size: Size, imm32: u32, gpr: GPR) -> Result<(), CompileError> {
2450 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2451 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2452 })?;
2453 self.assembler
2454 .emit_mov_imm(Location::GPR(tmp), imm32 as u64)?;
2455 self.assembler.emit_mul(
2456 size,
2457 Location::GPR(gpr),
2458 Location::GPR(tmp),
2459 Location::GPR(gpr),
2460 )?;
2461 self.release_gpr(tmp);
2462 Ok(())
2463 }
2464
2465 fn emit_relaxed_mov(
2466 &mut self,
2467 sz: Size,
2468 src: Location,
2469 dst: Location,
2470 ) -> Result<(), CompileError> {
2471 self.emit_relaxed_binop(Assembler::emit_mov, sz, src, dst, true)
2472 }
2473
2474 fn emit_relaxed_cmp(
2475 &mut self,
2476 sz: Size,
2477 src: Location,
2478 dst: Location,
2479 ) -> Result<(), CompileError> {
2480 self.emit_relaxed_binop(Assembler::emit_cmp, sz, src, dst, false)
2481 }
2482
2483 fn emit_relaxed_sign_extension(
2484 &mut self,
2485 sz_src: Size,
2486 src: Location,
2487 sz_dst: Size,
2488 dst: Location,
2489 ) -> Result<(), CompileError> {
2490 match (src, dst) {
2491 (Location::Memory(_, _), Location::GPR(_)) => match sz_src {
2492 Size::S8 => self.emit_relaxed_ldr8s(sz_dst, dst, src),
2493 Size::S16 => self.emit_relaxed_ldr16s(sz_dst, dst, src),
2494 Size::S32 => self.emit_relaxed_ldr32s(sz_dst, dst, src),
2495 _ => codegen_error!("singlepass emit_relaxed_sign_extension unreachable"),
2496 },
2497 _ => {
2498 let mut temps = vec![];
2499 let src =
2500 self.location_to_reg(sz_src, src, &mut temps, ImmType::None, true, None)?;
2501 let dest =
2502 self.location_to_reg(sz_dst, dst, &mut temps, ImmType::None, false, None)?;
2503 match sz_src {
2504 Size::S8 => self.assembler.emit_sxtb(sz_dst, src, dest),
2505 Size::S16 => self.assembler.emit_sxth(sz_dst, src, dest),
2506 Size::S32 => self.assembler.emit_sxtw(sz_dst, src, dest),
2507 _ => codegen_error!("singlepass emit_relaxed_sign_extension unreachable"),
2508 }?;
2509 if dst != dest {
2510 self.move_location(sz_dst, dest, dst)?;
2511 }
2512 for r in temps {
2513 self.release_gpr(r);
2514 }
2515 Ok(())
2516 }
2517 }
2518 }
2519
2520 fn emit_binop_add32(
2521 &mut self,
2522 loc_a: Location,
2523 loc_b: Location,
2524 ret: Location,
2525 ) -> Result<(), CompileError> {
2526 self.emit_relaxed_binop3(
2527 Assembler::emit_add,
2528 Size::S32,
2529 loc_a,
2530 loc_b,
2531 ret,
2532 ImmType::Bits12,
2533 )
2534 }
2535
2536 fn emit_binop_sub32(
2537 &mut self,
2538 loc_a: Location,
2539 loc_b: Location,
2540 ret: Location,
2541 ) -> Result<(), CompileError> {
2542 self.emit_relaxed_binop3(
2543 Assembler::emit_sub,
2544 Size::S32,
2545 loc_a,
2546 loc_b,
2547 ret,
2548 ImmType::Bits12,
2549 )
2550 }
2551
2552 fn emit_binop_mul32(
2553 &mut self,
2554 loc_a: Location,
2555 loc_b: Location,
2556 ret: Location,
2557 ) -> Result<(), CompileError> {
2558 self.emit_relaxed_binop3(
2559 Assembler::emit_mul,
2560 Size::S32,
2561 loc_a,
2562 loc_b,
2563 ret,
2564 ImmType::None,
2565 )
2566 }
2567
2568 fn emit_binop_udiv32(
2569 &mut self,
2570 loc_a: Location,
2571 loc_b: Location,
2572 ret: Location,
2573 integer_division_by_zero: Label,
2574 ) -> Result<usize, CompileError> {
2575 let mut temps = vec![];
2576 let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
2577 let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
2578 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
2579
2580 self.assembler
2581 .emit_cbz_label_far(Size::S32, src2, integer_division_by_zero)?;
2582 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
2583 self.assembler.emit_udiv(Size::S32, src1, src2, dest)?;
2584 if ret != dest {
2585 self.move_location(Size::S32, dest, ret)?;
2586 }
2587 for r in temps {
2588 self.release_gpr(r);
2589 }
2590 Ok(offset)
2591 }
2592
2593 fn emit_binop_sdiv32(
2594 &mut self,
2595 loc_a: Location,
2596 loc_b: Location,
2597 ret: Location,
2598 integer_division_by_zero: Label,
2599 integer_overflow: Label,
2600 ) -> Result<usize, CompileError> {
2601 let mut temps = vec![];
2602 let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
2603 let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
2604 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
2605
2606 self.assembler
2607 .emit_cbz_label_far(Size::S32, src2, integer_division_by_zero)?;
2608 let label_nooverflow = self.assembler.get_label();
2609 let tmp = self.location_to_reg(
2610 Size::S32,
2611 Location::Imm32(0x80000000),
2612 &mut temps,
2613 ImmType::None,
2614 true,
2615 None,
2616 )?;
2617 self.assembler.emit_cmp(Size::S32, tmp, src1)?;
2618 self.assembler
2619 .emit_bcond_label(Condition::Ne, label_nooverflow)?;
2620 self.assembler.emit_movn(Size::S32, tmp, 0)?;
2621 self.assembler.emit_cmp(Size::S32, tmp, src2)?;
2622 self.assembler
2623 .emit_bcond_label_far(Condition::Eq, integer_overflow)?;
2624 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
2625 self.assembler.emit_label(label_nooverflow)?;
2626 self.assembler.emit_sdiv(Size::S32, src1, src2, dest)?;
2627 if ret != dest {
2628 self.move_location(Size::S32, dest, ret)?;
2629 }
2630 for r in temps {
2631 self.release_gpr(r);
2632 }
2633 Ok(offset)
2634 }
2635
2636 fn emit_binop_urem32(
2637 &mut self,
2638 loc_a: Location,
2639 loc_b: Location,
2640 ret: Location,
2641 integer_division_by_zero: Label,
2642 ) -> Result<usize, CompileError> {
2643 let mut temps = vec![];
2644 let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
2645 let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
2646 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
2647 let dest = if dest == src1 || dest == src2 {
2648 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2649 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2650 })?;
2651 temps.push(tmp);
2652 self.assembler
2653 .emit_mov(Size::S32, dest, Location::GPR(tmp))?;
2654 Location::GPR(tmp)
2655 } else {
2656 dest
2657 };
2658 self.assembler
2659 .emit_cbz_label_far(Size::S32, src2, integer_division_by_zero)?;
2660 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
2661 self.assembler.emit_udiv(Size::S32, src1, src2, dest)?;
2662 self.assembler
2664 .emit_msub(Size::S32, dest, src2, src1, dest)?;
2665 if ret != dest {
2666 self.move_location(Size::S32, dest, ret)?;
2667 }
2668 for r in temps {
2669 self.release_gpr(r);
2670 }
2671 Ok(offset)
2672 }
2673
2674 fn emit_binop_srem32(
2675 &mut self,
2676 loc_a: Location,
2677 loc_b: Location,
2678 ret: Location,
2679 integer_division_by_zero: Label,
2680 ) -> Result<usize, CompileError> {
2681 let mut temps = vec![];
2682 let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
2683 let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
2684 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
2685 let dest = if dest == src1 || dest == src2 {
2686 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2687 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2688 })?;
2689 temps.push(tmp);
2690 self.assembler
2691 .emit_mov(Size::S32, dest, Location::GPR(tmp))?;
2692 Location::GPR(tmp)
2693 } else {
2694 dest
2695 };
2696 self.assembler
2697 .emit_cbz_label_far(Size::S32, src2, integer_division_by_zero)?;
2698 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
2699 self.assembler.emit_sdiv(Size::S32, src1, src2, dest)?;
2700 self.assembler
2702 .emit_msub(Size::S32, dest, src2, src1, dest)?;
2703 if ret != dest {
2704 self.move_location(Size::S32, dest, ret)?;
2705 }
2706 for r in temps {
2707 self.release_gpr(r);
2708 }
2709 Ok(offset)
2710 }
2711
2712 fn emit_binop_and32(
2713 &mut self,
2714 loc_a: Location,
2715 loc_b: Location,
2716 ret: Location,
2717 ) -> Result<(), CompileError> {
2718 self.emit_relaxed_binop3(
2719 Assembler::emit_and,
2720 Size::S32,
2721 loc_a,
2722 loc_b,
2723 ret,
2724 ImmType::Logical32,
2725 )
2726 }
2727
2728 fn emit_binop_or32(
2729 &mut self,
2730 loc_a: Location,
2731 loc_b: Location,
2732 ret: Location,
2733 ) -> Result<(), CompileError> {
2734 self.emit_relaxed_binop3(
2735 Assembler::emit_or,
2736 Size::S32,
2737 loc_a,
2738 loc_b,
2739 ret,
2740 ImmType::Logical32,
2741 )
2742 }
2743
2744 fn emit_binop_xor32(
2745 &mut self,
2746 loc_a: Location,
2747 loc_b: Location,
2748 ret: Location,
2749 ) -> Result<(), CompileError> {
2750 self.emit_relaxed_binop3(
2751 Assembler::emit_eor,
2752 Size::S32,
2753 loc_a,
2754 loc_b,
2755 ret,
2756 ImmType::Logical32,
2757 )
2758 }
2759
2760 fn i32_cmp_ge_s(
2761 &mut self,
2762 loc_a: Location,
2763 loc_b: Location,
2764 ret: Location,
2765 ) -> Result<(), CompileError> {
2766 self.emit_cmpop_i32_dynamic_b(Condition::Ge, loc_a, loc_b, ret)
2767 }
2768
2769 fn i32_cmp_gt_s(
2770 &mut self,
2771 loc_a: Location,
2772 loc_b: Location,
2773 ret: Location,
2774 ) -> Result<(), CompileError> {
2775 self.emit_cmpop_i32_dynamic_b(Condition::Gt, loc_a, loc_b, ret)
2776 }
2777
2778 fn i32_cmp_le_s(
2779 &mut self,
2780 loc_a: Location,
2781 loc_b: Location,
2782 ret: Location,
2783 ) -> Result<(), CompileError> {
2784 self.emit_cmpop_i32_dynamic_b(Condition::Le, loc_a, loc_b, ret)
2785 }
2786
2787 fn i32_cmp_lt_s(
2788 &mut self,
2789 loc_a: Location,
2790 loc_b: Location,
2791 ret: Location,
2792 ) -> Result<(), CompileError> {
2793 self.emit_cmpop_i32_dynamic_b(Condition::Lt, loc_a, loc_b, ret)
2794 }
2795
2796 fn i32_cmp_ge_u(
2797 &mut self,
2798 loc_a: Location,
2799 loc_b: Location,
2800 ret: Location,
2801 ) -> Result<(), CompileError> {
2802 self.emit_cmpop_i32_dynamic_b(Condition::Cs, loc_a, loc_b, ret)
2803 }
2804
2805 fn i32_cmp_gt_u(
2806 &mut self,
2807 loc_a: Location,
2808 loc_b: Location,
2809 ret: Location,
2810 ) -> Result<(), CompileError> {
2811 self.emit_cmpop_i32_dynamic_b(Condition::Hi, loc_a, loc_b, ret)
2812 }
2813
2814 fn i32_cmp_le_u(
2815 &mut self,
2816 loc_a: Location,
2817 loc_b: Location,
2818 ret: Location,
2819 ) -> Result<(), CompileError> {
2820 self.emit_cmpop_i32_dynamic_b(Condition::Ls, loc_a, loc_b, ret)
2821 }
2822
2823 fn i32_cmp_lt_u(
2824 &mut self,
2825 loc_a: Location,
2826 loc_b: Location,
2827 ret: Location,
2828 ) -> Result<(), CompileError> {
2829 self.emit_cmpop_i32_dynamic_b(Condition::Cc, loc_a, loc_b, ret)
2830 }
2831
2832 fn i32_cmp_ne(
2833 &mut self,
2834 loc_a: Location,
2835 loc_b: Location,
2836 ret: Location,
2837 ) -> Result<(), CompileError> {
2838 self.emit_cmpop_i32_dynamic_b(Condition::Ne, loc_a, loc_b, ret)
2839 }
2840
2841 fn i32_cmp_eq(
2842 &mut self,
2843 loc_a: Location,
2844 loc_b: Location,
2845 ret: Location,
2846 ) -> Result<(), CompileError> {
2847 self.emit_cmpop_i32_dynamic_b(Condition::Eq, loc_a, loc_b, ret)
2848 }
2849
2850 fn i32_clz(&mut self, src: Location, dst: Location) -> Result<(), CompileError> {
2851 self.emit_relaxed_binop(Assembler::emit_clz, Size::S32, src, dst, true)
2852 }
2853
2854 fn i32_ctz(&mut self, src: Location, dst: Location) -> Result<(), CompileError> {
2855 let mut temps = vec![];
2856 let src = self.location_to_reg(Size::S32, src, &mut temps, ImmType::None, true, None)?;
2857 let dest = self.location_to_reg(Size::S32, dst, &mut temps, ImmType::None, false, None)?;
2858 self.assembler.emit_rbit(Size::S32, src, dest)?;
2859 self.assembler.emit_clz(Size::S32, dest, dest)?;
2860 if dst != dest {
2861 self.move_location(Size::S32, dest, dst)?;
2862 }
2863 for r in temps {
2864 self.release_gpr(r);
2865 }
2866 Ok(())
2867 }
2868
2869 fn i32_popcnt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
2870 if self.has_neon {
2871 let mut temps = vec![];
2872
2873 let src_gpr =
2874 self.location_to_reg(Size::S32, loc, &mut temps, ImmType::None, true, None)?;
2875 let dst_gpr =
2876 self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
2877
2878 let mut neon_temps = vec![];
2879 let neon_temp = self.acquire_temp_simd().ok_or_else(|| {
2880 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2881 })?;
2882 neon_temps.push(neon_temp);
2883
2884 self.assembler
2885 .emit_fmov(Size::S32, src_gpr, Size::S32, Location::SIMD(neon_temp))?;
2886 self.assembler.emit_cnt(neon_temp, neon_temp)?;
2887 self.assembler.emit_addv(neon_temp, neon_temp)?;
2888 self.assembler
2889 .emit_fmov(Size::S32, Location::SIMD(neon_temp), Size::S32, dst_gpr)?;
2890
2891 if ret != dst_gpr {
2892 self.move_location(Size::S32, dst_gpr, ret)?;
2893 }
2894
2895 for r in temps {
2896 self.release_gpr(r);
2897 }
2898
2899 for r in neon_temps {
2900 self.release_simd(r);
2901 }
2902 } else {
2903 let mut temps = vec![];
2904 let src =
2905 self.location_to_reg(Size::S32, loc, &mut temps, ImmType::None, true, None)?;
2906 let dest =
2907 self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
2908 let src = if src == loc {
2909 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2910 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2911 })?;
2912 temps.push(tmp);
2913 self.assembler
2914 .emit_mov(Size::S32, src, Location::GPR(tmp))?;
2915 Location::GPR(tmp)
2916 } else {
2917 src
2918 };
2919 let tmp = {
2920 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2921 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2922 })?;
2923 temps.push(tmp);
2924 Location::GPR(tmp)
2925 };
2926 let label_loop = self.assembler.get_label();
2927 let label_exit = self.assembler.get_label();
2928 self.assembler
2929 .emit_mov(Size::S32, Location::GPR(GPR::XzrSp), dest)?; self.assembler.emit_cbz_label(Size::S32, src, label_exit)?; self.assembler.emit_label(label_loop)?; self.assembler
2933 .emit_add(Size::S32, dest, Location::Imm8(1), dest)?; self.assembler.emit_clz(Size::S32, src, tmp)?; self.assembler.emit_lsl(Size::S32, src, tmp, src)?; self.assembler
2937 .emit_lsl(Size::S32, src, Location::Imm8(1), src)?; self.assembler.emit_cbnz_label(Size::S32, src, label_loop)?; self.assembler.emit_label(label_exit)?;
2940 if ret != dest {
2941 self.move_location(Size::S32, dest, ret)?;
2942 }
2943 for r in temps {
2944 self.release_gpr(r);
2945 }
2946 }
2947 Ok(())
2948 }
2949
2950 fn i32_shl(
2951 &mut self,
2952 loc_a: Location,
2953 loc_b: Location,
2954 ret: Location,
2955 ) -> Result<(), CompileError> {
2956 self.emit_relaxed_binop3(
2957 Assembler::emit_lsl,
2958 Size::S32,
2959 loc_a,
2960 loc_b,
2961 ret,
2962 ImmType::Shift32No0,
2963 )
2964 }
2965
2966 fn i32_shr(
2967 &mut self,
2968 loc_a: Location,
2969 loc_b: Location,
2970 ret: Location,
2971 ) -> Result<(), CompileError> {
2972 self.emit_relaxed_binop3(
2973 Assembler::emit_lsr,
2974 Size::S32,
2975 loc_a,
2976 loc_b,
2977 ret,
2978 ImmType::Shift32No0,
2979 )
2980 }
2981
2982 fn i32_sar(
2983 &mut self,
2984 loc_a: Location,
2985 loc_b: Location,
2986 ret: Location,
2987 ) -> Result<(), CompileError> {
2988 self.emit_relaxed_binop3(
2989 Assembler::emit_asr,
2990 Size::S32,
2991 loc_a,
2992 loc_b,
2993 ret,
2994 ImmType::Shift32No0,
2995 )
2996 }
2997
2998 fn i32_rol(
2999 &mut self,
3000 loc_a: Location,
3001 loc_b: Location,
3002 ret: Location,
3003 ) -> Result<(), CompileError> {
3004 let mut temps = vec![];
3005 let src2 = match loc_b {
3006 Location::Imm8(imm) => Location::Imm8(32 - (imm & 31)),
3007 Location::Imm32(imm) => Location::Imm8(32 - (imm & 31) as u8),
3008 Location::Imm64(imm) => Location::Imm8(32 - (imm & 31) as u8),
3009 _ => {
3010 let tmp1 = self.location_to_reg(
3011 Size::S32,
3012 Location::Imm32(32),
3013 &mut temps,
3014 ImmType::None,
3015 true,
3016 None,
3017 )?;
3018 let tmp2 =
3019 self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
3020 self.assembler.emit_sub(Size::S32, tmp1, tmp2, tmp1)?;
3021 tmp1
3022 }
3023 };
3024 self.emit_relaxed_binop3(
3025 Assembler::emit_ror,
3026 Size::S32,
3027 loc_a,
3028 src2,
3029 ret,
3030 ImmType::Shift32No0,
3031 )?;
3032 for r in temps {
3033 self.release_gpr(r);
3034 }
3035 Ok(())
3036 }
3037
3038 fn i32_ror(
3039 &mut self,
3040 loc_a: Location,
3041 loc_b: Location,
3042 ret: Location,
3043 ) -> Result<(), CompileError> {
3044 self.emit_relaxed_binop3(
3045 Assembler::emit_ror,
3046 Size::S32,
3047 loc_a,
3048 loc_b,
3049 ret,
3050 ImmType::Shift32No0,
3051 )
3052 }
3053
3054 fn i32_load(
3055 &mut self,
3056 addr: Location,
3057 memarg: &MemArg,
3058 ret: Location,
3059 need_check: bool,
3060 imported_memories: bool,
3061 offset: i32,
3062 heap_access_oob: Label,
3063 unaligned_atomic: Label,
3064 ) -> Result<(), CompileError> {
3065 self.memory_op(
3066 addr,
3067 memarg,
3068 false,
3069 4,
3070 need_check,
3071 imported_memories,
3072 offset,
3073 heap_access_oob,
3074 unaligned_atomic,
3075 |this, addr| this.emit_relaxed_ldr32(Size::S32, ret, Location::Memory(addr, 0)),
3076 )
3077 }
3078
3079 fn i32_load_8u(
3080 &mut self,
3081 addr: Location,
3082 memarg: &MemArg,
3083 ret: Location,
3084 need_check: bool,
3085 imported_memories: bool,
3086 offset: i32,
3087 heap_access_oob: Label,
3088 unaligned_atomic: Label,
3089 ) -> Result<(), CompileError> {
3090 self.memory_op(
3091 addr,
3092 memarg,
3093 false,
3094 1,
3095 need_check,
3096 imported_memories,
3097 offset,
3098 heap_access_oob,
3099 unaligned_atomic,
3100 |this, addr| this.emit_relaxed_ldr8(Size::S32, ret, Location::Memory(addr, 0)),
3101 )
3102 }
3103
3104 fn i32_load_8s(
3105 &mut self,
3106 addr: Location,
3107 memarg: &MemArg,
3108 ret: Location,
3109 need_check: bool,
3110 imported_memories: bool,
3111 offset: i32,
3112 heap_access_oob: Label,
3113 unaligned_atomic: Label,
3114 ) -> Result<(), CompileError> {
3115 self.memory_op(
3116 addr,
3117 memarg,
3118 false,
3119 1,
3120 need_check,
3121 imported_memories,
3122 offset,
3123 heap_access_oob,
3124 unaligned_atomic,
3125 |this, addr| this.emit_relaxed_ldr8s(Size::S32, ret, Location::Memory(addr, 0)),
3126 )
3127 }
3128
3129 fn i32_load_16u(
3130 &mut self,
3131 addr: Location,
3132 memarg: &MemArg,
3133 ret: Location,
3134 need_check: bool,
3135 imported_memories: bool,
3136 offset: i32,
3137 heap_access_oob: Label,
3138 unaligned_atomic: Label,
3139 ) -> Result<(), CompileError> {
3140 self.memory_op(
3141 addr,
3142 memarg,
3143 false,
3144 2,
3145 need_check,
3146 imported_memories,
3147 offset,
3148 heap_access_oob,
3149 unaligned_atomic,
3150 |this, addr| this.emit_relaxed_ldr16(Size::S32, ret, Location::Memory(addr, 0)),
3151 )
3152 }
3153
3154 fn i32_load_16s(
3155 &mut self,
3156 addr: Location,
3157 memarg: &MemArg,
3158 ret: Location,
3159 need_check: bool,
3160 imported_memories: bool,
3161 offset: i32,
3162 heap_access_oob: Label,
3163 unaligned_atomic: Label,
3164 ) -> Result<(), CompileError> {
3165 self.memory_op(
3166 addr,
3167 memarg,
3168 false,
3169 2,
3170 need_check,
3171 imported_memories,
3172 offset,
3173 heap_access_oob,
3174 unaligned_atomic,
3175 |this, addr| this.emit_relaxed_ldr16s(Size::S32, ret, Location::Memory(addr, 0)),
3176 )
3177 }
3178
3179 fn i32_atomic_load(
3180 &mut self,
3181 addr: Location,
3182 memarg: &MemArg,
3183 ret: Location,
3184 need_check: bool,
3185 imported_memories: bool,
3186 offset: i32,
3187 heap_access_oob: Label,
3188 unaligned_atomic: Label,
3189 ) -> Result<(), CompileError> {
3190 self.memory_op(
3191 addr,
3192 memarg,
3193 true,
3194 4,
3195 need_check,
3196 imported_memories,
3197 offset,
3198 heap_access_oob,
3199 unaligned_atomic,
3200 |this, addr| this.emit_relaxed_ldr32(Size::S32, ret, Location::Memory(addr, 0)),
3201 )
3202 }
3203
3204 fn i32_atomic_load_8u(
3205 &mut self,
3206 addr: Location,
3207 memarg: &MemArg,
3208 ret: Location,
3209 need_check: bool,
3210 imported_memories: bool,
3211 offset: i32,
3212 heap_access_oob: Label,
3213 unaligned_atomic: Label,
3214 ) -> Result<(), CompileError> {
3215 self.memory_op(
3216 addr,
3217 memarg,
3218 true,
3219 1,
3220 need_check,
3221 imported_memories,
3222 offset,
3223 heap_access_oob,
3224 unaligned_atomic,
3225 |this, addr| this.emit_relaxed_ldr8(Size::S32, ret, Location::Memory(addr, 0)),
3226 )
3227 }
3228
3229 fn i32_atomic_load_16u(
3230 &mut self,
3231 addr: Location,
3232 memarg: &MemArg,
3233 ret: Location,
3234 need_check: bool,
3235 imported_memories: bool,
3236 offset: i32,
3237 heap_access_oob: Label,
3238 unaligned_atomic: Label,
3239 ) -> Result<(), CompileError> {
3240 self.memory_op(
3241 addr,
3242 memarg,
3243 true,
3244 2,
3245 need_check,
3246 imported_memories,
3247 offset,
3248 heap_access_oob,
3249 unaligned_atomic,
3250 |this, addr| this.emit_relaxed_ldr16(Size::S32, ret, Location::Memory(addr, 0)),
3251 )
3252 }
3253
3254 fn i32_save(
3255 &mut self,
3256 target_value: Location,
3257 memarg: &MemArg,
3258 target_addr: Location,
3259 need_check: bool,
3260 imported_memories: bool,
3261 offset: i32,
3262 heap_access_oob: Label,
3263 unaligned_atomic: Label,
3264 ) -> Result<(), CompileError> {
3265 self.memory_op(
3266 target_addr,
3267 memarg,
3268 false,
3269 4,
3270 need_check,
3271 imported_memories,
3272 offset,
3273 heap_access_oob,
3274 unaligned_atomic,
3275 |this, addr| this.emit_relaxed_str32(target_value, Location::Memory(addr, 0)),
3276 )
3277 }
3278
3279 fn i32_save_8(
3280 &mut self,
3281 target_value: Location,
3282 memarg: &MemArg,
3283 target_addr: Location,
3284 need_check: bool,
3285 imported_memories: bool,
3286 offset: i32,
3287 heap_access_oob: Label,
3288 unaligned_atomic: Label,
3289 ) -> Result<(), CompileError> {
3290 self.memory_op(
3291 target_addr,
3292 memarg,
3293 false,
3294 4,
3295 need_check,
3296 imported_memories,
3297 offset,
3298 heap_access_oob,
3299 unaligned_atomic,
3300 |this, addr| this.emit_relaxed_str8(target_value, Location::Memory(addr, 0)),
3301 )
3302 }
3303
3304 fn i32_save_16(
3305 &mut self,
3306 target_value: Location,
3307 memarg: &MemArg,
3308 target_addr: Location,
3309 need_check: bool,
3310 imported_memories: bool,
3311 offset: i32,
3312 heap_access_oob: Label,
3313 unaligned_atomic: Label,
3314 ) -> Result<(), CompileError> {
3315 self.memory_op(
3316 target_addr,
3317 memarg,
3318 false,
3319 4,
3320 need_check,
3321 imported_memories,
3322 offset,
3323 heap_access_oob,
3324 unaligned_atomic,
3325 |this, addr| this.emit_relaxed_str16(target_value, Location::Memory(addr, 0)),
3326 )
3327 }
3328
3329 fn i32_atomic_save(
3330 &mut self,
3331 target_value: Location,
3332 memarg: &MemArg,
3333 target_addr: Location,
3334 need_check: bool,
3335 imported_memories: bool,
3336 offset: i32,
3337 heap_access_oob: Label,
3338 unaligned_atomic: Label,
3339 ) -> Result<(), CompileError> {
3340 self.memory_op(
3341 target_addr,
3342 memarg,
3343 true,
3344 4,
3345 need_check,
3346 imported_memories,
3347 offset,
3348 heap_access_oob,
3349 unaligned_atomic,
3350 |this, addr| this.emit_relaxed_str32(target_value, Location::Memory(addr, 0)),
3351 )?;
3352 self.assembler.emit_dmb()
3353 }
3354
3355 fn i32_atomic_save_8(
3356 &mut self,
3357 target_value: Location,
3358 memarg: &MemArg,
3359 target_addr: Location,
3360 need_check: bool,
3361 imported_memories: bool,
3362 offset: i32,
3363 heap_access_oob: Label,
3364 unaligned_atomic: Label,
3365 ) -> Result<(), CompileError> {
3366 self.memory_op(
3367 target_addr,
3368 memarg,
3369 true,
3370 1,
3371 need_check,
3372 imported_memories,
3373 offset,
3374 heap_access_oob,
3375 unaligned_atomic,
3376 |this, addr| this.emit_relaxed_str8(target_value, Location::Memory(addr, 0)),
3377 )?;
3378 self.assembler.emit_dmb()
3379 }
3380
3381 fn i32_atomic_save_16(
3382 &mut self,
3383 target_value: Location,
3384 memarg: &MemArg,
3385 target_addr: Location,
3386 need_check: bool,
3387 imported_memories: bool,
3388 offset: i32,
3389 heap_access_oob: Label,
3390 unaligned_atomic: Label,
3391 ) -> Result<(), CompileError> {
3392 self.memory_op(
3393 target_addr,
3394 memarg,
3395 true,
3396 2,
3397 need_check,
3398 imported_memories,
3399 offset,
3400 heap_access_oob,
3401 unaligned_atomic,
3402 |this, addr| this.emit_relaxed_str16(target_value, Location::Memory(addr, 0)),
3403 )?;
3404 self.assembler.emit_dmb()
3405 }
3406
3407 fn i32_atomic_add(
3408 &mut self,
3409 loc: Location,
3410 target: Location,
3411 memarg: &MemArg,
3412 ret: Location,
3413 need_check: bool,
3414 imported_memories: bool,
3415 offset: i32,
3416 heap_access_oob: Label,
3417 unaligned_atomic: Label,
3418 ) -> Result<(), CompileError> {
3419 self.memory_op(
3420 target,
3421 memarg,
3422 true,
3423 4,
3424 need_check,
3425 imported_memories,
3426 offset,
3427 heap_access_oob,
3428 unaligned_atomic,
3429 |this, addr| {
3430 let mut temps = vec![];
3431 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3432 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3433 })?;
3434 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3435 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3436 })?;
3437 let dst =
3438 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3439 let reread = this.get_label();
3440
3441 this.emit_label(reread)?;
3442 this.assembler
3443 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
3444 this.emit_binop_add32(dst, loc, Location::GPR(tmp1))?;
3445 this.assembler.emit_stlxr(
3446 Size::S32,
3447 Location::GPR(tmp2),
3448 Location::GPR(tmp1),
3449 Location::GPR(addr),
3450 )?;
3451 this.assembler
3452 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3453 this.assembler.emit_dmb()?;
3454
3455 if dst != ret {
3456 this.move_location(Size::S32, ret, dst)?;
3457 }
3458 for r in temps {
3459 this.release_gpr(r);
3460 }
3461 this.release_gpr(tmp1);
3462 this.release_gpr(tmp2);
3463 Ok(())
3464 },
3465 )
3466 }
3467
3468 fn i32_atomic_add_8u(
3469 &mut self,
3470 loc: Location,
3471 target: Location,
3472 memarg: &MemArg,
3473 ret: Location,
3474 need_check: bool,
3475 imported_memories: bool,
3476 offset: i32,
3477 heap_access_oob: Label,
3478 unaligned_atomic: Label,
3479 ) -> Result<(), CompileError> {
3480 self.memory_op(
3481 target,
3482 memarg,
3483 true,
3484 1,
3485 need_check,
3486 imported_memories,
3487 offset,
3488 heap_access_oob,
3489 unaligned_atomic,
3490 |this, addr| {
3491 let mut temps = vec![];
3492 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3493 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3494 })?;
3495 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3496 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3497 })?;
3498 let dst =
3499 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3500 let reread = this.get_label();
3501
3502 this.emit_label(reread)?;
3503 this.assembler
3504 .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?;
3505 this.emit_binop_add32(dst, loc, Location::GPR(tmp1))?;
3506 this.assembler.emit_stlxrb(
3507 Size::S32,
3508 Location::GPR(tmp2),
3509 Location::GPR(tmp1),
3510 Location::GPR(addr),
3511 )?;
3512 this.assembler
3513 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3514 this.assembler.emit_dmb()?;
3515
3516 if dst != ret {
3517 this.move_location(Size::S32, ret, dst)?;
3518 }
3519 for r in temps {
3520 this.release_gpr(r);
3521 }
3522 this.release_gpr(tmp1);
3523 this.release_gpr(tmp2);
3524 Ok(())
3525 },
3526 )
3527 }
3528
3529 fn i32_atomic_add_16u(
3530 &mut self,
3531 loc: Location,
3532 target: Location,
3533 memarg: &MemArg,
3534 ret: Location,
3535 need_check: bool,
3536 imported_memories: bool,
3537 offset: i32,
3538 heap_access_oob: Label,
3539 unaligned_atomic: Label,
3540 ) -> Result<(), CompileError> {
3541 self.memory_op(
3542 target,
3543 memarg,
3544 true,
3545 2,
3546 need_check,
3547 imported_memories,
3548 offset,
3549 heap_access_oob,
3550 unaligned_atomic,
3551 |this, addr| {
3552 let mut temps = vec![];
3553 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3554 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3555 })?;
3556 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3557 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3558 })?;
3559 let dst =
3560 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3561 let reread = this.get_label();
3562
3563 this.emit_label(reread)?;
3564 this.assembler
3565 .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?;
3566 this.emit_binop_add32(dst, loc, Location::GPR(tmp1))?;
3567 this.assembler.emit_stlxrh(
3568 Size::S32,
3569 Location::GPR(tmp2),
3570 Location::GPR(tmp1),
3571 Location::GPR(addr),
3572 )?;
3573 this.assembler
3574 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3575 this.assembler.emit_dmb()?;
3576
3577 if dst != ret {
3578 this.move_location(Size::S32, ret, dst)?;
3579 }
3580 for r in temps {
3581 this.release_gpr(r);
3582 }
3583 this.release_gpr(tmp1);
3584 this.release_gpr(tmp2);
3585 Ok(())
3586 },
3587 )
3588 }
3589
3590 fn i32_atomic_sub(
3591 &mut self,
3592 loc: Location,
3593 target: Location,
3594 memarg: &MemArg,
3595 ret: Location,
3596 need_check: bool,
3597 imported_memories: bool,
3598 offset: i32,
3599 heap_access_oob: Label,
3600 unaligned_atomic: Label,
3601 ) -> Result<(), CompileError> {
3602 self.memory_op(
3603 target,
3604 memarg,
3605 true,
3606 4,
3607 need_check,
3608 imported_memories,
3609 offset,
3610 heap_access_oob,
3611 unaligned_atomic,
3612 |this, addr| {
3613 let mut temps = vec![];
3614 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3615 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3616 })?;
3617 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3618 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3619 })?;
3620 let dst =
3621 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3622 let reread = this.get_label();
3623
3624 this.emit_label(reread)?;
3625 this.assembler
3626 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
3627 this.emit_binop_sub32(dst, loc, Location::GPR(tmp1))?;
3628 this.assembler.emit_stlxr(
3629 Size::S32,
3630 Location::GPR(tmp2),
3631 Location::GPR(tmp1),
3632 Location::GPR(addr),
3633 )?;
3634 this.assembler
3635 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3636 this.assembler.emit_dmb()?;
3637
3638 if dst != ret {
3639 this.move_location(Size::S32, ret, dst)?;
3640 }
3641 for r in temps {
3642 this.release_gpr(r);
3643 }
3644 this.release_gpr(tmp1);
3645 this.release_gpr(tmp2);
3646 Ok(())
3647 },
3648 )
3649 }
3650
3651 fn i32_atomic_sub_8u(
3652 &mut self,
3653 loc: Location,
3654 target: Location,
3655 memarg: &MemArg,
3656 ret: Location,
3657 need_check: bool,
3658 imported_memories: bool,
3659 offset: i32,
3660 heap_access_oob: Label,
3661 unaligned_atomic: Label,
3662 ) -> Result<(), CompileError> {
3663 self.memory_op(
3664 target,
3665 memarg,
3666 true,
3667 1,
3668 need_check,
3669 imported_memories,
3670 offset,
3671 heap_access_oob,
3672 unaligned_atomic,
3673 |this, addr| {
3674 let mut temps = vec![];
3675 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3676 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3677 })?;
3678 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3679 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3680 })?;
3681 let dst =
3682 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3683 let reread = this.get_label();
3684
3685 this.emit_label(reread)?;
3686 this.assembler
3687 .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?;
3688 this.emit_binop_sub32(dst, loc, Location::GPR(tmp1))?;
3689 this.assembler.emit_stlxrb(
3690 Size::S32,
3691 Location::GPR(tmp2),
3692 Location::GPR(tmp1),
3693 Location::GPR(addr),
3694 )?;
3695 this.assembler
3696 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3697 this.assembler.emit_dmb()?;
3698
3699 if dst != ret {
3700 this.move_location(Size::S32, ret, dst)?;
3701 }
3702 for r in temps {
3703 this.release_gpr(r);
3704 }
3705 this.release_gpr(tmp1);
3706 this.release_gpr(tmp2);
3707 Ok(())
3708 },
3709 )
3710 }
3711
3712 fn i32_atomic_sub_16u(
3713 &mut self,
3714 loc: Location,
3715 target: Location,
3716 memarg: &MemArg,
3717 ret: Location,
3718 need_check: bool,
3719 imported_memories: bool,
3720 offset: i32,
3721 heap_access_oob: Label,
3722 unaligned_atomic: Label,
3723 ) -> Result<(), CompileError> {
3724 self.memory_op(
3725 target,
3726 memarg,
3727 true,
3728 2,
3729 need_check,
3730 imported_memories,
3731 offset,
3732 heap_access_oob,
3733 unaligned_atomic,
3734 |this, addr| {
3735 let mut temps = vec![];
3736 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3737 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3738 })?;
3739 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3740 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3741 })?;
3742 let dst =
3743 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3744 let reread = this.get_label();
3745
3746 this.emit_label(reread)?;
3747 this.assembler
3748 .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?;
3749 this.emit_binop_sub32(dst, loc, Location::GPR(tmp1))?;
3750 this.assembler.emit_stlxrh(
3751 Size::S32,
3752 Location::GPR(tmp2),
3753 Location::GPR(tmp1),
3754 Location::GPR(addr),
3755 )?;
3756 this.assembler
3757 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3758 this.assembler.emit_dmb()?;
3759
3760 if dst != ret {
3761 this.move_location(Size::S32, ret, dst)?;
3762 }
3763 for r in temps {
3764 this.release_gpr(r);
3765 }
3766 this.release_gpr(tmp1);
3767 this.release_gpr(tmp2);
3768 Ok(())
3769 },
3770 )
3771 }
3772
3773 fn i32_atomic_and(
3774 &mut self,
3775 loc: Location,
3776 target: Location,
3777 memarg: &MemArg,
3778 ret: Location,
3779 need_check: bool,
3780 imported_memories: bool,
3781 offset: i32,
3782 heap_access_oob: Label,
3783 unaligned_atomic: Label,
3784 ) -> Result<(), CompileError> {
3785 self.memory_op(
3786 target,
3787 memarg,
3788 true,
3789 4,
3790 need_check,
3791 imported_memories,
3792 offset,
3793 heap_access_oob,
3794 unaligned_atomic,
3795 |this, addr| {
3796 let mut temps = vec![];
3797 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3798 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3799 })?;
3800 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3801 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3802 })?;
3803 let dst =
3804 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3805 let reread = this.get_label();
3806
3807 this.emit_label(reread)?;
3808 this.assembler
3809 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
3810 this.emit_binop_and32(dst, loc, Location::GPR(tmp1))?;
3811 this.assembler.emit_stlxr(
3812 Size::S32,
3813 Location::GPR(tmp2),
3814 Location::GPR(tmp1),
3815 Location::GPR(addr),
3816 )?;
3817 this.assembler
3818 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3819 this.assembler.emit_dmb()?;
3820
3821 if dst != ret {
3822 this.move_location(Size::S32, ret, dst)?;
3823 }
3824 for r in temps {
3825 this.release_gpr(r);
3826 }
3827 this.release_gpr(tmp1);
3828 this.release_gpr(tmp2);
3829 Ok(())
3830 },
3831 )
3832 }
3833
3834 fn i32_atomic_and_8u(
3835 &mut self,
3836 loc: Location,
3837 target: Location,
3838 memarg: &MemArg,
3839 ret: Location,
3840 need_check: bool,
3841 imported_memories: bool,
3842 offset: i32,
3843 heap_access_oob: Label,
3844 unaligned_atomic: Label,
3845 ) -> Result<(), CompileError> {
3846 self.memory_op(
3847 target,
3848 memarg,
3849 true,
3850 1,
3851 need_check,
3852 imported_memories,
3853 offset,
3854 heap_access_oob,
3855 unaligned_atomic,
3856 |this, addr| {
3857 let mut temps = vec![];
3858 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3859 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3860 })?;
3861 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3862 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3863 })?;
3864 let dst =
3865 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3866 let reread = this.get_label();
3867
3868 this.emit_label(reread)?;
3869 this.assembler
3870 .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?;
3871 this.emit_binop_and32(dst, loc, Location::GPR(tmp1))?;
3872 this.assembler.emit_stlxrb(
3873 Size::S32,
3874 Location::GPR(tmp2),
3875 Location::GPR(tmp1),
3876 Location::GPR(addr),
3877 )?;
3878 this.assembler
3879 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3880 this.assembler.emit_dmb()?;
3881
3882 if dst != ret {
3883 this.move_location(Size::S32, ret, dst)?;
3884 }
3885 for r in temps {
3886 this.release_gpr(r);
3887 }
3888 this.release_gpr(tmp1);
3889 this.release_gpr(tmp2);
3890 Ok(())
3891 },
3892 )
3893 }
3894
3895 fn i32_atomic_and_16u(
3896 &mut self,
3897 loc: Location,
3898 target: Location,
3899 memarg: &MemArg,
3900 ret: Location,
3901 need_check: bool,
3902 imported_memories: bool,
3903 offset: i32,
3904 heap_access_oob: Label,
3905 unaligned_atomic: Label,
3906 ) -> Result<(), CompileError> {
3907 self.memory_op(
3908 target,
3909 memarg,
3910 true,
3911 2,
3912 need_check,
3913 imported_memories,
3914 offset,
3915 heap_access_oob,
3916 unaligned_atomic,
3917 |this, addr| {
3918 let mut temps = vec![];
3919 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3920 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3921 })?;
3922 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3923 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3924 })?;
3925 let dst =
3926 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3927 let reread = this.get_label();
3928
3929 this.emit_label(reread)?;
3930 this.assembler
3931 .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?;
3932 this.emit_binop_and32(dst, loc, Location::GPR(tmp1))?;
3933 this.assembler.emit_stlxrh(
3934 Size::S32,
3935 Location::GPR(tmp2),
3936 Location::GPR(tmp1),
3937 Location::GPR(addr),
3938 )?;
3939 this.assembler
3940 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3941 this.assembler.emit_dmb()?;
3942
3943 if dst != ret {
3944 this.move_location(Size::S32, ret, dst)?;
3945 }
3946 for r in temps {
3947 this.release_gpr(r);
3948 }
3949 this.release_gpr(tmp1);
3950 this.release_gpr(tmp2);
3951 Ok(())
3952 },
3953 )
3954 }
3955
3956 fn i32_atomic_or(
3957 &mut self,
3958 loc: Location,
3959 target: Location,
3960 memarg: &MemArg,
3961 ret: Location,
3962 need_check: bool,
3963 imported_memories: bool,
3964 offset: i32,
3965 heap_access_oob: Label,
3966 unaligned_atomic: Label,
3967 ) -> Result<(), CompileError> {
3968 self.memory_op(
3969 target,
3970 memarg,
3971 true,
3972 4,
3973 need_check,
3974 imported_memories,
3975 offset,
3976 heap_access_oob,
3977 unaligned_atomic,
3978 |this, addr| {
3979 let mut temps = vec![];
3980 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3981 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3982 })?;
3983 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3984 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3985 })?;
3986 let dst =
3987 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3988 let reread = this.get_label();
3989
3990 this.emit_label(reread)?;
3991 this.assembler
3992 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
3993 this.emit_binop_or32(dst, loc, Location::GPR(tmp1))?;
3994 this.assembler.emit_stlxr(
3995 Size::S32,
3996 Location::GPR(tmp2),
3997 Location::GPR(tmp1),
3998 Location::GPR(addr),
3999 )?;
4000 this.assembler
4001 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
4002 this.assembler.emit_dmb()?;
4003
4004 if dst != ret {
4005 this.move_location(Size::S32, ret, dst)?;
4006 }
4007 for r in temps {
4008 this.release_gpr(r);
4009 }
4010 this.release_gpr(tmp1);
4011 this.release_gpr(tmp2);
4012 Ok(())
4013 },
4014 )
4015 }
4016
4017 fn i32_atomic_or_8u(
4018 &mut self,
4019 loc: Location,
4020 target: Location,
4021 memarg: &MemArg,
4022 ret: Location,
4023 need_check: bool,
4024 imported_memories: bool,
4025 offset: i32,
4026 heap_access_oob: Label,
4027 unaligned_atomic: Label,
4028 ) -> Result<(), CompileError> {
4029 self.memory_op(
4030 target,
4031 memarg,
4032 true,
4033 1,
4034 need_check,
4035 imported_memories,
4036 offset,
4037 heap_access_oob,
4038 unaligned_atomic,
4039 |this, addr| {
4040 let mut temps = vec![];
4041 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
4042 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4043 })?;
4044 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
4045 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4046 })?;
4047 let dst =
4048 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4049 let reread = this.get_label();
4050
4051 this.emit_label(reread)?;
4052 this.assembler
4053 .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?;
4054 this.emit_binop_or32(dst, loc, Location::GPR(tmp1))?;
4055 this.assembler.emit_stlxrb(
4056 Size::S32,
4057 Location::GPR(tmp2),
4058 Location::GPR(tmp1),
4059 Location::GPR(addr),
4060 )?;
4061 this.assembler
4062 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
4063 this.assembler.emit_dmb()?;
4064
4065 if dst != ret {
4066 this.move_location(Size::S32, ret, dst)?;
4067 }
4068 for r in temps {
4069 this.release_gpr(r);
4070 }
4071 this.release_gpr(tmp1);
4072 this.release_gpr(tmp2);
4073 Ok(())
4074 },
4075 )
4076 }
4077
4078 fn i32_atomic_or_16u(
4079 &mut self,
4080 loc: Location,
4081 target: Location,
4082 memarg: &MemArg,
4083 ret: Location,
4084 need_check: bool,
4085 imported_memories: bool,
4086 offset: i32,
4087 heap_access_oob: Label,
4088 unaligned_atomic: Label,
4089 ) -> Result<(), CompileError> {
4090 self.memory_op(
4091 target,
4092 memarg,
4093 true,
4094 2,
4095 need_check,
4096 imported_memories,
4097 offset,
4098 heap_access_oob,
4099 unaligned_atomic,
4100 |this, addr| {
4101 let mut temps = vec![];
4102 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
4103 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4104 })?;
4105 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
4106 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4107 })?;
4108 let dst =
4109 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4110 let reread = this.get_label();
4111
4112 this.emit_label(reread)?;
4113 this.assembler
4114 .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?;
4115 this.emit_binop_or32(dst, loc, Location::GPR(tmp1))?;
4116 this.assembler.emit_stlxrh(
4117 Size::S32,
4118 Location::GPR(tmp2),
4119 Location::GPR(tmp1),
4120 Location::GPR(addr),
4121 )?;
4122 this.assembler
4123 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
4124 this.assembler.emit_dmb()?;
4125
4126 if dst != ret {
4127 this.move_location(Size::S32, ret, dst)?;
4128 }
4129 for r in temps {
4130 this.release_gpr(r);
4131 }
4132 this.release_gpr(tmp1);
4133 this.release_gpr(tmp2);
4134 Ok(())
4135 },
4136 )
4137 }
4138
4139 fn i32_atomic_xor(
4140 &mut self,
4141 loc: Location,
4142 target: Location,
4143 memarg: &MemArg,
4144 ret: Location,
4145 need_check: bool,
4146 imported_memories: bool,
4147 offset: i32,
4148 heap_access_oob: Label,
4149 unaligned_atomic: Label,
4150 ) -> Result<(), CompileError> {
4151 self.memory_op(
4152 target,
4153 memarg,
4154 true,
4155 4,
4156 need_check,
4157 imported_memories,
4158 offset,
4159 heap_access_oob,
4160 unaligned_atomic,
4161 |this, addr| {
4162 let mut temps = vec![];
4163 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
4164 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4165 })?;
4166 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
4167 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4168 })?;
4169 let dst =
4170 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4171 let reread = this.get_label();
4172
4173 this.emit_label(reread)?;
4174 this.assembler
4175 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
4176 this.emit_binop_xor32(dst, loc, Location::GPR(tmp1))?;
4177 this.assembler.emit_stlxr(
4178 Size::S32,
4179 Location::GPR(tmp2),
4180 Location::GPR(tmp1),
4181 Location::GPR(addr),
4182 )?;
4183 this.assembler
4184 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
4185 this.assembler.emit_dmb()?;
4186
4187 if dst != ret {
4188 this.move_location(Size::S32, ret, dst)?;
4189 }
4190 for r in temps {
4191 this.release_gpr(r);
4192 }
4193 this.release_gpr(tmp1);
4194 this.release_gpr(tmp2);
4195 Ok(())
4196 },
4197 )
4198 }
4199
4200 fn i32_atomic_xor_8u(
4201 &mut self,
4202 loc: Location,
4203 target: Location,
4204 memarg: &MemArg,
4205 ret: Location,
4206 need_check: bool,
4207 imported_memories: bool,
4208 offset: i32,
4209 heap_access_oob: Label,
4210 unaligned_atomic: Label,
4211 ) -> Result<(), CompileError> {
4212 self.memory_op(
4213 target,
4214 memarg,
4215 true,
4216 1,
4217 need_check,
4218 imported_memories,
4219 offset,
4220 heap_access_oob,
4221 unaligned_atomic,
4222 |this, addr| {
4223 let mut temps = vec![];
4224 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
4225 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4226 })?;
4227 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
4228 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4229 })?;
4230 let dst =
4231 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4232 let reread = this.get_label();
4233
4234 this.emit_label(reread)?;
4235 this.assembler
4236 .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?;
4237 this.emit_binop_xor32(dst, loc, Location::GPR(tmp1))?;
4238 this.assembler.emit_stlxrb(
4239 Size::S32,
4240 Location::GPR(tmp2),
4241 Location::GPR(tmp1),
4242 Location::GPR(addr),
4243 )?;
4244 this.assembler
4245 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
4246 this.assembler.emit_dmb()?;
4247
4248 if dst != ret {
4249 this.move_location(Size::S32, ret, dst)?;
4250 }
4251 for r in temps {
4252 this.release_gpr(r);
4253 }
4254 this.release_gpr(tmp1);
4255 this.release_gpr(tmp2);
4256 Ok(())
4257 },
4258 )
4259 }
4260
4261 fn i32_atomic_xor_16u(
4262 &mut self,
4263 loc: Location,
4264 target: Location,
4265 memarg: &MemArg,
4266 ret: Location,
4267 need_check: bool,
4268 imported_memories: bool,
4269 offset: i32,
4270 heap_access_oob: Label,
4271 unaligned_atomic: Label,
4272 ) -> Result<(), CompileError> {
4273 self.memory_op(
4274 target,
4275 memarg,
4276 true,
4277 2,
4278 need_check,
4279 imported_memories,
4280 offset,
4281 heap_access_oob,
4282 unaligned_atomic,
4283 |this, addr| {
4284 let mut temps = vec![];
4285 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
4286 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4287 })?;
4288 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
4289 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4290 })?;
4291 let dst =
4292 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4293 let reread = this.get_label();
4294
4295 this.emit_label(reread)?;
4296 this.assembler
4297 .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?;
4298 this.emit_binop_xor32(dst, loc, Location::GPR(tmp1))?;
4299 this.assembler.emit_stlxrh(
4300 Size::S32,
4301 Location::GPR(tmp2),
4302 Location::GPR(tmp1),
4303 Location::GPR(addr),
4304 )?;
4305 this.assembler
4306 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
4307 this.assembler.emit_dmb()?;
4308
4309 if dst != ret {
4310 this.move_location(Size::S32, ret, dst)?;
4311 }
4312 for r in temps {
4313 this.release_gpr(r);
4314 }
4315 this.release_gpr(tmp1);
4316 this.release_gpr(tmp2);
4317 Ok(())
4318 },
4319 )
4320 }
4321
4322 fn i32_atomic_xchg(
4323 &mut self,
4324 loc: Location,
4325 target: Location,
4326 memarg: &MemArg,
4327 ret: Location,
4328 need_check: bool,
4329 imported_memories: bool,
4330 offset: i32,
4331 heap_access_oob: Label,
4332 unaligned_atomic: Label,
4333 ) -> Result<(), CompileError> {
4334 self.memory_op(
4335 target,
4336 memarg,
4337 true,
4338 4,
4339 need_check,
4340 imported_memories,
4341 offset,
4342 heap_access_oob,
4343 unaligned_atomic,
4344 |this, addr| {
4345 let mut temps = vec![];
4346 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
4347 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4348 })?;
4349 let dst =
4350 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4351 let org =
4352 this.location_to_reg(Size::S32, loc, &mut temps, ImmType::None, false, None)?;
4353 let reread = this.get_label();
4354
4355 this.emit_label(reread)?;
4356 this.assembler
4357 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
4358 this.assembler.emit_stlxr(
4359 Size::S32,
4360 Location::GPR(tmp),
4361 org,
4362 Location::GPR(addr),
4363 )?;
4364 this.assembler
4365 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
4366 this.assembler.emit_dmb()?;
4367
4368 if dst != ret {
4369 this.move_location(Size::S32, ret, dst)?;
4370 }
4371 for r in temps {
4372 this.release_gpr(r);
4373 }
4374 this.release_gpr(tmp);
4375 Ok(())
4376 },
4377 )
4378 }
4379
4380 fn i32_atomic_xchg_8u(
4381 &mut self,
4382 loc: Location,
4383 target: Location,
4384 memarg: &MemArg,
4385 ret: Location,
4386 need_check: bool,
4387 imported_memories: bool,
4388 offset: i32,
4389 heap_access_oob: Label,
4390 unaligned_atomic: Label,
4391 ) -> Result<(), CompileError> {
4392 self.memory_op(
4393 target,
4394 memarg,
4395 true,
4396 1,
4397 need_check,
4398 imported_memories,
4399 offset,
4400 heap_access_oob,
4401 unaligned_atomic,
4402 |this, addr| {
4403 let mut temps = vec![];
4404 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
4405 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4406 })?;
4407 let dst =
4408 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4409 let org =
4410 this.location_to_reg(Size::S32, loc, &mut temps, ImmType::None, false, None)?;
4411 let reread = this.get_label();
4412
4413 this.emit_label(reread)?;
4414 this.assembler
4415 .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?;
4416 this.assembler.emit_stlxrb(
4417 Size::S32,
4418 Location::GPR(tmp),
4419 org,
4420 Location::GPR(addr),
4421 )?;
4422 this.assembler
4423 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
4424 this.assembler.emit_dmb()?;
4425
4426 if dst != ret {
4427 this.move_location(Size::S32, ret, dst)?;
4428 }
4429 for r in temps {
4430 this.release_gpr(r);
4431 }
4432 this.release_gpr(tmp);
4433 Ok(())
4434 },
4435 )
4436 }
4437
4438 fn i32_atomic_xchg_16u(
4439 &mut self,
4440 loc: Location,
4441 target: Location,
4442 memarg: &MemArg,
4443 ret: Location,
4444 need_check: bool,
4445 imported_memories: bool,
4446 offset: i32,
4447 heap_access_oob: Label,
4448 unaligned_atomic: Label,
4449 ) -> Result<(), CompileError> {
4450 self.memory_op(
4451 target,
4452 memarg,
4453 true,
4454 2,
4455 need_check,
4456 imported_memories,
4457 offset,
4458 heap_access_oob,
4459 unaligned_atomic,
4460 |this, addr| {
4461 let mut temps = vec![];
4462 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
4463 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4464 })?;
4465 let dst =
4466 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4467 let org =
4468 this.location_to_reg(Size::S32, loc, &mut temps, ImmType::None, false, None)?;
4469 let reread = this.get_label();
4470
4471 this.emit_label(reread)?;
4472 this.assembler
4473 .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?;
4474 this.assembler.emit_stlxrh(
4475 Size::S32,
4476 Location::GPR(tmp),
4477 org,
4478 Location::GPR(addr),
4479 )?;
4480 this.assembler
4481 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
4482 this.assembler.emit_dmb()?;
4483
4484 if dst != ret {
4485 this.move_location(Size::S32, ret, dst)?;
4486 }
4487 for r in temps {
4488 this.release_gpr(r);
4489 }
4490 this.release_gpr(tmp);
4491 Ok(())
4492 },
4493 )
4494 }
4495
4496 fn i32_atomic_cmpxchg(
4497 &mut self,
4498 new: Location,
4499 cmp: Location,
4500 target: Location,
4501 memarg: &MemArg,
4502 ret: Location,
4503 need_check: bool,
4504 imported_memories: bool,
4505 offset: i32,
4506 heap_access_oob: Label,
4507 unaligned_atomic: Label,
4508 ) -> Result<(), CompileError> {
4509 self.memory_op(
4510 target,
4511 memarg,
4512 true,
4513 4,
4514 need_check,
4515 imported_memories,
4516 offset,
4517 heap_access_oob,
4518 unaligned_atomic,
4519 |this, addr| {
4520 let mut temps = vec![];
4521 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
4522 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4523 })?;
4524 let dst =
4525 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4526 let org =
4527 this.location_to_reg(Size::S32, new, &mut temps, ImmType::None, false, None)?;
4528 let reread = this.get_label();
4529 let nosame = this.get_label();
4530
4531 this.emit_label(reread)?;
4532 this.assembler
4533 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
4534 this.emit_relaxed_cmp(Size::S32, dst, cmp)?;
4535 this.assembler.emit_bcond_label(Condition::Ne, nosame)?;
4536 this.assembler.emit_stlxr(
4537 Size::S32,
4538 Location::GPR(tmp),
4539 org,
4540 Location::GPR(addr),
4541 )?;
4542 this.assembler
4543 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
4544 this.assembler.emit_dmb()?;
4545
4546 this.emit_label(nosame)?;
4547 if dst != ret {
4548 this.move_location(Size::S32, ret, dst)?;
4549 }
4550 for r in temps {
4551 this.release_gpr(r);
4552 }
4553 this.release_gpr(tmp);
4554 Ok(())
4555 },
4556 )
4557 }
4558
4559 fn i32_atomic_cmpxchg_8u(
4560 &mut self,
4561 new: Location,
4562 cmp: Location,
4563 target: Location,
4564 memarg: &MemArg,
4565 ret: Location,
4566 need_check: bool,
4567 imported_memories: bool,
4568 offset: i32,
4569 heap_access_oob: Label,
4570 unaligned_atomic: Label,
4571 ) -> Result<(), CompileError> {
4572 self.memory_op(
4573 target,
4574 memarg,
4575 true,
4576 1,
4577 need_check,
4578 imported_memories,
4579 offset,
4580 heap_access_oob,
4581 unaligned_atomic,
4582 |this, addr| {
4583 let mut temps = vec![];
4584 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
4585 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4586 })?;
4587 let dst =
4588 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4589 let org =
4590 this.location_to_reg(Size::S32, new, &mut temps, ImmType::None, false, None)?;
4591 let reread = this.get_label();
4592 let nosame = this.get_label();
4593
4594 this.emit_label(reread)?;
4595 this.assembler
4596 .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?;
4597 this.emit_relaxed_cmp(Size::S32, dst, cmp)?;
4598 this.assembler.emit_bcond_label(Condition::Ne, nosame)?;
4599 this.assembler.emit_stlxrb(
4600 Size::S32,
4601 Location::GPR(tmp),
4602 org,
4603 Location::GPR(addr),
4604 )?;
4605 this.assembler
4606 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
4607 this.assembler.emit_dmb()?;
4608
4609 this.emit_label(nosame)?;
4610 if dst != ret {
4611 this.move_location(Size::S32, ret, dst)?;
4612 }
4613 for r in temps {
4614 this.release_gpr(r);
4615 }
4616 this.release_gpr(tmp);
4617 Ok(())
4618 },
4619 )
4620 }
4621
4622 fn i32_atomic_cmpxchg_16u(
4623 &mut self,
4624 new: Location,
4625 cmp: Location,
4626 target: Location,
4627 memarg: &MemArg,
4628 ret: Location,
4629 need_check: bool,
4630 imported_memories: bool,
4631 offset: i32,
4632 heap_access_oob: Label,
4633 unaligned_atomic: Label,
4634 ) -> Result<(), CompileError> {
4635 self.memory_op(
4636 target,
4637 memarg,
4638 true,
4639 2,
4640 need_check,
4641 imported_memories,
4642 offset,
4643 heap_access_oob,
4644 unaligned_atomic,
4645 |this, addr| {
4646 let mut temps = vec![];
4647 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
4648 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4649 })?;
4650 let dst =
4651 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4652 let org =
4653 this.location_to_reg(Size::S32, new, &mut temps, ImmType::None, false, None)?;
4654 let reread = this.get_label();
4655 let nosame = this.get_label();
4656
4657 this.emit_label(reread)?;
4658 this.assembler
4659 .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?;
4660 this.emit_relaxed_cmp(Size::S32, dst, cmp)?;
4661 this.assembler.emit_bcond_label(Condition::Ne, nosame)?;
4662 this.assembler.emit_stlxrh(
4663 Size::S32,
4664 Location::GPR(tmp),
4665 org,
4666 Location::GPR(addr),
4667 )?;
4668 this.assembler
4669 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
4670 this.assembler.emit_dmb()?;
4671
4672 this.emit_label(nosame)?;
4673 if dst != ret {
4674 this.move_location(Size::S32, ret, dst)?;
4675 }
4676 for r in temps {
4677 this.release_gpr(r);
4678 }
4679 this.release_gpr(tmp);
4680 Ok(())
4681 },
4682 )
4683 }
4684
4685 fn emit_call_with_reloc(
4686 &mut self,
4687 _calling_convention: CallingConvention,
4688 reloc_target: RelocationTarget,
4689 ) -> Result<Vec<Relocation>, CompileError> {
4690 let mut relocations = vec![];
4691 let next = self.get_label();
4692 let reloc_at = self.assembler.get_offset().0;
4693 self.emit_label(next)?; self.assembler.emit_call_label(next)?;
4695 relocations.push(Relocation {
4696 kind: RelocationKind::Arm64Call,
4697 reloc_target,
4698 offset: reloc_at as u32,
4699 addend: 0,
4700 });
4701 Ok(relocations)
4702 }
4703
4704 fn emit_binop_add64(
4705 &mut self,
4706 loc_a: Location,
4707 loc_b: Location,
4708 ret: Location,
4709 ) -> Result<(), CompileError> {
4710 self.emit_relaxed_binop3(
4711 Assembler::emit_add,
4712 Size::S64,
4713 loc_a,
4714 loc_b,
4715 ret,
4716 ImmType::Bits12,
4717 )
4718 }
4719
4720 fn emit_binop_sub64(
4721 &mut self,
4722 loc_a: Location,
4723 loc_b: Location,
4724 ret: Location,
4725 ) -> Result<(), CompileError> {
4726 self.emit_relaxed_binop3(
4727 Assembler::emit_sub,
4728 Size::S64,
4729 loc_a,
4730 loc_b,
4731 ret,
4732 ImmType::Bits12,
4733 )
4734 }
4735
4736 fn emit_binop_mul64(
4737 &mut self,
4738 loc_a: Location,
4739 loc_b: Location,
4740 ret: Location,
4741 ) -> Result<(), CompileError> {
4742 self.emit_relaxed_binop3(
4743 Assembler::emit_mul,
4744 Size::S64,
4745 loc_a,
4746 loc_b,
4747 ret,
4748 ImmType::None,
4749 )
4750 }
4751
4752 fn emit_binop_udiv64(
4753 &mut self,
4754 loc_a: Location,
4755 loc_b: Location,
4756 ret: Location,
4757 integer_division_by_zero: Label,
4758 ) -> Result<usize, CompileError> {
4759 let mut temps = vec![];
4760 let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4761 let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4762 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4763
4764 self.assembler
4765 .emit_cbz_label(Size::S64, src2, integer_division_by_zero)?;
4766 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
4767 self.assembler.emit_udiv(Size::S64, src1, src2, dest)?;
4768 if ret != dest {
4769 self.move_location(Size::S64, dest, ret)?;
4770 }
4771 for r in temps {
4772 self.release_gpr(r);
4773 }
4774 Ok(offset)
4775 }
4776
4777 fn emit_binop_sdiv64(
4778 &mut self,
4779 loc_a: Location,
4780 loc_b: Location,
4781 ret: Location,
4782 integer_division_by_zero: Label,
4783 integer_overflow: Label,
4784 ) -> Result<usize, CompileError> {
4785 let mut temps = vec![];
4786 let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4787 let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4788 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4789
4790 self.assembler
4791 .emit_cbz_label_far(Size::S64, src2, integer_division_by_zero)?;
4792 let label_nooverflow = self.assembler.get_label();
4793 let tmp = self.location_to_reg(
4794 Size::S64,
4795 Location::Imm64(0x8000000000000000),
4796 &mut temps,
4797 ImmType::None,
4798 true,
4799 None,
4800 )?;
4801 self.assembler.emit_cmp(Size::S64, tmp, src1)?;
4802 self.assembler
4803 .emit_bcond_label(Condition::Ne, label_nooverflow)?;
4804 self.assembler.emit_movn(Size::S64, tmp, 0)?;
4805 self.assembler.emit_cmp(Size::S64, tmp, src2)?;
4806 self.assembler
4807 .emit_bcond_label_far(Condition::Eq, integer_overflow)?;
4808 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
4809 self.assembler.emit_label(label_nooverflow)?;
4810 self.assembler.emit_sdiv(Size::S64, src1, src2, dest)?;
4811 if ret != dest {
4812 self.move_location(Size::S64, dest, ret)?;
4813 }
4814 for r in temps {
4815 self.release_gpr(r);
4816 }
4817 Ok(offset)
4818 }
4819
4820 fn emit_binop_urem64(
4821 &mut self,
4822 loc_a: Location,
4823 loc_b: Location,
4824 ret: Location,
4825 integer_division_by_zero: Label,
4826 ) -> Result<usize, CompileError> {
4827 let mut temps = vec![];
4828 let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4829 let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4830 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4831 let dest = if dest == src1 || dest == src2 {
4832 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
4833 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4834 })?;
4835 temps.push(tmp);
4836 self.assembler
4837 .emit_mov(Size::S32, dest, Location::GPR(tmp))?;
4838 Location::GPR(tmp)
4839 } else {
4840 dest
4841 };
4842 self.assembler
4843 .emit_cbz_label_far(Size::S64, src2, integer_division_by_zero)?;
4844 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
4845 self.assembler.emit_udiv(Size::S64, src1, src2, dest)?;
4846 self.assembler
4848 .emit_msub(Size::S64, dest, src2, src1, dest)?;
4849 if ret != dest {
4850 self.move_location(Size::S64, dest, ret)?;
4851 }
4852 for r in temps {
4853 self.release_gpr(r);
4854 }
4855 Ok(offset)
4856 }
4857
4858 fn emit_binop_srem64(
4859 &mut self,
4860 loc_a: Location,
4861 loc_b: Location,
4862 ret: Location,
4863 integer_division_by_zero: Label,
4864 ) -> Result<usize, CompileError> {
4865 let mut temps = vec![];
4866 let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4867 let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4868 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4869 let dest = if dest == src1 || dest == src2 {
4870 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
4871 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4872 })?;
4873 temps.push(tmp);
4874 self.assembler
4875 .emit_mov(Size::S64, dest, Location::GPR(tmp))?;
4876 Location::GPR(tmp)
4877 } else {
4878 dest
4879 };
4880 self.assembler
4881 .emit_cbz_label_far(Size::S64, src2, integer_division_by_zero)?;
4882 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
4883 self.assembler.emit_sdiv(Size::S64, src1, src2, dest)?;
4884 self.assembler
4886 .emit_msub(Size::S64, dest, src2, src1, dest)?;
4887 if ret != dest {
4888 self.move_location(Size::S64, dest, ret)?;
4889 }
4890 for r in temps {
4891 self.release_gpr(r);
4892 }
4893 Ok(offset)
4894 }
4895
4896 fn emit_binop_and64(
4897 &mut self,
4898 loc_a: Location,
4899 loc_b: Location,
4900 ret: Location,
4901 ) -> Result<(), CompileError> {
4902 self.emit_relaxed_binop3(
4903 Assembler::emit_and,
4904 Size::S64,
4905 loc_a,
4906 loc_b,
4907 ret,
4908 ImmType::Logical64,
4909 )
4910 }
4911
4912 fn emit_binop_or64(
4913 &mut self,
4914 loc_a: Location,
4915 loc_b: Location,
4916 ret: Location,
4917 ) -> Result<(), CompileError> {
4918 self.emit_relaxed_binop3(
4919 Assembler::emit_or,
4920 Size::S64,
4921 loc_a,
4922 loc_b,
4923 ret,
4924 ImmType::Logical64,
4925 )
4926 }
4927
4928 fn emit_binop_xor64(
4929 &mut self,
4930 loc_a: Location,
4931 loc_b: Location,
4932 ret: Location,
4933 ) -> Result<(), CompileError> {
4934 self.emit_relaxed_binop3(
4935 Assembler::emit_eor,
4936 Size::S64,
4937 loc_a,
4938 loc_b,
4939 ret,
4940 ImmType::Logical64,
4941 )
4942 }
4943
4944 fn i64_cmp_ge_s(
4945 &mut self,
4946 loc_a: Location,
4947 loc_b: Location,
4948 ret: Location,
4949 ) -> Result<(), CompileError> {
4950 self.emit_cmpop_i64_dynamic_b(Condition::Ge, loc_a, loc_b, ret)
4951 }
4952
4953 fn i64_cmp_gt_s(
4954 &mut self,
4955 loc_a: Location,
4956 loc_b: Location,
4957 ret: Location,
4958 ) -> Result<(), CompileError> {
4959 self.emit_cmpop_i64_dynamic_b(Condition::Gt, loc_a, loc_b, ret)
4960 }
4961
4962 fn i64_cmp_le_s(
4963 &mut self,
4964 loc_a: Location,
4965 loc_b: Location,
4966 ret: Location,
4967 ) -> Result<(), CompileError> {
4968 self.emit_cmpop_i64_dynamic_b(Condition::Le, loc_a, loc_b, ret)
4969 }
4970
4971 fn i64_cmp_lt_s(
4972 &mut self,
4973 loc_a: Location,
4974 loc_b: Location,
4975 ret: Location,
4976 ) -> Result<(), CompileError> {
4977 self.emit_cmpop_i64_dynamic_b(Condition::Lt, loc_a, loc_b, ret)
4978 }
4979
4980 fn i64_cmp_ge_u(
4981 &mut self,
4982 loc_a: Location,
4983 loc_b: Location,
4984 ret: Location,
4985 ) -> Result<(), CompileError> {
4986 self.emit_cmpop_i64_dynamic_b(Condition::Cs, loc_a, loc_b, ret)
4987 }
4988
4989 fn i64_cmp_gt_u(
4990 &mut self,
4991 loc_a: Location,
4992 loc_b: Location,
4993 ret: Location,
4994 ) -> Result<(), CompileError> {
4995 self.emit_cmpop_i64_dynamic_b(Condition::Hi, loc_a, loc_b, ret)
4996 }
4997
4998 fn i64_cmp_le_u(
4999 &mut self,
5000 loc_a: Location,
5001 loc_b: Location,
5002 ret: Location,
5003 ) -> Result<(), CompileError> {
5004 self.emit_cmpop_i64_dynamic_b(Condition::Ls, loc_a, loc_b, ret)
5005 }
5006
5007 fn i64_cmp_lt_u(
5008 &mut self,
5009 loc_a: Location,
5010 loc_b: Location,
5011 ret: Location,
5012 ) -> Result<(), CompileError> {
5013 self.emit_cmpop_i64_dynamic_b(Condition::Cc, loc_a, loc_b, ret)
5014 }
5015
5016 fn i64_cmp_ne(
5017 &mut self,
5018 loc_a: Location,
5019 loc_b: Location,
5020 ret: Location,
5021 ) -> Result<(), CompileError> {
5022 self.emit_cmpop_i64_dynamic_b(Condition::Ne, loc_a, loc_b, ret)
5023 }
5024
5025 fn i64_cmp_eq(
5026 &mut self,
5027 loc_a: Location,
5028 loc_b: Location,
5029 ret: Location,
5030 ) -> Result<(), CompileError> {
5031 self.emit_cmpop_i64_dynamic_b(Condition::Eq, loc_a, loc_b, ret)
5032 }
5033
5034 fn i64_clz(&mut self, src: Location, dst: Location) -> Result<(), CompileError> {
5035 self.emit_relaxed_binop(Assembler::emit_clz, Size::S64, src, dst, true)
5036 }
5037
5038 fn i64_ctz(&mut self, src: Location, dst: Location) -> Result<(), CompileError> {
5039 let mut temps = vec![];
5040 let src = self.location_to_reg(Size::S64, src, &mut temps, ImmType::None, true, None)?;
5041 let dest = self.location_to_reg(Size::S64, dst, &mut temps, ImmType::None, false, None)?;
5042 self.assembler.emit_rbit(Size::S64, src, dest)?;
5043 self.assembler.emit_clz(Size::S64, dest, dest)?;
5044 if dst != dest {
5045 self.move_location(Size::S64, dest, dst)?;
5046 }
5047 for r in temps {
5048 self.release_gpr(r);
5049 }
5050 Ok(())
5051 }
5052
5053 fn i64_popcnt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
5054 if self.has_neon {
5055 let mut temps = vec![];
5056
5057 let src_gpr =
5058 self.location_to_reg(Size::S64, loc, &mut temps, ImmType::None, true, None)?;
5059 let dst_gpr =
5060 self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
5061
5062 let mut neon_temps = vec![];
5063 let neon_temp = self.acquire_temp_simd().ok_or_else(|| {
5064 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5065 })?;
5066 neon_temps.push(neon_temp);
5067
5068 self.assembler
5069 .emit_fmov(Size::S64, src_gpr, Size::S64, Location::SIMD(neon_temp))?;
5070 self.assembler.emit_cnt(neon_temp, neon_temp)?;
5071 self.assembler.emit_addv(neon_temp, neon_temp)?;
5072 self.assembler
5073 .emit_fmov(Size::S64, Location::SIMD(neon_temp), Size::S64, dst_gpr)?;
5074
5075 if ret != dst_gpr {
5076 self.move_location(Size::S64, dst_gpr, ret)?;
5077 }
5078
5079 for r in temps {
5080 self.release_gpr(r);
5081 }
5082
5083 for r in neon_temps {
5084 self.release_simd(r);
5085 }
5086 } else {
5087 let mut temps = vec![];
5088 let src =
5089 self.location_to_reg(Size::S64, loc, &mut temps, ImmType::None, true, None)?;
5090 let dest =
5091 self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
5092 let src = if src == loc {
5093 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
5094 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5095 })?;
5096 temps.push(tmp);
5097 self.assembler
5098 .emit_mov(Size::S64, src, Location::GPR(tmp))?;
5099 Location::GPR(tmp)
5100 } else {
5101 src
5102 };
5103 let tmp = {
5104 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
5105 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5106 })?;
5107 temps.push(tmp);
5108 Location::GPR(tmp)
5109 };
5110 let label_loop = self.assembler.get_label();
5111 let label_exit = self.assembler.get_label();
5112 self.assembler
5113 .emit_mov(Size::S32, Location::GPR(GPR::XzrSp), dest)?; self.assembler.emit_cbz_label(Size::S64, src, label_exit)?; self.assembler.emit_label(label_loop)?;
5116 self.assembler
5117 .emit_add(Size::S32, dest, Location::Imm8(1), dest)?; self.assembler.emit_clz(Size::S64, src, tmp)?; self.assembler.emit_lsl(Size::S64, src, tmp, src)?; self.assembler
5121 .emit_lsl(Size::S64, src, Location::Imm8(1), src)?; self.assembler.emit_cbnz_label(Size::S64, src, label_loop)?; self.assembler.emit_label(label_exit)?;
5124 if ret != dest {
5125 self.move_location(Size::S64, dest, ret)?;
5126 }
5127 for r in temps {
5128 self.release_gpr(r);
5129 }
5130 }
5131
5132 Ok(())
5133 }
5134
5135 fn i64_shl(
5136 &mut self,
5137 loc_a: Location,
5138 loc_b: Location,
5139 ret: Location,
5140 ) -> Result<(), CompileError> {
5141 self.emit_relaxed_binop3(
5142 Assembler::emit_lsl,
5143 Size::S64,
5144 loc_a,
5145 loc_b,
5146 ret,
5147 ImmType::Shift64No0,
5148 )
5149 }
5150
5151 fn i64_shr(
5152 &mut self,
5153 loc_a: Location,
5154 loc_b: Location,
5155 ret: Location,
5156 ) -> Result<(), CompileError> {
5157 self.emit_relaxed_binop3(
5158 Assembler::emit_lsr,
5159 Size::S64,
5160 loc_a,
5161 loc_b,
5162 ret,
5163 ImmType::Shift64No0,
5164 )
5165 }
5166
5167 fn i64_sar(
5168 &mut self,
5169 loc_a: Location,
5170 loc_b: Location,
5171 ret: Location,
5172 ) -> Result<(), CompileError> {
5173 self.emit_relaxed_binop3(
5174 Assembler::emit_asr,
5175 Size::S64,
5176 loc_a,
5177 loc_b,
5178 ret,
5179 ImmType::Shift64No0,
5180 )
5181 }
5182
5183 fn i64_rol(
5184 &mut self,
5185 loc_a: Location,
5186 loc_b: Location,
5187 ret: Location,
5188 ) -> Result<(), CompileError> {
5189 let mut temps = vec![];
5191 let src2 = match loc_b {
5192 Location::Imm8(imm) => Location::Imm8(64 - (imm & 63)),
5193 Location::Imm32(imm) => Location::Imm8(64 - (imm & 63) as u8),
5194 Location::Imm64(imm) => Location::Imm8(64 - (imm & 63) as u8),
5195 _ => {
5196 let tmp1 = self.location_to_reg(
5197 Size::S64,
5198 Location::Imm32(64),
5199 &mut temps,
5200 ImmType::None,
5201 true,
5202 None,
5203 )?;
5204 let tmp2 =
5205 self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
5206 self.assembler.emit_sub(Size::S64, tmp1, tmp2, tmp1)?;
5207 tmp1
5208 }
5209 };
5210 self.emit_relaxed_binop3(
5211 Assembler::emit_ror,
5212 Size::S64,
5213 loc_a,
5214 src2,
5215 ret,
5216 ImmType::Shift64No0,
5217 )?;
5218 for r in temps {
5219 self.release_gpr(r);
5220 }
5221 Ok(())
5222 }
5223
5224 fn i64_ror(
5225 &mut self,
5226 loc_a: Location,
5227 loc_b: Location,
5228 ret: Location,
5229 ) -> Result<(), CompileError> {
5230 self.emit_relaxed_binop3(
5231 Assembler::emit_ror,
5232 Size::S64,
5233 loc_a,
5234 loc_b,
5235 ret,
5236 ImmType::Shift64No0,
5237 )
5238 }
5239
5240 fn i64_load(
5241 &mut self,
5242 addr: Location,
5243 memarg: &MemArg,
5244 ret: Location,
5245 need_check: bool,
5246 imported_memories: bool,
5247 offset: i32,
5248 heap_access_oob: Label,
5249 unaligned_atomic: Label,
5250 ) -> Result<(), CompileError> {
5251 self.memory_op(
5252 addr,
5253 memarg,
5254 false,
5255 8,
5256 need_check,
5257 imported_memories,
5258 offset,
5259 heap_access_oob,
5260 unaligned_atomic,
5261 |this, addr| this.emit_relaxed_ldr64(Size::S64, ret, Location::Memory(addr, 0)),
5262 )
5263 }
5264
5265 fn i64_load_8u(
5266 &mut self,
5267 addr: Location,
5268 memarg: &MemArg,
5269 ret: Location,
5270 need_check: bool,
5271 imported_memories: bool,
5272 offset: i32,
5273 heap_access_oob: Label,
5274 unaligned_atomic: Label,
5275 ) -> Result<(), CompileError> {
5276 self.memory_op(
5277 addr,
5278 memarg,
5279 false,
5280 1,
5281 need_check,
5282 imported_memories,
5283 offset,
5284 heap_access_oob,
5285 unaligned_atomic,
5286 |this, addr| this.emit_relaxed_ldr8(Size::S64, ret, Location::Memory(addr, 0)),
5287 )
5288 }
5289
5290 fn i64_load_8s(
5291 &mut self,
5292 addr: Location,
5293 memarg: &MemArg,
5294 ret: Location,
5295 need_check: bool,
5296 imported_memories: bool,
5297 offset: i32,
5298 heap_access_oob: Label,
5299 unaligned_atomic: Label,
5300 ) -> Result<(), CompileError> {
5301 self.memory_op(
5302 addr,
5303 memarg,
5304 false,
5305 1,
5306 need_check,
5307 imported_memories,
5308 offset,
5309 heap_access_oob,
5310 unaligned_atomic,
5311 |this, addr| this.emit_relaxed_ldr8s(Size::S64, ret, Location::Memory(addr, 0)),
5312 )
5313 }
5314
5315 fn i64_load_16u(
5316 &mut self,
5317 addr: Location,
5318 memarg: &MemArg,
5319 ret: Location,
5320 need_check: bool,
5321 imported_memories: bool,
5322 offset: i32,
5323 heap_access_oob: Label,
5324 unaligned_atomic: Label,
5325 ) -> Result<(), CompileError> {
5326 self.memory_op(
5327 addr,
5328 memarg,
5329 false,
5330 2,
5331 need_check,
5332 imported_memories,
5333 offset,
5334 heap_access_oob,
5335 unaligned_atomic,
5336 |this, addr| this.emit_relaxed_ldr16(Size::S64, ret, Location::Memory(addr, 0)),
5337 )
5338 }
5339
5340 fn i64_load_16s(
5341 &mut self,
5342 addr: Location,
5343 memarg: &MemArg,
5344 ret: Location,
5345 need_check: bool,
5346 imported_memories: bool,
5347 offset: i32,
5348 heap_access_oob: Label,
5349 unaligned_atomic: Label,
5350 ) -> Result<(), CompileError> {
5351 self.memory_op(
5352 addr,
5353 memarg,
5354 false,
5355 2,
5356 need_check,
5357 imported_memories,
5358 offset,
5359 heap_access_oob,
5360 unaligned_atomic,
5361 |this, addr| this.emit_relaxed_ldr16s(Size::S64, ret, Location::Memory(addr, 0)),
5362 )
5363 }
5364
5365 fn i64_load_32u(
5366 &mut self,
5367 addr: Location,
5368 memarg: &MemArg,
5369 ret: Location,
5370 need_check: bool,
5371 imported_memories: bool,
5372 offset: i32,
5373 heap_access_oob: Label,
5374 unaligned_atomic: Label,
5375 ) -> Result<(), CompileError> {
5376 self.memory_op(
5377 addr,
5378 memarg,
5379 false,
5380 4,
5381 need_check,
5382 imported_memories,
5383 offset,
5384 heap_access_oob,
5385 unaligned_atomic,
5386 |this, addr| this.emit_relaxed_ldr32(Size::S64, ret, Location::Memory(addr, 0)),
5387 )
5388 }
5389
5390 fn i64_load_32s(
5391 &mut self,
5392 addr: Location,
5393 memarg: &MemArg,
5394 ret: Location,
5395 need_check: bool,
5396 imported_memories: bool,
5397 offset: i32,
5398 heap_access_oob: Label,
5399 unaligned_atomic: Label,
5400 ) -> Result<(), CompileError> {
5401 self.memory_op(
5402 addr,
5403 memarg,
5404 false,
5405 4,
5406 need_check,
5407 imported_memories,
5408 offset,
5409 heap_access_oob,
5410 unaligned_atomic,
5411 |this, addr| this.emit_relaxed_ldr32s(Size::S64, ret, Location::Memory(addr, 0)),
5412 )
5413 }
5414
5415 fn i64_atomic_load(
5416 &mut self,
5417 addr: Location,
5418 memarg: &MemArg,
5419 ret: Location,
5420 need_check: bool,
5421 imported_memories: bool,
5422 offset: i32,
5423 heap_access_oob: Label,
5424 unaligned_atomic: Label,
5425 ) -> Result<(), CompileError> {
5426 self.memory_op(
5427 addr,
5428 memarg,
5429 true,
5430 8,
5431 need_check,
5432 imported_memories,
5433 offset,
5434 heap_access_oob,
5435 unaligned_atomic,
5436 |this, addr| this.emit_relaxed_ldr64(Size::S64, ret, Location::Memory(addr, 0)),
5437 )
5438 }
5439
5440 fn i64_atomic_load_8u(
5441 &mut self,
5442 addr: Location,
5443 memarg: &MemArg,
5444 ret: Location,
5445 need_check: bool,
5446 imported_memories: bool,
5447 offset: i32,
5448 heap_access_oob: Label,
5449 unaligned_atomic: Label,
5450 ) -> Result<(), CompileError> {
5451 self.memory_op(
5452 addr,
5453 memarg,
5454 true,
5455 1,
5456 need_check,
5457 imported_memories,
5458 offset,
5459 heap_access_oob,
5460 unaligned_atomic,
5461 |this, addr| this.emit_relaxed_ldr8(Size::S64, ret, Location::Memory(addr, 0)),
5462 )
5463 }
5464
5465 fn i64_atomic_load_16u(
5466 &mut self,
5467 addr: Location,
5468 memarg: &MemArg,
5469 ret: Location,
5470 need_check: bool,
5471 imported_memories: bool,
5472 offset: i32,
5473 heap_access_oob: Label,
5474 unaligned_atomic: Label,
5475 ) -> Result<(), CompileError> {
5476 self.memory_op(
5477 addr,
5478 memarg,
5479 true,
5480 2,
5481 need_check,
5482 imported_memories,
5483 offset,
5484 heap_access_oob,
5485 unaligned_atomic,
5486 |this, addr| this.emit_relaxed_ldr16(Size::S64, ret, Location::Memory(addr, 0)),
5487 )
5488 }
5489
5490 fn i64_atomic_load_32u(
5491 &mut self,
5492 addr: Location,
5493 memarg: &MemArg,
5494 ret: Location,
5495 need_check: bool,
5496 imported_memories: bool,
5497 offset: i32,
5498 heap_access_oob: Label,
5499 unaligned_atomic: Label,
5500 ) -> Result<(), CompileError> {
5501 self.memory_op(
5502 addr,
5503 memarg,
5504 true,
5505 4,
5506 need_check,
5507 imported_memories,
5508 offset,
5509 heap_access_oob,
5510 unaligned_atomic,
5511 |this, addr| this.emit_relaxed_ldr32(Size::S64, ret, Location::Memory(addr, 0)),
5512 )
5513 }
5514
5515 fn i64_save(
5516 &mut self,
5517 target_value: Location,
5518 memarg: &MemArg,
5519 target_addr: Location,
5520 need_check: bool,
5521 imported_memories: bool,
5522 offset: i32,
5523 heap_access_oob: Label,
5524 unaligned_atomic: Label,
5525 ) -> Result<(), CompileError> {
5526 self.memory_op(
5527 target_addr,
5528 memarg,
5529 false,
5530 8,
5531 need_check,
5532 imported_memories,
5533 offset,
5534 heap_access_oob,
5535 unaligned_atomic,
5536 |this, addr| this.emit_relaxed_str64(target_value, Location::Memory(addr, 0)),
5537 )
5538 }
5539
5540 fn i64_save_8(
5541 &mut self,
5542 target_value: Location,
5543 memarg: &MemArg,
5544 target_addr: Location,
5545 need_check: bool,
5546 imported_memories: bool,
5547 offset: i32,
5548 heap_access_oob: Label,
5549 unaligned_atomic: Label,
5550 ) -> Result<(), CompileError> {
5551 self.memory_op(
5552 target_addr,
5553 memarg,
5554 false,
5555 1,
5556 need_check,
5557 imported_memories,
5558 offset,
5559 heap_access_oob,
5560 unaligned_atomic,
5561 |this, addr| this.emit_relaxed_str8(target_value, Location::Memory(addr, 0)),
5562 )
5563 }
5564
5565 fn i64_save_16(
5566 &mut self,
5567 target_value: Location,
5568 memarg: &MemArg,
5569 target_addr: Location,
5570 need_check: bool,
5571 imported_memories: bool,
5572 offset: i32,
5573 heap_access_oob: Label,
5574 unaligned_atomic: Label,
5575 ) -> Result<(), CompileError> {
5576 self.memory_op(
5577 target_addr,
5578 memarg,
5579 false,
5580 2,
5581 need_check,
5582 imported_memories,
5583 offset,
5584 heap_access_oob,
5585 unaligned_atomic,
5586 |this, addr| this.emit_relaxed_str16(target_value, Location::Memory(addr, 0)),
5587 )
5588 }
5589
5590 fn i64_save_32(
5591 &mut self,
5592 target_value: Location,
5593 memarg: &MemArg,
5594 target_addr: Location,
5595 need_check: bool,
5596 imported_memories: bool,
5597 offset: i32,
5598 heap_access_oob: Label,
5599 unaligned_atomic: Label,
5600 ) -> Result<(), CompileError> {
5601 self.memory_op(
5602 target_addr,
5603 memarg,
5604 false,
5605 4,
5606 need_check,
5607 imported_memories,
5608 offset,
5609 heap_access_oob,
5610 unaligned_atomic,
5611 |this, addr| this.emit_relaxed_str32(target_value, Location::Memory(addr, 0)),
5612 )
5613 }
5614
5615 fn i64_atomic_save(
5616 &mut self,
5617 target_value: Location,
5618 memarg: &MemArg,
5619 target_addr: Location,
5620 need_check: bool,
5621 imported_memories: bool,
5622 offset: i32,
5623 heap_access_oob: Label,
5624 unaligned_atomic: Label,
5625 ) -> Result<(), CompileError> {
5626 self.memory_op(
5627 target_addr,
5628 memarg,
5629 true,
5630 8,
5631 need_check,
5632 imported_memories,
5633 offset,
5634 heap_access_oob,
5635 unaligned_atomic,
5636 |this, addr| this.emit_relaxed_str64(target_value, Location::Memory(addr, 0)),
5637 )?;
5638 self.assembler.emit_dmb()
5639 }
5640
5641 fn i64_atomic_save_8(
5642 &mut self,
5643 target_value: Location,
5644 memarg: &MemArg,
5645 target_addr: Location,
5646 need_check: bool,
5647 imported_memories: bool,
5648 offset: i32,
5649 heap_access_oob: Label,
5650 unaligned_atomic: Label,
5651 ) -> Result<(), CompileError> {
5652 self.memory_op(
5653 target_addr,
5654 memarg,
5655 true,
5656 1,
5657 need_check,
5658 imported_memories,
5659 offset,
5660 heap_access_oob,
5661 unaligned_atomic,
5662 |this, addr| this.emit_relaxed_str8(target_value, Location::Memory(addr, 0)),
5663 )?;
5664 self.assembler.emit_dmb()
5665 }
5666
5667 fn i64_atomic_save_16(
5668 &mut self,
5669 target_value: Location,
5670 memarg: &MemArg,
5671 target_addr: Location,
5672 need_check: bool,
5673 imported_memories: bool,
5674 offset: i32,
5675 heap_access_oob: Label,
5676 unaligned_atomic: Label,
5677 ) -> Result<(), CompileError> {
5678 self.memory_op(
5679 target_addr,
5680 memarg,
5681 true,
5682 2,
5683 need_check,
5684 imported_memories,
5685 offset,
5686 heap_access_oob,
5687 unaligned_atomic,
5688 |this, addr| this.emit_relaxed_str16(target_value, Location::Memory(addr, 0)),
5689 )?;
5690 self.assembler.emit_dmb()
5691 }
5692
5693 fn i64_atomic_save_32(
5694 &mut self,
5695 target_value: Location,
5696 memarg: &MemArg,
5697 target_addr: Location,
5698 need_check: bool,
5699 imported_memories: bool,
5700 offset: i32,
5701 heap_access_oob: Label,
5702 unaligned_atomic: Label,
5703 ) -> Result<(), CompileError> {
5704 self.memory_op(
5705 target_addr,
5706 memarg,
5707 true,
5708 4,
5709 need_check,
5710 imported_memories,
5711 offset,
5712 heap_access_oob,
5713 unaligned_atomic,
5714 |this, addr| this.emit_relaxed_str32(target_value, Location::Memory(addr, 0)),
5715 )?;
5716 self.assembler.emit_dmb()
5717 }
5718
5719 fn i64_atomic_add(
5720 &mut self,
5721 loc: Location,
5722 target: Location,
5723 memarg: &MemArg,
5724 ret: Location,
5725 need_check: bool,
5726 imported_memories: bool,
5727 offset: i32,
5728 heap_access_oob: Label,
5729 unaligned_atomic: Label,
5730 ) -> Result<(), CompileError> {
5731 self.memory_op(
5732 target,
5733 memarg,
5734 true,
5735 8,
5736 need_check,
5737 imported_memories,
5738 offset,
5739 heap_access_oob,
5740 unaligned_atomic,
5741 |this, addr| {
5742 let mut temps = vec![];
5743 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
5744 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5745 })?;
5746 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
5747 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5748 })?;
5749 let dst =
5750 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
5751 let reread = this.get_label();
5752
5753 this.emit_label(reread)?;
5754 this.assembler
5755 .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?;
5756 this.emit_binop_add64(dst, loc, Location::GPR(tmp1))?;
5757 this.assembler.emit_stlxr(
5758 Size::S64,
5759 Location::GPR(tmp2),
5760 Location::GPR(tmp1),
5761 Location::GPR(addr),
5762 )?;
5763 this.assembler
5764 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
5765 this.assembler.emit_dmb()?;
5766
5767 if dst != ret {
5768 this.move_location(Size::S64, ret, dst)?;
5769 }
5770 for r in temps {
5771 this.release_gpr(r);
5772 }
5773 this.release_gpr(tmp1);
5774 this.release_gpr(tmp2);
5775 Ok(())
5776 },
5777 )
5778 }
5779
5780 fn i64_atomic_add_8u(
5781 &mut self,
5782 loc: Location,
5783 target: Location,
5784 memarg: &MemArg,
5785 ret: Location,
5786 need_check: bool,
5787 imported_memories: bool,
5788 offset: i32,
5789 heap_access_oob: Label,
5790 unaligned_atomic: Label,
5791 ) -> Result<(), CompileError> {
5792 self.memory_op(
5793 target,
5794 memarg,
5795 true,
5796 1,
5797 need_check,
5798 imported_memories,
5799 offset,
5800 heap_access_oob,
5801 unaligned_atomic,
5802 |this, addr| {
5803 let mut temps = vec![];
5804 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
5805 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5806 })?;
5807 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
5808 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5809 })?;
5810 let dst =
5811 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
5812 let reread = this.get_label();
5813
5814 this.emit_label(reread)?;
5815 this.assembler
5816 .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?;
5817 this.emit_binop_add64(dst, loc, Location::GPR(tmp1))?;
5818 this.assembler.emit_stlxrb(
5819 Size::S64,
5820 Location::GPR(tmp2),
5821 Location::GPR(tmp1),
5822 Location::GPR(addr),
5823 )?;
5824 this.assembler
5825 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
5826 this.assembler.emit_dmb()?;
5827
5828 if dst != ret {
5829 this.move_location(Size::S64, ret, dst)?;
5830 }
5831 for r in temps {
5832 this.release_gpr(r);
5833 }
5834 this.release_gpr(tmp1);
5835 this.release_gpr(tmp2);
5836 Ok(())
5837 },
5838 )
5839 }
5840
5841 fn i64_atomic_add_16u(
5842 &mut self,
5843 loc: Location,
5844 target: Location,
5845 memarg: &MemArg,
5846 ret: Location,
5847 need_check: bool,
5848 imported_memories: bool,
5849 offset: i32,
5850 heap_access_oob: Label,
5851 unaligned_atomic: Label,
5852 ) -> Result<(), CompileError> {
5853 self.memory_op(
5854 target,
5855 memarg,
5856 true,
5857 2,
5858 need_check,
5859 imported_memories,
5860 offset,
5861 heap_access_oob,
5862 unaligned_atomic,
5863 |this, addr| {
5864 let mut temps = vec![];
5865 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
5866 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5867 })?;
5868 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
5869 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5870 })?;
5871 let dst =
5872 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
5873 let reread = this.get_label();
5874
5875 this.emit_label(reread)?;
5876 this.assembler
5877 .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?;
5878 this.emit_binop_add64(dst, loc, Location::GPR(tmp1))?;
5879 this.assembler.emit_stlxrh(
5880 Size::S64,
5881 Location::GPR(tmp2),
5882 Location::GPR(tmp1),
5883 Location::GPR(addr),
5884 )?;
5885 this.assembler
5886 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
5887 this.assembler.emit_dmb()?;
5888
5889 if dst != ret {
5890 this.move_location(Size::S64, ret, dst)?;
5891 }
5892 for r in temps {
5893 this.release_gpr(r);
5894 }
5895 this.release_gpr(tmp1);
5896 this.release_gpr(tmp2);
5897 Ok(())
5898 },
5899 )
5900 }
5901
5902 fn i64_atomic_add_32u(
5903 &mut self,
5904 loc: Location,
5905 target: Location,
5906 memarg: &MemArg,
5907 ret: Location,
5908 need_check: bool,
5909 imported_memories: bool,
5910 offset: i32,
5911 heap_access_oob: Label,
5912 unaligned_atomic: Label,
5913 ) -> Result<(), CompileError> {
5914 self.memory_op(
5915 target,
5916 memarg,
5917 true,
5918 4,
5919 need_check,
5920 imported_memories,
5921 offset,
5922 heap_access_oob,
5923 unaligned_atomic,
5924 |this, addr| {
5925 let mut temps = vec![];
5926 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
5927 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5928 })?;
5929 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
5930 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5931 })?;
5932 let dst =
5933 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
5934 let reread = this.get_label();
5935
5936 this.emit_label(reread)?;
5937 this.assembler
5938 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
5939 this.emit_binop_add64(dst, loc, Location::GPR(tmp1))?;
5940 this.assembler.emit_stlxr(
5941 Size::S32,
5942 Location::GPR(tmp2),
5943 Location::GPR(tmp1),
5944 Location::GPR(addr),
5945 )?;
5946 this.assembler
5947 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
5948 this.assembler.emit_dmb()?;
5949
5950 if dst != ret {
5951 this.move_location(Size::S64, ret, dst)?;
5952 }
5953 for r in temps {
5954 this.release_gpr(r);
5955 }
5956 this.release_gpr(tmp1);
5957 this.release_gpr(tmp2);
5958 Ok(())
5959 },
5960 )
5961 }
5962
5963 fn i64_atomic_sub(
5964 &mut self,
5965 loc: Location,
5966 target: Location,
5967 memarg: &MemArg,
5968 ret: Location,
5969 need_check: bool,
5970 imported_memories: bool,
5971 offset: i32,
5972 heap_access_oob: Label,
5973 unaligned_atomic: Label,
5974 ) -> Result<(), CompileError> {
5975 self.memory_op(
5976 target,
5977 memarg,
5978 true,
5979 8,
5980 need_check,
5981 imported_memories,
5982 offset,
5983 heap_access_oob,
5984 unaligned_atomic,
5985 |this, addr| {
5986 let mut temps = vec![];
5987 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
5988 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5989 })?;
5990 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
5991 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5992 })?;
5993 let dst =
5994 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
5995 let reread = this.get_label();
5996
5997 this.emit_label(reread)?;
5998 this.assembler
5999 .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?;
6000 this.emit_binop_sub64(dst, loc, Location::GPR(tmp1))?;
6001 this.assembler.emit_stlxr(
6002 Size::S64,
6003 Location::GPR(tmp2),
6004 Location::GPR(tmp1),
6005 Location::GPR(addr),
6006 )?;
6007 this.assembler
6008 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6009 this.assembler.emit_dmb()?;
6010
6011 if dst != ret {
6012 this.move_location(Size::S64, ret, dst)?;
6013 }
6014 for r in temps {
6015 this.release_gpr(r);
6016 }
6017 this.release_gpr(tmp1);
6018 this.release_gpr(tmp2);
6019 Ok(())
6020 },
6021 )
6022 }
6023
6024 fn i64_atomic_sub_8u(
6025 &mut self,
6026 loc: Location,
6027 target: Location,
6028 memarg: &MemArg,
6029 ret: Location,
6030 need_check: bool,
6031 imported_memories: bool,
6032 offset: i32,
6033 heap_access_oob: Label,
6034 unaligned_atomic: Label,
6035 ) -> Result<(), CompileError> {
6036 self.memory_op(
6037 target,
6038 memarg,
6039 true,
6040 1,
6041 need_check,
6042 imported_memories,
6043 offset,
6044 heap_access_oob,
6045 unaligned_atomic,
6046 |this, addr| {
6047 let mut temps = vec![];
6048 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6049 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6050 })?;
6051 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6052 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6053 })?;
6054 let dst =
6055 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6056 let reread = this.get_label();
6057
6058 this.emit_label(reread)?;
6059 this.assembler
6060 .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?;
6061 this.emit_binop_sub64(dst, loc, Location::GPR(tmp1))?;
6062 this.assembler.emit_stlxrb(
6063 Size::S64,
6064 Location::GPR(tmp2),
6065 Location::GPR(tmp1),
6066 Location::GPR(addr),
6067 )?;
6068 this.assembler
6069 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6070 this.assembler.emit_dmb()?;
6071
6072 if dst != ret {
6073 this.move_location(Size::S64, ret, dst)?;
6074 }
6075 for r in temps {
6076 this.release_gpr(r);
6077 }
6078 this.release_gpr(tmp1);
6079 this.release_gpr(tmp2);
6080 Ok(())
6081 },
6082 )
6083 }
6084
6085 fn i64_atomic_sub_16u(
6086 &mut self,
6087 loc: Location,
6088 target: Location,
6089 memarg: &MemArg,
6090 ret: Location,
6091 need_check: bool,
6092 imported_memories: bool,
6093 offset: i32,
6094 heap_access_oob: Label,
6095 unaligned_atomic: Label,
6096 ) -> Result<(), CompileError> {
6097 self.memory_op(
6098 target,
6099 memarg,
6100 true,
6101 2,
6102 need_check,
6103 imported_memories,
6104 offset,
6105 heap_access_oob,
6106 unaligned_atomic,
6107 |this, addr| {
6108 let mut temps = vec![];
6109 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6110 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6111 })?;
6112 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6113 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6114 })?;
6115 let dst =
6116 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6117 let reread = this.get_label();
6118
6119 this.emit_label(reread)?;
6120 this.assembler
6121 .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?;
6122 this.emit_binop_sub64(dst, loc, Location::GPR(tmp1))?;
6123 this.assembler.emit_stlxrh(
6124 Size::S64,
6125 Location::GPR(tmp2),
6126 Location::GPR(tmp1),
6127 Location::GPR(addr),
6128 )?;
6129 this.assembler
6130 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6131 this.assembler.emit_dmb()?;
6132
6133 if dst != ret {
6134 this.move_location(Size::S64, ret, dst)?;
6135 }
6136 for r in temps {
6137 this.release_gpr(r);
6138 }
6139 this.release_gpr(tmp1);
6140 this.release_gpr(tmp2);
6141 Ok(())
6142 },
6143 )
6144 }
6145
6146 fn i64_atomic_sub_32u(
6147 &mut self,
6148 loc: Location,
6149 target: Location,
6150 memarg: &MemArg,
6151 ret: Location,
6152 need_check: bool,
6153 imported_memories: bool,
6154 offset: i32,
6155 heap_access_oob: Label,
6156 unaligned_atomic: Label,
6157 ) -> Result<(), CompileError> {
6158 self.memory_op(
6159 target,
6160 memarg,
6161 true,
6162 4,
6163 need_check,
6164 imported_memories,
6165 offset,
6166 heap_access_oob,
6167 unaligned_atomic,
6168 |this, addr| {
6169 let mut temps = vec![];
6170 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6171 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6172 })?;
6173 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6174 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6175 })?;
6176 let dst =
6177 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6178 let reread = this.get_label();
6179
6180 this.emit_label(reread)?;
6181 this.assembler
6182 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
6183 this.emit_binop_sub64(dst, loc, Location::GPR(tmp1))?;
6184 this.assembler.emit_stlxr(
6185 Size::S32,
6186 Location::GPR(tmp2),
6187 Location::GPR(tmp1),
6188 Location::GPR(addr),
6189 )?;
6190 this.assembler
6191 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6192 this.assembler.emit_dmb()?;
6193
6194 if dst != ret {
6195 this.move_location(Size::S64, ret, dst)?;
6196 }
6197 for r in temps {
6198 this.release_gpr(r);
6199 }
6200 this.release_gpr(tmp1);
6201 this.release_gpr(tmp2);
6202 Ok(())
6203 },
6204 )
6205 }
6206
6207 fn i64_atomic_and(
6208 &mut self,
6209 loc: Location,
6210 target: Location,
6211 memarg: &MemArg,
6212 ret: Location,
6213 need_check: bool,
6214 imported_memories: bool,
6215 offset: i32,
6216 heap_access_oob: Label,
6217 unaligned_atomic: Label,
6218 ) -> Result<(), CompileError> {
6219 self.memory_op(
6220 target,
6221 memarg,
6222 true,
6223 8,
6224 need_check,
6225 imported_memories,
6226 offset,
6227 heap_access_oob,
6228 unaligned_atomic,
6229 |this, addr| {
6230 let mut temps = vec![];
6231 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6232 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6233 })?;
6234 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6235 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6236 })?;
6237 let dst =
6238 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6239 let reread = this.get_label();
6240
6241 this.emit_label(reread)?;
6242 this.assembler
6243 .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?;
6244 this.emit_binop_and64(dst, loc, Location::GPR(tmp1))?;
6245 this.assembler.emit_stlxr(
6246 Size::S64,
6247 Location::GPR(tmp2),
6248 Location::GPR(tmp1),
6249 Location::GPR(addr),
6250 )?;
6251 this.assembler
6252 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6253 this.assembler.emit_dmb()?;
6254
6255 if dst != ret {
6256 this.move_location(Size::S64, ret, dst)?;
6257 }
6258 for r in temps {
6259 this.release_gpr(r);
6260 }
6261 this.release_gpr(tmp1);
6262 this.release_gpr(tmp2);
6263 Ok(())
6264 },
6265 )
6266 }
6267
6268 fn i64_atomic_and_8u(
6269 &mut self,
6270 loc: Location,
6271 target: Location,
6272 memarg: &MemArg,
6273 ret: Location,
6274 need_check: bool,
6275 imported_memories: bool,
6276 offset: i32,
6277 heap_access_oob: Label,
6278 unaligned_atomic: Label,
6279 ) -> Result<(), CompileError> {
6280 self.memory_op(
6281 target,
6282 memarg,
6283 true,
6284 1,
6285 need_check,
6286 imported_memories,
6287 offset,
6288 heap_access_oob,
6289 unaligned_atomic,
6290 |this, addr| {
6291 let mut temps = vec![];
6292 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6293 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6294 })?;
6295 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6296 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6297 })?;
6298 let dst =
6299 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6300 let reread = this.get_label();
6301
6302 this.emit_label(reread)?;
6303 this.assembler
6304 .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?;
6305 this.emit_binop_and64(dst, loc, Location::GPR(tmp1))?;
6306 this.assembler.emit_stlxrb(
6307 Size::S64,
6308 Location::GPR(tmp2),
6309 Location::GPR(tmp1),
6310 Location::GPR(addr),
6311 )?;
6312 this.assembler
6313 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6314 this.assembler.emit_dmb()?;
6315
6316 if dst != ret {
6317 this.move_location(Size::S64, ret, dst)?;
6318 }
6319 for r in temps {
6320 this.release_gpr(r);
6321 }
6322 this.release_gpr(tmp1);
6323 this.release_gpr(tmp2);
6324 Ok(())
6325 },
6326 )
6327 }
6328
6329 fn i64_atomic_and_16u(
6330 &mut self,
6331 loc: Location,
6332 target: Location,
6333 memarg: &MemArg,
6334 ret: Location,
6335 need_check: bool,
6336 imported_memories: bool,
6337 offset: i32,
6338 heap_access_oob: Label,
6339 unaligned_atomic: Label,
6340 ) -> Result<(), CompileError> {
6341 self.memory_op(
6342 target,
6343 memarg,
6344 true,
6345 2,
6346 need_check,
6347 imported_memories,
6348 offset,
6349 heap_access_oob,
6350 unaligned_atomic,
6351 |this, addr| {
6352 let mut temps = vec![];
6353 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6354 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6355 })?;
6356 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6357 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6358 })?;
6359 let dst =
6360 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6361 let reread = this.get_label();
6362
6363 this.emit_label(reread)?;
6364 this.assembler
6365 .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?;
6366 this.emit_binop_and64(dst, loc, Location::GPR(tmp1))?;
6367 this.assembler.emit_stlxrh(
6368 Size::S64,
6369 Location::GPR(tmp2),
6370 Location::GPR(tmp1),
6371 Location::GPR(addr),
6372 )?;
6373 this.assembler
6374 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6375 this.assembler.emit_dmb()?;
6376
6377 if dst != ret {
6378 this.move_location(Size::S64, ret, dst)?;
6379 }
6380 for r in temps {
6381 this.release_gpr(r);
6382 }
6383 this.release_gpr(tmp1);
6384 this.release_gpr(tmp2);
6385 Ok(())
6386 },
6387 )
6388 }
6389
6390 fn i64_atomic_and_32u(
6391 &mut self,
6392 loc: Location,
6393 target: Location,
6394 memarg: &MemArg,
6395 ret: Location,
6396 need_check: bool,
6397 imported_memories: bool,
6398 offset: i32,
6399 heap_access_oob: Label,
6400 unaligned_atomic: Label,
6401 ) -> Result<(), CompileError> {
6402 self.memory_op(
6403 target,
6404 memarg,
6405 true,
6406 4,
6407 need_check,
6408 imported_memories,
6409 offset,
6410 heap_access_oob,
6411 unaligned_atomic,
6412 |this, addr| {
6413 let mut temps = vec![];
6414 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6415 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6416 })?;
6417 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6418 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6419 })?;
6420 let dst =
6421 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6422 let reread = this.get_label();
6423
6424 this.emit_label(reread)?;
6425 this.assembler
6426 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
6427 this.emit_binop_and64(dst, loc, Location::GPR(tmp1))?;
6428 this.assembler.emit_stlxr(
6429 Size::S32,
6430 Location::GPR(tmp2),
6431 Location::GPR(tmp1),
6432 Location::GPR(addr),
6433 )?;
6434 this.assembler
6435 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6436 this.assembler.emit_dmb()?;
6437
6438 if dst != ret {
6439 this.move_location(Size::S64, ret, dst)?;
6440 }
6441 for r in temps {
6442 this.release_gpr(r);
6443 }
6444 this.release_gpr(tmp1);
6445 this.release_gpr(tmp2);
6446 Ok(())
6447 },
6448 )
6449 }
6450
6451 fn i64_atomic_or(
6452 &mut self,
6453 loc: Location,
6454 target: Location,
6455 memarg: &MemArg,
6456 ret: Location,
6457 need_check: bool,
6458 imported_memories: bool,
6459 offset: i32,
6460 heap_access_oob: Label,
6461 unaligned_atomic: Label,
6462 ) -> Result<(), CompileError> {
6463 self.memory_op(
6464 target,
6465 memarg,
6466 true,
6467 8,
6468 need_check,
6469 imported_memories,
6470 offset,
6471 heap_access_oob,
6472 unaligned_atomic,
6473 |this, addr| {
6474 let mut temps = vec![];
6475 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6476 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6477 })?;
6478 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6479 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6480 })?;
6481 let dst =
6482 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6483 let reread = this.get_label();
6484
6485 this.emit_label(reread)?;
6486 this.assembler
6487 .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?;
6488 this.emit_binop_or64(dst, loc, Location::GPR(tmp1))?;
6489 this.assembler.emit_stlxr(
6490 Size::S64,
6491 Location::GPR(tmp2),
6492 Location::GPR(tmp1),
6493 Location::GPR(addr),
6494 )?;
6495 this.assembler
6496 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6497 this.assembler.emit_dmb()?;
6498
6499 if dst != ret {
6500 this.move_location(Size::S64, ret, dst)?;
6501 }
6502 for r in temps {
6503 this.release_gpr(r);
6504 }
6505 this.release_gpr(tmp1);
6506 this.release_gpr(tmp2);
6507 Ok(())
6508 },
6509 )
6510 }
6511
6512 fn i64_atomic_or_8u(
6513 &mut self,
6514 loc: Location,
6515 target: Location,
6516 memarg: &MemArg,
6517 ret: Location,
6518 need_check: bool,
6519 imported_memories: bool,
6520 offset: i32,
6521 heap_access_oob: Label,
6522 unaligned_atomic: Label,
6523 ) -> Result<(), CompileError> {
6524 self.memory_op(
6525 target,
6526 memarg,
6527 true,
6528 1,
6529 need_check,
6530 imported_memories,
6531 offset,
6532 heap_access_oob,
6533 unaligned_atomic,
6534 |this, addr| {
6535 let mut temps = vec![];
6536 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6537 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6538 })?;
6539 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6540 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6541 })?;
6542 let dst =
6543 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6544 let reread = this.get_label();
6545
6546 this.emit_label(reread)?;
6547 this.assembler
6548 .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?;
6549 this.emit_binop_or64(dst, loc, Location::GPR(tmp1))?;
6550 this.assembler.emit_stlxrb(
6551 Size::S64,
6552 Location::GPR(tmp2),
6553 Location::GPR(tmp1),
6554 Location::GPR(addr),
6555 )?;
6556 this.assembler
6557 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6558 this.assembler.emit_dmb()?;
6559
6560 if dst != ret {
6561 this.move_location(Size::S64, ret, dst)?;
6562 }
6563 for r in temps {
6564 this.release_gpr(r);
6565 }
6566 this.release_gpr(tmp1);
6567 this.release_gpr(tmp2);
6568 Ok(())
6569 },
6570 )
6571 }
6572
6573 fn i64_atomic_or_16u(
6574 &mut self,
6575 loc: Location,
6576 target: Location,
6577 memarg: &MemArg,
6578 ret: Location,
6579 need_check: bool,
6580 imported_memories: bool,
6581 offset: i32,
6582 heap_access_oob: Label,
6583 unaligned_atomic: Label,
6584 ) -> Result<(), CompileError> {
6585 self.memory_op(
6586 target,
6587 memarg,
6588 true,
6589 2,
6590 need_check,
6591 imported_memories,
6592 offset,
6593 heap_access_oob,
6594 unaligned_atomic,
6595 |this, addr| {
6596 let mut temps = vec![];
6597 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6598 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6599 })?;
6600 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6601 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6602 })?;
6603 let dst =
6604 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6605 let reread = this.get_label();
6606
6607 this.emit_label(reread)?;
6608 this.assembler
6609 .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?;
6610 this.emit_binop_or64(dst, loc, Location::GPR(tmp1))?;
6611 this.assembler.emit_stlxrh(
6612 Size::S64,
6613 Location::GPR(tmp2),
6614 Location::GPR(tmp1),
6615 Location::GPR(addr),
6616 )?;
6617 this.assembler
6618 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6619 this.assembler.emit_dmb()?;
6620
6621 if dst != ret {
6622 this.move_location(Size::S64, ret, dst)?;
6623 }
6624 for r in temps {
6625 this.release_gpr(r);
6626 }
6627 this.release_gpr(tmp1);
6628 this.release_gpr(tmp2);
6629 Ok(())
6630 },
6631 )
6632 }
6633
6634 fn i64_atomic_or_32u(
6635 &mut self,
6636 loc: Location,
6637 target: Location,
6638 memarg: &MemArg,
6639 ret: Location,
6640 need_check: bool,
6641 imported_memories: bool,
6642 offset: i32,
6643 heap_access_oob: Label,
6644 unaligned_atomic: Label,
6645 ) -> Result<(), CompileError> {
6646 self.memory_op(
6647 target,
6648 memarg,
6649 true,
6650 4,
6651 need_check,
6652 imported_memories,
6653 offset,
6654 heap_access_oob,
6655 unaligned_atomic,
6656 |this, addr| {
6657 let mut temps = vec![];
6658 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6659 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6660 })?;
6661 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6662 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6663 })?;
6664 let dst =
6665 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6666 let reread = this.get_label();
6667
6668 this.emit_label(reread)?;
6669 this.assembler
6670 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
6671 this.emit_binop_or64(dst, loc, Location::GPR(tmp1))?;
6672 this.assembler.emit_stlxr(
6673 Size::S32,
6674 Location::GPR(tmp2),
6675 Location::GPR(tmp1),
6676 Location::GPR(addr),
6677 )?;
6678 this.assembler
6679 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6680 this.assembler.emit_dmb()?;
6681
6682 if dst != ret {
6683 this.move_location(Size::S64, ret, dst)?;
6684 }
6685 for r in temps {
6686 this.release_gpr(r);
6687 }
6688 this.release_gpr(tmp1);
6689 this.release_gpr(tmp2);
6690 Ok(())
6691 },
6692 )
6693 }
6694
6695 fn i64_atomic_xor(
6696 &mut self,
6697 loc: Location,
6698 target: Location,
6699 memarg: &MemArg,
6700 ret: Location,
6701 need_check: bool,
6702 imported_memories: bool,
6703 offset: i32,
6704 heap_access_oob: Label,
6705 unaligned_atomic: Label,
6706 ) -> Result<(), CompileError> {
6707 self.memory_op(
6708 target,
6709 memarg,
6710 true,
6711 8,
6712 need_check,
6713 imported_memories,
6714 offset,
6715 heap_access_oob,
6716 unaligned_atomic,
6717 |this, addr| {
6718 let mut temps = vec![];
6719 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6720 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6721 })?;
6722 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6723 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6724 })?;
6725 let dst =
6726 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6727 let reread = this.get_label();
6728
6729 this.emit_label(reread)?;
6730 this.assembler
6731 .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?;
6732 this.emit_binop_xor64(dst, loc, Location::GPR(tmp1))?;
6733 this.assembler.emit_stlxr(
6734 Size::S64,
6735 Location::GPR(tmp2),
6736 Location::GPR(tmp1),
6737 Location::GPR(addr),
6738 )?;
6739 this.assembler
6740 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6741 this.assembler.emit_dmb()?;
6742
6743 if dst != ret {
6744 this.move_location(Size::S64, ret, dst)?;
6745 }
6746 for r in temps {
6747 this.release_gpr(r);
6748 }
6749 this.release_gpr(tmp1);
6750 this.release_gpr(tmp2);
6751 Ok(())
6752 },
6753 )
6754 }
6755
6756 fn i64_atomic_xor_8u(
6757 &mut self,
6758 loc: Location,
6759 target: Location,
6760 memarg: &MemArg,
6761 ret: Location,
6762 need_check: bool,
6763 imported_memories: bool,
6764 offset: i32,
6765 heap_access_oob: Label,
6766 unaligned_atomic: Label,
6767 ) -> Result<(), CompileError> {
6768 self.memory_op(
6769 target,
6770 memarg,
6771 true,
6772 1,
6773 need_check,
6774 imported_memories,
6775 offset,
6776 heap_access_oob,
6777 unaligned_atomic,
6778 |this, addr| {
6779 let mut temps = vec![];
6780 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6781 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6782 })?;
6783 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6784 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6785 })?;
6786 let dst =
6787 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6788 let reread = this.get_label();
6789
6790 this.emit_label(reread)?;
6791 this.assembler
6792 .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?;
6793 this.emit_binop_xor64(dst, loc, Location::GPR(tmp1))?;
6794 this.assembler.emit_stlxrb(
6795 Size::S64,
6796 Location::GPR(tmp2),
6797 Location::GPR(tmp1),
6798 Location::GPR(addr),
6799 )?;
6800 this.assembler
6801 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6802 this.assembler.emit_dmb()?;
6803
6804 if dst != ret {
6805 this.move_location(Size::S64, ret, dst)?;
6806 }
6807 for r in temps {
6808 this.release_gpr(r);
6809 }
6810 this.release_gpr(tmp1);
6811 this.release_gpr(tmp2);
6812 Ok(())
6813 },
6814 )
6815 }
6816
6817 fn i64_atomic_xor_16u(
6818 &mut self,
6819 loc: Location,
6820 target: Location,
6821 memarg: &MemArg,
6822 ret: Location,
6823 need_check: bool,
6824 imported_memories: bool,
6825 offset: i32,
6826 heap_access_oob: Label,
6827 unaligned_atomic: Label,
6828 ) -> Result<(), CompileError> {
6829 self.memory_op(
6830 target,
6831 memarg,
6832 true,
6833 2,
6834 need_check,
6835 imported_memories,
6836 offset,
6837 heap_access_oob,
6838 unaligned_atomic,
6839 |this, addr| {
6840 let mut temps = vec![];
6841 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6842 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6843 })?;
6844 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6845 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6846 })?;
6847 let dst =
6848 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6849 let reread = this.get_label();
6850
6851 this.emit_label(reread)?;
6852 this.assembler
6853 .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?;
6854 this.emit_binop_xor64(dst, loc, Location::GPR(tmp1))?;
6855 this.assembler.emit_stlxrh(
6856 Size::S64,
6857 Location::GPR(tmp2),
6858 Location::GPR(tmp1),
6859 Location::GPR(addr),
6860 )?;
6861 this.assembler
6862 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6863 this.assembler.emit_dmb()?;
6864
6865 if dst != ret {
6866 this.move_location(Size::S64, ret, dst)?;
6867 }
6868 for r in temps {
6869 this.release_gpr(r);
6870 }
6871 this.release_gpr(tmp1);
6872 this.release_gpr(tmp2);
6873 Ok(())
6874 },
6875 )
6876 }
6877
6878 fn i64_atomic_xor_32u(
6879 &mut self,
6880 loc: Location,
6881 target: Location,
6882 memarg: &MemArg,
6883 ret: Location,
6884 need_check: bool,
6885 imported_memories: bool,
6886 offset: i32,
6887 heap_access_oob: Label,
6888 unaligned_atomic: Label,
6889 ) -> Result<(), CompileError> {
6890 self.memory_op(
6891 target,
6892 memarg,
6893 true,
6894 4,
6895 need_check,
6896 imported_memories,
6897 offset,
6898 heap_access_oob,
6899 unaligned_atomic,
6900 |this, addr| {
6901 let mut temps = vec![];
6902 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6903 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6904 })?;
6905 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6906 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6907 })?;
6908 let dst =
6909 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6910 let reread = this.get_label();
6911
6912 this.emit_label(reread)?;
6913 this.assembler
6914 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
6915 this.emit_binop_xor64(dst, loc, Location::GPR(tmp1))?;
6916 this.assembler.emit_stlxr(
6917 Size::S32,
6918 Location::GPR(tmp2),
6919 Location::GPR(tmp1),
6920 Location::GPR(addr),
6921 )?;
6922 this.assembler
6923 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6924 this.assembler.emit_dmb()?;
6925
6926 if dst != ret {
6927 this.move_location(Size::S64, ret, dst)?;
6928 }
6929 for r in temps {
6930 this.release_gpr(r);
6931 }
6932 this.release_gpr(tmp1);
6933 this.release_gpr(tmp2);
6934 Ok(())
6935 },
6936 )
6937 }
6938
6939 fn i64_atomic_xchg(
6940 &mut self,
6941 loc: Location,
6942 target: Location,
6943 memarg: &MemArg,
6944 ret: Location,
6945 need_check: bool,
6946 imported_memories: bool,
6947 offset: i32,
6948 heap_access_oob: Label,
6949 unaligned_atomic: Label,
6950 ) -> Result<(), CompileError> {
6951 self.memory_op(
6952 target,
6953 memarg,
6954 true,
6955 8,
6956 need_check,
6957 imported_memories,
6958 offset,
6959 heap_access_oob,
6960 unaligned_atomic,
6961 |this, addr| {
6962 let mut temps = vec![];
6963 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
6964 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6965 })?;
6966 let dst =
6967 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6968 let org =
6969 this.location_to_reg(Size::S64, loc, &mut temps, ImmType::None, false, None)?;
6970 let reread = this.get_label();
6971
6972 this.emit_label(reread)?;
6973 this.assembler
6974 .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?;
6975 this.assembler.emit_stlxr(
6976 Size::S64,
6977 Location::GPR(tmp),
6978 org,
6979 Location::GPR(addr),
6980 )?;
6981 this.assembler
6982 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
6983 this.assembler.emit_dmb()?;
6984
6985 if dst != ret {
6986 this.move_location(Size::S64, ret, dst)?;
6987 }
6988 for r in temps {
6989 this.release_gpr(r);
6990 }
6991 this.release_gpr(tmp);
6992 Ok(())
6993 },
6994 )
6995 }
6996
6997 fn i64_atomic_xchg_8u(
6998 &mut self,
6999 loc: Location,
7000 target: Location,
7001 memarg: &MemArg,
7002 ret: Location,
7003 need_check: bool,
7004 imported_memories: bool,
7005 offset: i32,
7006 heap_access_oob: Label,
7007 unaligned_atomic: Label,
7008 ) -> Result<(), CompileError> {
7009 self.memory_op(
7010 target,
7011 memarg,
7012 true,
7013 1,
7014 need_check,
7015 imported_memories,
7016 offset,
7017 heap_access_oob,
7018 unaligned_atomic,
7019 |this, addr| {
7020 let mut temps = vec![];
7021 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
7022 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7023 })?;
7024 let dst =
7025 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7026 let org =
7027 this.location_to_reg(Size::S64, loc, &mut temps, ImmType::None, false, None)?;
7028 let reread = this.get_label();
7029
7030 this.emit_label(reread)?;
7031 this.assembler
7032 .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?;
7033 this.assembler.emit_stlxrb(
7034 Size::S64,
7035 Location::GPR(tmp),
7036 org,
7037 Location::GPR(addr),
7038 )?;
7039 this.assembler
7040 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7041 this.assembler.emit_dmb()?;
7042
7043 if dst != ret {
7044 this.move_location(Size::S64, ret, dst)?;
7045 }
7046 for r in temps {
7047 this.release_gpr(r);
7048 }
7049 this.release_gpr(tmp);
7050 Ok(())
7051 },
7052 )
7053 }
7054
7055 fn i64_atomic_xchg_16u(
7056 &mut self,
7057 loc: Location,
7058 target: Location,
7059 memarg: &MemArg,
7060 ret: Location,
7061 need_check: bool,
7062 imported_memories: bool,
7063 offset: i32,
7064 heap_access_oob: Label,
7065 unaligned_atomic: Label,
7066 ) -> Result<(), CompileError> {
7067 self.memory_op(
7068 target,
7069 memarg,
7070 true,
7071 2,
7072 need_check,
7073 imported_memories,
7074 offset,
7075 heap_access_oob,
7076 unaligned_atomic,
7077 |this, addr| {
7078 let mut temps = vec![];
7079 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
7080 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7081 })?;
7082 let dst =
7083 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7084 let org =
7085 this.location_to_reg(Size::S64, loc, &mut temps, ImmType::None, false, None)?;
7086 let reread = this.get_label();
7087
7088 this.emit_label(reread)?;
7089 this.assembler
7090 .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?;
7091 this.assembler.emit_stlxrh(
7092 Size::S64,
7093 Location::GPR(tmp),
7094 org,
7095 Location::GPR(addr),
7096 )?;
7097 this.assembler
7098 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7099 this.assembler.emit_dmb()?;
7100
7101 if dst != ret {
7102 this.move_location(Size::S64, ret, dst)?;
7103 }
7104 for r in temps {
7105 this.release_gpr(r);
7106 }
7107 this.release_gpr(tmp);
7108 Ok(())
7109 },
7110 )
7111 }
7112
7113 fn i64_atomic_xchg_32u(
7114 &mut self,
7115 loc: Location,
7116 target: Location,
7117 memarg: &MemArg,
7118 ret: Location,
7119 need_check: bool,
7120 imported_memories: bool,
7121 offset: i32,
7122 heap_access_oob: Label,
7123 unaligned_atomic: Label,
7124 ) -> Result<(), CompileError> {
7125 self.memory_op(
7126 target,
7127 memarg,
7128 true,
7129 4,
7130 need_check,
7131 imported_memories,
7132 offset,
7133 heap_access_oob,
7134 unaligned_atomic,
7135 |this, addr| {
7136 let mut temps = vec![];
7137 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
7138 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7139 })?;
7140 let dst =
7141 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7142 let org =
7143 this.location_to_reg(Size::S64, loc, &mut temps, ImmType::None, false, None)?;
7144 let reread = this.get_label();
7145
7146 this.emit_label(reread)?;
7147 this.assembler
7148 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
7149 this.assembler.emit_stlxr(
7150 Size::S32,
7151 Location::GPR(tmp),
7152 org,
7153 Location::GPR(addr),
7154 )?;
7155 this.assembler
7156 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7157 this.assembler.emit_dmb()?;
7158
7159 if dst != ret {
7160 this.move_location(Size::S64, ret, dst)?;
7161 }
7162 for r in temps {
7163 this.release_gpr(r);
7164 }
7165 this.release_gpr(tmp);
7166 Ok(())
7167 },
7168 )
7169 }
7170
7171 fn i64_atomic_cmpxchg(
7172 &mut self,
7173 new: Location,
7174 cmp: Location,
7175 target: Location,
7176 memarg: &MemArg,
7177 ret: Location,
7178 need_check: bool,
7179 imported_memories: bool,
7180 offset: i32,
7181 heap_access_oob: Label,
7182 unaligned_atomic: Label,
7183 ) -> Result<(), CompileError> {
7184 self.memory_op(
7185 target,
7186 memarg,
7187 true,
7188 8,
7189 need_check,
7190 imported_memories,
7191 offset,
7192 heap_access_oob,
7193 unaligned_atomic,
7194 |this, addr| {
7195 let mut temps = vec![];
7196 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
7197 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7198 })?;
7199 let dst =
7200 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7201 let org =
7202 this.location_to_reg(Size::S64, new, &mut temps, ImmType::None, false, None)?;
7203 let reread = this.get_label();
7204 let nosame = this.get_label();
7205
7206 this.emit_label(reread)?;
7207 this.assembler
7208 .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?;
7209 this.emit_relaxed_cmp(Size::S64, dst, cmp)?;
7210 this.assembler.emit_bcond_label(Condition::Ne, nosame)?;
7211 this.assembler.emit_stlxr(
7212 Size::S64,
7213 Location::GPR(tmp),
7214 org,
7215 Location::GPR(addr),
7216 )?;
7217 this.assembler
7218 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7219 this.assembler.emit_dmb()?;
7220
7221 this.emit_label(nosame)?;
7222 if dst != ret {
7223 this.move_location(Size::S64, ret, dst)?;
7224 }
7225 for r in temps {
7226 this.release_gpr(r);
7227 }
7228 this.release_gpr(tmp);
7229 Ok(())
7230 },
7231 )
7232 }
7233
7234 fn i64_atomic_cmpxchg_8u(
7235 &mut self,
7236 new: Location,
7237 cmp: Location,
7238 target: Location,
7239 memarg: &MemArg,
7240 ret: Location,
7241 need_check: bool,
7242 imported_memories: bool,
7243 offset: i32,
7244 heap_access_oob: Label,
7245 unaligned_atomic: Label,
7246 ) -> Result<(), CompileError> {
7247 self.memory_op(
7248 target,
7249 memarg,
7250 true,
7251 1,
7252 need_check,
7253 imported_memories,
7254 offset,
7255 heap_access_oob,
7256 unaligned_atomic,
7257 |this, addr| {
7258 let mut temps = vec![];
7259 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
7260 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7261 })?;
7262 let dst =
7263 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7264 let org =
7265 this.location_to_reg(Size::S64, new, &mut temps, ImmType::None, false, None)?;
7266 let reread = this.get_label();
7267 let nosame = this.get_label();
7268
7269 this.emit_label(reread)?;
7270 this.assembler
7271 .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?;
7272 this.emit_relaxed_cmp(Size::S64, dst, cmp)?;
7273 this.assembler.emit_bcond_label(Condition::Ne, nosame)?;
7274 this.assembler.emit_stlxrb(
7275 Size::S64,
7276 Location::GPR(tmp),
7277 org,
7278 Location::GPR(addr),
7279 )?;
7280 this.assembler
7281 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7282 this.assembler.emit_dmb()?;
7283
7284 this.emit_label(nosame)?;
7285 if dst != ret {
7286 this.move_location(Size::S64, ret, dst)?;
7287 }
7288 for r in temps {
7289 this.release_gpr(r);
7290 }
7291 this.release_gpr(tmp);
7292 Ok(())
7293 },
7294 )
7295 }
7296
7297 fn i64_atomic_cmpxchg_16u(
7298 &mut self,
7299 new: Location,
7300 cmp: Location,
7301 target: Location,
7302 memarg: &MemArg,
7303 ret: Location,
7304 need_check: bool,
7305 imported_memories: bool,
7306 offset: i32,
7307 heap_access_oob: Label,
7308 unaligned_atomic: Label,
7309 ) -> Result<(), CompileError> {
7310 self.memory_op(
7311 target,
7312 memarg,
7313 true,
7314 2,
7315 need_check,
7316 imported_memories,
7317 offset,
7318 heap_access_oob,
7319 unaligned_atomic,
7320 |this, addr| {
7321 let mut temps = vec![];
7322 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
7323 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7324 })?;
7325 let dst =
7326 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7327 let org =
7328 this.location_to_reg(Size::S64, new, &mut temps, ImmType::None, false, None)?;
7329 let reread = this.get_label();
7330 let nosame = this.get_label();
7331
7332 this.emit_label(reread)?;
7333 this.assembler
7334 .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?;
7335 this.emit_relaxed_cmp(Size::S64, dst, cmp)?;
7336 this.assembler.emit_bcond_label(Condition::Ne, nosame)?;
7337 this.assembler.emit_stlxrh(
7338 Size::S64,
7339 Location::GPR(tmp),
7340 org,
7341 Location::GPR(addr),
7342 )?;
7343 this.assembler
7344 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7345 this.assembler.emit_dmb()?;
7346
7347 this.emit_label(nosame)?;
7348 if dst != ret {
7349 this.move_location(Size::S64, ret, dst)?;
7350 }
7351 for r in temps {
7352 this.release_gpr(r);
7353 }
7354 this.release_gpr(tmp);
7355 Ok(())
7356 },
7357 )
7358 }
7359
7360 fn i64_atomic_cmpxchg_32u(
7361 &mut self,
7362 new: Location,
7363 cmp: Location,
7364 target: Location,
7365 memarg: &MemArg,
7366 ret: Location,
7367 need_check: bool,
7368 imported_memories: bool,
7369 offset: i32,
7370 heap_access_oob: Label,
7371 unaligned_atomic: Label,
7372 ) -> Result<(), CompileError> {
7373 self.memory_op(
7374 target,
7375 memarg,
7376 true,
7377 4,
7378 need_check,
7379 imported_memories,
7380 offset,
7381 heap_access_oob,
7382 unaligned_atomic,
7383 |this, addr| {
7384 let mut temps = vec![];
7385 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
7386 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7387 })?;
7388 let dst =
7389 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7390 let org =
7391 this.location_to_reg(Size::S64, new, &mut temps, ImmType::None, false, None)?;
7392 let reread = this.get_label();
7393 let nosame = this.get_label();
7394
7395 this.emit_label(reread)?;
7396 this.assembler
7397 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
7398 this.emit_relaxed_cmp(Size::S64, dst, cmp)?;
7399 this.assembler.emit_bcond_label(Condition::Ne, nosame)?;
7400 this.assembler.emit_stlxr(
7401 Size::S32,
7402 Location::GPR(tmp),
7403 org,
7404 Location::GPR(addr),
7405 )?;
7406 this.assembler
7407 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7408 this.assembler.emit_dmb()?;
7409
7410 this.emit_label(nosame)?;
7411 if dst != ret {
7412 this.move_location(Size::S64, ret, dst)?;
7413 }
7414 for r in temps {
7415 this.release_gpr(r);
7416 }
7417 this.release_gpr(tmp);
7418 Ok(())
7419 },
7420 )
7421 }
7422
7423 fn f32_load(
7424 &mut self,
7425 addr: Location,
7426 memarg: &MemArg,
7427 ret: Location,
7428 need_check: bool,
7429 imported_memories: bool,
7430 offset: i32,
7431 heap_access_oob: Label,
7432 unaligned_atomic: Label,
7433 ) -> Result<(), CompileError> {
7434 self.memory_op(
7435 addr,
7436 memarg,
7437 false,
7438 4,
7439 need_check,
7440 imported_memories,
7441 offset,
7442 heap_access_oob,
7443 unaligned_atomic,
7444 |this, addr| this.emit_relaxed_ldr32(Size::S32, ret, Location::Memory(addr, 0)),
7445 )
7446 }
7447
7448 fn f32_save(
7449 &mut self,
7450 target_value: Location,
7451 memarg: &MemArg,
7452 target_addr: Location,
7453 canonicalize: bool,
7454 need_check: bool,
7455 imported_memories: bool,
7456 offset: i32,
7457 heap_access_oob: Label,
7458 unaligned_atomic: Label,
7459 ) -> Result<(), CompileError> {
7460 self.memory_op(
7461 target_addr,
7462 memarg,
7463 false,
7464 4,
7465 need_check,
7466 imported_memories,
7467 offset,
7468 heap_access_oob,
7469 unaligned_atomic,
7470 |this, addr| {
7471 if !canonicalize {
7472 this.emit_relaxed_str32(target_value, Location::Memory(addr, 0))
7473 } else {
7474 this.canonicalize_nan(Size::S32, target_value, Location::Memory(addr, 0))
7475 }
7476 },
7477 )
7478 }
7479
7480 fn f64_load(
7481 &mut self,
7482 addr: Location,
7483 memarg: &MemArg,
7484 ret: Location,
7485 need_check: bool,
7486 imported_memories: bool,
7487 offset: i32,
7488 heap_access_oob: Label,
7489 unaligned_atomic: Label,
7490 ) -> Result<(), CompileError> {
7491 self.memory_op(
7492 addr,
7493 memarg,
7494 false,
7495 8,
7496 need_check,
7497 imported_memories,
7498 offset,
7499 heap_access_oob,
7500 unaligned_atomic,
7501 |this, addr| this.emit_relaxed_ldr64(Size::S64, ret, Location::Memory(addr, 0)),
7502 )
7503 }
7504
7505 fn f64_save(
7506 &mut self,
7507 target_value: Location,
7508 memarg: &MemArg,
7509 target_addr: Location,
7510 canonicalize: bool,
7511 need_check: bool,
7512 imported_memories: bool,
7513 offset: i32,
7514 heap_access_oob: Label,
7515 unaligned_atomic: Label,
7516 ) -> Result<(), CompileError> {
7517 self.memory_op(
7518 target_addr,
7519 memarg,
7520 false,
7521 8,
7522 need_check,
7523 imported_memories,
7524 offset,
7525 heap_access_oob,
7526 unaligned_atomic,
7527 |this, addr| {
7528 if !canonicalize {
7529 this.emit_relaxed_str64(target_value, Location::Memory(addr, 0))
7530 } else {
7531 this.canonicalize_nan(Size::S64, target_value, Location::Memory(addr, 0))
7532 }
7533 },
7534 )
7535 }
7536
7537 fn convert_f64_i64(
7538 &mut self,
7539 loc: Location,
7540 signed: bool,
7541 ret: Location,
7542 ) -> Result<(), CompileError> {
7543 let mut gprs = vec![];
7544 let mut neons = vec![];
7545 let src = self.location_to_reg(Size::S64, loc, &mut gprs, ImmType::NoneXzr, true, None)?;
7546 let dest = self.location_to_neon(Size::S64, ret, &mut neons, ImmType::None, false)?;
7547 if signed {
7548 self.assembler.emit_scvtf(Size::S64, src, Size::S64, dest)?;
7549 } else {
7550 self.assembler.emit_ucvtf(Size::S64, src, Size::S64, dest)?;
7551 }
7552 if ret != dest {
7553 self.move_location(Size::S64, dest, ret)?;
7554 }
7555 for r in gprs {
7556 self.release_gpr(r);
7557 }
7558 for r in neons {
7559 self.release_simd(r);
7560 }
7561 Ok(())
7562 }
7563
7564 fn convert_f64_i32(
7565 &mut self,
7566 loc: Location,
7567 signed: bool,
7568 ret: Location,
7569 ) -> Result<(), CompileError> {
7570 let mut gprs = vec![];
7571 let mut neons = vec![];
7572 let src = self.location_to_reg(Size::S32, loc, &mut gprs, ImmType::NoneXzr, true, None)?;
7573 let dest = self.location_to_neon(Size::S64, ret, &mut neons, ImmType::None, false)?;
7574 if signed {
7575 self.assembler.emit_scvtf(Size::S32, src, Size::S64, dest)?;
7576 } else {
7577 self.assembler.emit_ucvtf(Size::S32, src, Size::S64, dest)?;
7578 }
7579 if ret != dest {
7580 self.move_location(Size::S64, dest, ret)?;
7581 }
7582 for r in gprs {
7583 self.release_gpr(r);
7584 }
7585 for r in neons {
7586 self.release_simd(r);
7587 }
7588 Ok(())
7589 }
7590
7591 fn convert_f32_i64(
7592 &mut self,
7593 loc: Location,
7594 signed: bool,
7595 ret: Location,
7596 ) -> Result<(), CompileError> {
7597 let mut gprs = vec![];
7598 let mut neons = vec![];
7599 let src = self.location_to_reg(Size::S64, loc, &mut gprs, ImmType::NoneXzr, true, None)?;
7600 let dest = self.location_to_neon(Size::S32, ret, &mut neons, ImmType::None, false)?;
7601 if signed {
7602 self.assembler.emit_scvtf(Size::S64, src, Size::S32, dest)?;
7603 } else {
7604 self.assembler.emit_ucvtf(Size::S64, src, Size::S32, dest)?;
7605 }
7606 if ret != dest {
7607 self.move_location(Size::S32, dest, ret)?;
7608 }
7609 for r in gprs {
7610 self.release_gpr(r);
7611 }
7612 for r in neons {
7613 self.release_simd(r);
7614 }
7615 Ok(())
7616 }
7617
7618 fn convert_f32_i32(
7619 &mut self,
7620 loc: Location,
7621 signed: bool,
7622 ret: Location,
7623 ) -> Result<(), CompileError> {
7624 let mut gprs = vec![];
7625 let mut neons = vec![];
7626 let src = self.location_to_reg(Size::S32, loc, &mut gprs, ImmType::NoneXzr, true, None)?;
7627 let dest = self.location_to_neon(Size::S32, ret, &mut neons, ImmType::None, false)?;
7628 if signed {
7629 self.assembler.emit_scvtf(Size::S32, src, Size::S32, dest)?;
7630 } else {
7631 self.assembler.emit_ucvtf(Size::S32, src, Size::S32, dest)?;
7632 }
7633 if ret != dest {
7634 self.move_location(Size::S32, dest, ret)?;
7635 }
7636 for r in gprs {
7637 self.release_gpr(r);
7638 }
7639 for r in neons {
7640 self.release_simd(r);
7641 }
7642 Ok(())
7643 }
7644
7645 fn convert_i64_f64(
7646 &mut self,
7647 loc: Location,
7648 ret: Location,
7649 signed: bool,
7650 sat: bool,
7651 ) -> Result<(), CompileError> {
7652 let mut gprs = vec![];
7653 let mut neons = vec![];
7654 let src = self.location_to_neon(Size::S64, loc, &mut neons, ImmType::None, true)?;
7655 let dest = self.location_to_reg(Size::S64, ret, &mut gprs, ImmType::None, false, None)?;
7656 let old_fpcr = if !sat {
7657 self.reset_exception_fpsr()?;
7658 self.set_trap_enabled(&mut gprs)?
7659 } else {
7660 GPR::XzrSp
7661 };
7662 if signed {
7663 self.assembler
7664 .emit_fcvtzs(Size::S64, src, Size::S64, dest)?;
7665 } else {
7666 self.assembler
7667 .emit_fcvtzu(Size::S64, src, Size::S64, dest)?;
7668 }
7669 if !sat {
7670 self.trap_float_convertion_errors(old_fpcr, Size::S64, src, &mut gprs)?;
7671 }
7672 if ret != dest {
7673 self.move_location(Size::S64, dest, ret)?;
7674 }
7675 for r in gprs {
7676 self.release_gpr(r);
7677 }
7678 for r in neons {
7679 self.release_simd(r);
7680 }
7681 Ok(())
7682 }
7683
7684 fn convert_i32_f64(
7685 &mut self,
7686 loc: Location,
7687 ret: Location,
7688 signed: bool,
7689 sat: bool,
7690 ) -> Result<(), CompileError> {
7691 let mut gprs = vec![];
7692 let mut neons = vec![];
7693 let src = self.location_to_neon(Size::S64, loc, &mut neons, ImmType::None, true)?;
7694 let dest = self.location_to_reg(Size::S32, ret, &mut gprs, ImmType::None, false, None)?;
7695 let old_fpcr = if !sat {
7696 self.reset_exception_fpsr()?;
7697 self.set_trap_enabled(&mut gprs)?
7698 } else {
7699 GPR::XzrSp
7700 };
7701 if signed {
7702 self.assembler
7703 .emit_fcvtzs(Size::S64, src, Size::S32, dest)?;
7704 } else {
7705 self.assembler
7706 .emit_fcvtzu(Size::S64, src, Size::S32, dest)?;
7707 }
7708 if !sat {
7709 self.trap_float_convertion_errors(old_fpcr, Size::S64, src, &mut gprs)?;
7710 }
7711 if ret != dest {
7712 self.move_location(Size::S32, dest, ret)?;
7713 }
7714 for r in gprs {
7715 self.release_gpr(r);
7716 }
7717 for r in neons {
7718 self.release_simd(r);
7719 }
7720 Ok(())
7721 }
7722
7723 fn convert_i64_f32(
7724 &mut self,
7725 loc: Location,
7726 ret: Location,
7727 signed: bool,
7728 sat: bool,
7729 ) -> Result<(), CompileError> {
7730 let mut gprs = vec![];
7731 let mut neons = vec![];
7732 let src = self.location_to_neon(Size::S32, loc, &mut neons, ImmType::None, true)?;
7733 let dest = self.location_to_reg(Size::S64, ret, &mut gprs, ImmType::None, false, None)?;
7734 let old_fpcr = if !sat {
7735 self.reset_exception_fpsr()?;
7736 self.set_trap_enabled(&mut gprs)?
7737 } else {
7738 GPR::XzrSp
7739 };
7740 if signed {
7741 self.assembler
7742 .emit_fcvtzs(Size::S32, src, Size::S64, dest)?;
7743 } else {
7744 self.assembler
7745 .emit_fcvtzu(Size::S32, src, Size::S64, dest)?;
7746 }
7747 if !sat {
7748 self.trap_float_convertion_errors(old_fpcr, Size::S32, src, &mut gprs)?;
7749 }
7750 if ret != dest {
7751 self.move_location(Size::S64, dest, ret)?;
7752 }
7753 for r in gprs {
7754 self.release_gpr(r);
7755 }
7756 for r in neons {
7757 self.release_simd(r);
7758 }
7759 Ok(())
7760 }
7761
7762 fn convert_i32_f32(
7763 &mut self,
7764 loc: Location,
7765 ret: Location,
7766 signed: bool,
7767 sat: bool,
7768 ) -> Result<(), CompileError> {
7769 let mut gprs = vec![];
7770 let mut neons = vec![];
7771 let src = self.location_to_neon(Size::S32, loc, &mut neons, ImmType::None, true)?;
7772 let dest = self.location_to_reg(Size::S32, ret, &mut gprs, ImmType::None, false, None)?;
7773 let old_fpcr = if !sat {
7774 self.reset_exception_fpsr()?;
7775 self.set_trap_enabled(&mut gprs)?
7776 } else {
7777 GPR::XzrSp
7778 };
7779 if signed {
7780 self.assembler
7781 .emit_fcvtzs(Size::S32, src, Size::S32, dest)?;
7782 } else {
7783 self.assembler
7784 .emit_fcvtzu(Size::S32, src, Size::S32, dest)?;
7785 }
7786 if !sat {
7787 self.trap_float_convertion_errors(old_fpcr, Size::S32, src, &mut gprs)?;
7788 }
7789 if ret != dest {
7790 self.move_location(Size::S32, dest, ret)?;
7791 }
7792 for r in gprs {
7793 self.release_gpr(r);
7794 }
7795 for r in neons {
7796 self.release_simd(r);
7797 }
7798 Ok(())
7799 }
7800
7801 fn convert_f64_f32(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7802 self.emit_relaxed_binop_neon(Assembler::emit_fcvt, Size::S32, loc, ret, true)
7803 }
7804
7805 fn convert_f32_f64(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7806 self.emit_relaxed_binop_neon(Assembler::emit_fcvt, Size::S64, loc, ret, true)
7807 }
7808
7809 fn f64_neg(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7810 self.emit_relaxed_binop_neon(Assembler::emit_fneg, Size::S64, loc, ret, true)
7811 }
7812
7813 fn f64_abs(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7814 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
7815 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7816 })?;
7817
7818 self.move_location(Size::S64, loc, Location::GPR(tmp))?;
7819 self.assembler.emit_and(
7820 Size::S64,
7821 Location::GPR(tmp),
7822 Location::Imm64(0x7fffffffffffffffu64),
7823 Location::GPR(tmp),
7824 )?;
7825 self.move_location(Size::S64, Location::GPR(tmp), ret)?;
7826
7827 self.release_gpr(tmp);
7828 Ok(())
7829 }
7830
7831 fn emit_i64_copysign(&mut self, tmp1: GPR, tmp2: GPR) -> Result<(), CompileError> {
7832 self.assembler.emit_and(
7833 Size::S64,
7834 Location::GPR(tmp1),
7835 Location::Imm64(0x7fffffffffffffffu64),
7836 Location::GPR(tmp1),
7837 )?;
7838
7839 self.assembler.emit_and(
7840 Size::S64,
7841 Location::GPR(tmp2),
7842 Location::Imm64(0x8000000000000000u64),
7843 Location::GPR(tmp2),
7844 )?;
7845
7846 self.assembler.emit_or(
7847 Size::S64,
7848 Location::GPR(tmp1),
7849 Location::GPR(tmp2),
7850 Location::GPR(tmp1),
7851 )
7852 }
7853
7854 fn f64_sqrt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7855 self.emit_relaxed_binop_neon(Assembler::emit_fsqrt, Size::S64, loc, ret, true)
7856 }
7857
7858 fn f64_trunc(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7859 self.emit_relaxed_binop_neon(Assembler::emit_frintz, Size::S64, loc, ret, true)
7860 }
7861
7862 fn f64_ceil(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7863 self.emit_relaxed_binop_neon(Assembler::emit_frintp, Size::S64, loc, ret, true)
7864 }
7865
7866 fn f64_floor(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7867 self.emit_relaxed_binop_neon(Assembler::emit_frintm, Size::S64, loc, ret, true)
7868 }
7869
7870 fn f64_nearest(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7871 self.emit_relaxed_binop_neon(Assembler::emit_frintn, Size::S64, loc, ret, true)
7872 }
7873
7874 fn f64_cmp_ge(
7875 &mut self,
7876 loc_a: Location,
7877 loc_b: Location,
7878 ret: Location,
7879 ) -> Result<(), CompileError> {
7880 let mut temps = vec![];
7881 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7882 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S64, loc_b, loc_a, false)?;
7883 self.assembler.emit_cset(Size::S32, dest, Condition::Ls)?;
7884 if ret != dest {
7885 self.move_location(Size::S32, dest, ret)?;
7886 }
7887 for r in temps {
7888 self.release_gpr(r);
7889 }
7890 Ok(())
7891 }
7892
7893 fn f64_cmp_gt(
7894 &mut self,
7895 loc_a: Location,
7896 loc_b: Location,
7897 ret: Location,
7898 ) -> Result<(), CompileError> {
7899 let mut temps = vec![];
7900 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7901 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S64, loc_b, loc_a, false)?;
7902 self.assembler.emit_cset(Size::S32, dest, Condition::Cc)?;
7903 if ret != dest {
7904 self.move_location(Size::S32, dest, ret)?;
7905 }
7906 for r in temps {
7907 self.release_gpr(r);
7908 }
7909 Ok(())
7910 }
7911
7912 fn f64_cmp_le(
7913 &mut self,
7914 loc_a: Location,
7915 loc_b: Location,
7916 ret: Location,
7917 ) -> Result<(), CompileError> {
7918 let mut temps = vec![];
7919 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7920 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S64, loc_a, loc_b, false)?;
7921 self.assembler.emit_cset(Size::S32, dest, Condition::Ls)?;
7922 if ret != dest {
7923 self.move_location(Size::S32, dest, ret)?;
7924 }
7925 for r in temps {
7926 self.release_gpr(r);
7927 }
7928 Ok(())
7929 }
7930
7931 fn f64_cmp_lt(
7932 &mut self,
7933 loc_a: Location,
7934 loc_b: Location,
7935 ret: Location,
7936 ) -> Result<(), CompileError> {
7937 let mut temps = vec![];
7938 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7939 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S64, loc_a, loc_b, false)?;
7940 self.assembler.emit_cset(Size::S32, dest, Condition::Cc)?;
7941 if ret != dest {
7942 self.move_location(Size::S32, dest, ret)?;
7943 }
7944 for r in temps {
7945 self.release_gpr(r);
7946 }
7947 Ok(())
7948 }
7949
7950 fn f64_cmp_ne(
7951 &mut self,
7952 loc_a: Location,
7953 loc_b: Location,
7954 ret: Location,
7955 ) -> Result<(), CompileError> {
7956 let mut temps = vec![];
7957 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7958 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S64, loc_a, loc_b, false)?;
7959 self.assembler.emit_cset(Size::S32, dest, Condition::Ne)?;
7960 if ret != dest {
7961 self.move_location(Size::S32, dest, ret)?;
7962 }
7963 for r in temps {
7964 self.release_gpr(r);
7965 }
7966 Ok(())
7967 }
7968
7969 fn f64_cmp_eq(
7970 &mut self,
7971 loc_a: Location,
7972 loc_b: Location,
7973 ret: Location,
7974 ) -> Result<(), CompileError> {
7975 let mut temps = vec![];
7976 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7977 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S64, loc_a, loc_b, false)?;
7978 self.assembler.emit_cset(Size::S32, dest, Condition::Eq)?;
7979 if ret != dest {
7980 self.move_location(Size::S32, dest, ret)?;
7981 }
7982 for r in temps {
7983 self.release_gpr(r);
7984 }
7985 Ok(())
7986 }
7987
7988 fn f64_min(
7989 &mut self,
7990 loc_a: Location,
7991 loc_b: Location,
7992 ret: Location,
7993 ) -> Result<(), CompileError> {
7994 let mut temps = vec![];
7995 let old_fpcr = self.set_default_nan(&mut temps)?;
7996 self.emit_relaxed_binop3_neon(
7997 Assembler::emit_fmin,
7998 Size::S64,
7999 loc_a,
8000 loc_b,
8001 ret,
8002 ImmType::None,
8003 )?;
8004 self.restore_fpcr(old_fpcr)?;
8005 for r in temps {
8006 self.release_gpr(r);
8007 }
8008 Ok(())
8009 }
8010
8011 fn f64_max(
8012 &mut self,
8013 loc_a: Location,
8014 loc_b: Location,
8015 ret: Location,
8016 ) -> Result<(), CompileError> {
8017 let mut temps = vec![];
8018 let old_fpcr = self.set_default_nan(&mut temps)?;
8019 self.emit_relaxed_binop3_neon(
8020 Assembler::emit_fmax,
8021 Size::S64,
8022 loc_a,
8023 loc_b,
8024 ret,
8025 ImmType::None,
8026 )?;
8027 self.restore_fpcr(old_fpcr)?;
8028 for r in temps {
8029 self.release_gpr(r);
8030 }
8031 Ok(())
8032 }
8033
8034 fn f64_add(
8035 &mut self,
8036 loc_a: Location,
8037 loc_b: Location,
8038 ret: Location,
8039 ) -> Result<(), CompileError> {
8040 self.emit_relaxed_binop3_neon(
8041 Assembler::emit_fadd,
8042 Size::S64,
8043 loc_a,
8044 loc_b,
8045 ret,
8046 ImmType::None,
8047 )
8048 }
8049
8050 fn f64_sub(
8051 &mut self,
8052 loc_a: Location,
8053 loc_b: Location,
8054 ret: Location,
8055 ) -> Result<(), CompileError> {
8056 self.emit_relaxed_binop3_neon(
8057 Assembler::emit_fsub,
8058 Size::S64,
8059 loc_a,
8060 loc_b,
8061 ret,
8062 ImmType::None,
8063 )
8064 }
8065
8066 fn f64_mul(
8067 &mut self,
8068 loc_a: Location,
8069 loc_b: Location,
8070 ret: Location,
8071 ) -> Result<(), CompileError> {
8072 self.emit_relaxed_binop3_neon(
8073 Assembler::emit_fmul,
8074 Size::S64,
8075 loc_a,
8076 loc_b,
8077 ret,
8078 ImmType::None,
8079 )
8080 }
8081
8082 fn f64_div(
8083 &mut self,
8084 loc_a: Location,
8085 loc_b: Location,
8086 ret: Location,
8087 ) -> Result<(), CompileError> {
8088 self.emit_relaxed_binop3_neon(
8089 Assembler::emit_fdiv,
8090 Size::S64,
8091 loc_a,
8092 loc_b,
8093 ret,
8094 ImmType::None,
8095 )
8096 }
8097
8098 fn f32_neg(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
8099 self.emit_relaxed_binop_neon(Assembler::emit_fneg, Size::S32, loc, ret, true)
8100 }
8101
8102 fn f32_abs(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
8103 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
8104 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
8105 })?;
8106 self.move_location(Size::S32, loc, Location::GPR(tmp))?;
8107 self.assembler.emit_and(
8108 Size::S32,
8109 Location::GPR(tmp),
8110 Location::Imm32(0x7fffffffu32),
8111 Location::GPR(tmp),
8112 )?;
8113 self.move_location(Size::S32, Location::GPR(tmp), ret)?;
8114 self.release_gpr(tmp);
8115 Ok(())
8116 }
8117
8118 fn emit_i32_copysign(&mut self, tmp1: GPR, tmp2: GPR) -> Result<(), CompileError> {
8119 self.assembler.emit_and(
8120 Size::S32,
8121 Location::GPR(tmp1),
8122 Location::Imm32(0x7fffffffu32),
8123 Location::GPR(tmp1),
8124 )?;
8125 self.assembler.emit_and(
8126 Size::S32,
8127 Location::GPR(tmp2),
8128 Location::Imm32(0x80000000u32),
8129 Location::GPR(tmp2),
8130 )?;
8131 self.assembler.emit_or(
8132 Size::S32,
8133 Location::GPR(tmp1),
8134 Location::GPR(tmp2),
8135 Location::GPR(tmp1),
8136 )
8137 }
8138
8139 fn f32_sqrt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
8140 self.emit_relaxed_binop_neon(Assembler::emit_fsqrt, Size::S32, loc, ret, true)
8141 }
8142
8143 fn f32_trunc(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
8144 self.emit_relaxed_binop_neon(Assembler::emit_frintz, Size::S32, loc, ret, true)
8145 }
8146
8147 fn f32_ceil(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
8148 self.emit_relaxed_binop_neon(Assembler::emit_frintp, Size::S32, loc, ret, true)
8149 }
8150
8151 fn f32_floor(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
8152 self.emit_relaxed_binop_neon(Assembler::emit_frintm, Size::S32, loc, ret, true)
8153 }
8154
8155 fn f32_nearest(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
8156 self.emit_relaxed_binop_neon(Assembler::emit_frintn, Size::S32, loc, ret, true)
8157 }
8158
8159 fn f32_cmp_ge(
8160 &mut self,
8161 loc_a: Location,
8162 loc_b: Location,
8163 ret: Location,
8164 ) -> Result<(), CompileError> {
8165 let mut temps = vec![];
8166 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
8167 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S32, loc_b, loc_a, false)?;
8168 self.assembler.emit_cset(Size::S32, dest, Condition::Ls)?;
8169 if ret != dest {
8170 self.move_location(Size::S32, dest, ret)?;
8171 }
8172 for r in temps {
8173 self.release_gpr(r);
8174 }
8175 Ok(())
8176 }
8177
8178 fn f32_cmp_gt(
8179 &mut self,
8180 loc_a: Location,
8181 loc_b: Location,
8182 ret: Location,
8183 ) -> Result<(), CompileError> {
8184 let mut temps = vec![];
8185 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
8186 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S32, loc_b, loc_a, false)?;
8187 self.assembler.emit_cset(Size::S32, dest, Condition::Cc)?;
8188 if ret != dest {
8189 self.move_location(Size::S32, dest, ret)?;
8190 }
8191 for r in temps {
8192 self.release_gpr(r);
8193 }
8194 Ok(())
8195 }
8196
8197 fn f32_cmp_le(
8198 &mut self,
8199 loc_a: Location,
8200 loc_b: Location,
8201 ret: Location,
8202 ) -> Result<(), CompileError> {
8203 let mut temps = vec![];
8204 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
8205 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S32, loc_a, loc_b, false)?;
8206 self.assembler.emit_cset(Size::S32, dest, Condition::Ls)?;
8207 if ret != dest {
8208 self.move_location(Size::S32, dest, ret)?;
8209 }
8210 for r in temps {
8211 self.release_gpr(r);
8212 }
8213 Ok(())
8214 }
8215
8216 fn f32_cmp_lt(
8217 &mut self,
8218 loc_a: Location,
8219 loc_b: Location,
8220 ret: Location,
8221 ) -> Result<(), CompileError> {
8222 let mut temps = vec![];
8223 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
8224 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S32, loc_a, loc_b, false)?;
8225 self.assembler.emit_cset(Size::S32, dest, Condition::Cc)?;
8226 if ret != dest {
8227 self.move_location(Size::S32, dest, ret)?;
8228 }
8229 for r in temps {
8230 self.release_gpr(r);
8231 }
8232 Ok(())
8233 }
8234
8235 fn f32_cmp_ne(
8236 &mut self,
8237 loc_a: Location,
8238 loc_b: Location,
8239 ret: Location,
8240 ) -> Result<(), CompileError> {
8241 let mut temps = vec![];
8242 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
8243 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S32, loc_a, loc_b, false)?;
8244 self.assembler.emit_cset(Size::S32, dest, Condition::Ne)?;
8245 if ret != dest {
8246 self.move_location(Size::S32, dest, ret)?;
8247 }
8248 for r in temps {
8249 self.release_gpr(r);
8250 }
8251 Ok(())
8252 }
8253
8254 fn f32_cmp_eq(
8255 &mut self,
8256 loc_a: Location,
8257 loc_b: Location,
8258 ret: Location,
8259 ) -> Result<(), CompileError> {
8260 let mut temps = vec![];
8261 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
8262 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S32, loc_a, loc_b, false)?;
8263 self.assembler.emit_cset(Size::S32, dest, Condition::Eq)?;
8264 if ret != dest {
8265 self.move_location(Size::S32, dest, ret)?;
8266 }
8267 for r in temps {
8268 self.release_gpr(r);
8269 }
8270 Ok(())
8271 }
8272
8273 fn f32_min(
8274 &mut self,
8275 loc_a: Location,
8276 loc_b: Location,
8277 ret: Location,
8278 ) -> Result<(), CompileError> {
8279 let mut temps = vec![];
8280 let old_fpcr = self.set_default_nan(&mut temps)?;
8281 self.emit_relaxed_binop3_neon(
8282 Assembler::emit_fmin,
8283 Size::S32,
8284 loc_a,
8285 loc_b,
8286 ret,
8287 ImmType::None,
8288 )?;
8289 self.restore_fpcr(old_fpcr)?;
8290 for r in temps {
8291 self.release_gpr(r);
8292 }
8293 Ok(())
8294 }
8295
8296 fn f32_max(
8297 &mut self,
8298 loc_a: Location,
8299 loc_b: Location,
8300 ret: Location,
8301 ) -> Result<(), CompileError> {
8302 let mut temps = vec![];
8303 let old_fpcr = self.set_default_nan(&mut temps)?;
8304 self.emit_relaxed_binop3_neon(
8305 Assembler::emit_fmax,
8306 Size::S32,
8307 loc_a,
8308 loc_b,
8309 ret,
8310 ImmType::None,
8311 )?;
8312 self.restore_fpcr(old_fpcr)?;
8313 for r in temps {
8314 self.release_gpr(r);
8315 }
8316 Ok(())
8317 }
8318
8319 fn f32_add(
8320 &mut self,
8321 loc_a: Location,
8322 loc_b: Location,
8323 ret: Location,
8324 ) -> Result<(), CompileError> {
8325 self.emit_relaxed_binop3_neon(
8326 Assembler::emit_fadd,
8327 Size::S32,
8328 loc_a,
8329 loc_b,
8330 ret,
8331 ImmType::None,
8332 )
8333 }
8334
8335 fn f32_sub(
8336 &mut self,
8337 loc_a: Location,
8338 loc_b: Location,
8339 ret: Location,
8340 ) -> Result<(), CompileError> {
8341 self.emit_relaxed_binop3_neon(
8342 Assembler::emit_fsub,
8343 Size::S32,
8344 loc_a,
8345 loc_b,
8346 ret,
8347 ImmType::None,
8348 )
8349 }
8350
8351 fn f32_mul(
8352 &mut self,
8353 loc_a: Location,
8354 loc_b: Location,
8355 ret: Location,
8356 ) -> Result<(), CompileError> {
8357 self.emit_relaxed_binop3_neon(
8358 Assembler::emit_fmul,
8359 Size::S32,
8360 loc_a,
8361 loc_b,
8362 ret,
8363 ImmType::None,
8364 )
8365 }
8366
8367 fn f32_div(
8368 &mut self,
8369 loc_a: Location,
8370 loc_b: Location,
8371 ret: Location,
8372 ) -> Result<(), CompileError> {
8373 self.emit_relaxed_binop3_neon(
8374 Assembler::emit_fdiv,
8375 Size::S32,
8376 loc_a,
8377 loc_b,
8378 ret,
8379 ImmType::None,
8380 )
8381 }
8382
8383 fn gen_std_trampoline(
8384 &self,
8385 sig: &FunctionType,
8386 calling_convention: CallingConvention,
8387 ) -> Result<FunctionBody, CompileError> {
8388 gen_std_trampoline_arm64(sig, calling_convention)
8389 }
8390 fn gen_std_dynamic_import_trampoline(
8393 &self,
8394 vmoffsets: &VMOffsets,
8395 sig: &FunctionType,
8396 calling_convention: CallingConvention,
8397 ) -> Result<FunctionBody, CompileError> {
8398 gen_std_dynamic_import_trampoline_arm64(vmoffsets, sig, calling_convention)
8399 }
8400 fn gen_import_call_trampoline(
8403 &self,
8404 vmoffsets: &VMOffsets,
8405 index: FunctionIndex,
8406 sig: &FunctionType,
8407 calling_convention: CallingConvention,
8408 ) -> Result<CustomSection, CompileError> {
8409 gen_import_call_trampoline_arm64(vmoffsets, index, sig, calling_convention)
8410 }
8411
8412 #[cfg(feature = "unwind")]
8413 fn gen_dwarf_unwind_info(&mut self, code_len: usize) -> Option<UnwindInstructions> {
8414 let mut instructions = vec![];
8415 for &(instruction_offset, ref inst) in &self.unwind_ops {
8416 let instruction_offset = instruction_offset as u32;
8417 match *inst {
8418 UnwindOps::PushFP { up_to_sp } => {
8419 instructions.push((
8420 instruction_offset,
8421 CallFrameInstruction::CfaOffset(up_to_sp as i32),
8422 ));
8423 instructions.push((
8424 instruction_offset,
8425 CallFrameInstruction::Offset(AArch64::X29, -(up_to_sp as i32)),
8426 ));
8427 }
8428 UnwindOps::Push2Regs {
8429 reg1,
8430 reg2,
8431 up_to_sp,
8432 } => {
8433 instructions.push((
8434 instruction_offset,
8435 CallFrameInstruction::CfaOffset(up_to_sp as i32),
8436 ));
8437 instructions.push((
8438 instruction_offset,
8439 CallFrameInstruction::Offset(reg2.dwarf_index(), -(up_to_sp as i32) + 8),
8440 ));
8441 instructions.push((
8442 instruction_offset,
8443 CallFrameInstruction::Offset(reg1.dwarf_index(), -(up_to_sp as i32)),
8444 ));
8445 }
8446 UnwindOps::DefineNewFrame => {
8447 instructions.push((
8448 instruction_offset,
8449 CallFrameInstruction::CfaRegister(AArch64::X29),
8450 ));
8451 }
8452 UnwindOps::SaveRegister { reg, bp_neg_offset } => instructions.push((
8453 instruction_offset,
8454 CallFrameInstruction::Offset(reg.dwarf_index(), -bp_neg_offset),
8455 )),
8456 UnwindOps::SubtractFP { .. } => unimplemented!(),
8457 }
8458 }
8459 Some(UnwindInstructions {
8460 instructions,
8461 len: code_len as u32,
8462 })
8463 }
8464 #[cfg(not(feature = "unwind"))]
8465
8466 fn gen_dwarf_unwind_info(&mut self, _code_len: usize) -> Option<UnwindInstructions> {
8467 None
8468 }
8469
8470 fn gen_windows_unwind_info(&mut self, _code_len: usize) -> Option<Vec<u8>> {
8471 None
8472 }
8473}
8474
8475#[cfg(test)]
8476mod test {
8477 use super::*;
8478
8479 fn test_move_location(machine: &mut MachineARM64, size: Size) -> Result<(), CompileError> {
8480 machine.move_location(size, Location::GPR(GPR::X1), Location::GPR(GPR::X2))?;
8481 machine.move_location(size, Location::GPR(GPR::X1), Location::Memory(GPR::X2, 10))?;
8482 machine.move_location(size, Location::GPR(GPR::X1), Location::Memory(GPR::X2, -10))?;
8483 machine.move_location(
8484 size,
8485 Location::GPR(GPR::X1),
8486 Location::Memory(GPR::X2, 1024),
8487 )?;
8488 machine.move_location(
8489 size,
8490 Location::GPR(GPR::X1),
8491 Location::Memory(GPR::X2, -1024),
8492 )?;
8493 machine.move_location(size, Location::Memory(GPR::X2, 10), Location::GPR(GPR::X1))?;
8494 machine.move_location(size, Location::Memory(GPR::X2, -10), Location::GPR(GPR::X1))?;
8495 machine.move_location(
8496 size,
8497 Location::Memory(GPR::X2, 1024),
8498 Location::GPR(GPR::X1),
8499 )?;
8500 machine.move_location(
8501 size,
8502 Location::Memory(GPR::X2, -1024),
8503 Location::GPR(GPR::X1),
8504 )?;
8505 machine.move_location(size, Location::GPR(GPR::X1), Location::SIMD(NEON::V0))?;
8506 machine.move_location(size, Location::SIMD(NEON::V0), Location::GPR(GPR::X1))?;
8507 machine.move_location(
8508 size,
8509 Location::SIMD(NEON::V0),
8510 Location::Memory(GPR::X2, 10),
8511 )?;
8512 machine.move_location(
8513 size,
8514 Location::SIMD(NEON::V0),
8515 Location::Memory(GPR::X2, -10),
8516 )?;
8517 machine.move_location(
8518 size,
8519 Location::SIMD(NEON::V0),
8520 Location::Memory(GPR::X2, 1024),
8521 )?;
8522 machine.move_location(
8523 size,
8524 Location::SIMD(NEON::V0),
8525 Location::Memory(GPR::X2, -1024),
8526 )?;
8527 machine.move_location(
8528 size,
8529 Location::Memory(GPR::X2, 10),
8530 Location::SIMD(NEON::V0),
8531 )?;
8532 machine.move_location(
8533 size,
8534 Location::Memory(GPR::X2, -10),
8535 Location::SIMD(NEON::V0),
8536 )?;
8537 machine.move_location(
8538 size,
8539 Location::Memory(GPR::X2, 1024),
8540 Location::SIMD(NEON::V0),
8541 )?;
8542 machine.move_location(
8543 size,
8544 Location::Memory(GPR::X2, -1024),
8545 Location::SIMD(NEON::V0),
8546 )?;
8547
8548 Ok(())
8549 }
8550
8551 fn test_move_location_extended(
8552 machine: &mut MachineARM64,
8553 signed: bool,
8554 sized: Size,
8555 ) -> Result<(), CompileError> {
8556 machine.move_location_extend(
8557 sized,
8558 signed,
8559 Location::GPR(GPR::X0),
8560 Size::S64,
8561 Location::GPR(GPR::X1),
8562 )?;
8563 machine.move_location_extend(
8564 sized,
8565 signed,
8566 Location::GPR(GPR::X0),
8567 Size::S64,
8568 Location::Memory(GPR::X1, 10),
8569 )?;
8570 machine.move_location_extend(
8571 sized,
8572 signed,
8573 Location::GPR(GPR::X0),
8574 Size::S64,
8575 Location::Memory(GPR::X1, 16),
8576 )?;
8577 machine.move_location_extend(
8578 sized,
8579 signed,
8580 Location::GPR(GPR::X0),
8581 Size::S64,
8582 Location::Memory(GPR::X1, -16),
8583 )?;
8584 machine.move_location_extend(
8585 sized,
8586 signed,
8587 Location::GPR(GPR::X0),
8588 Size::S64,
8589 Location::Memory(GPR::X1, 1024),
8590 )?;
8591 machine.move_location_extend(
8592 sized,
8593 signed,
8594 Location::GPR(GPR::X0),
8595 Size::S64,
8596 Location::Memory(GPR::X1, -1024),
8597 )?;
8598 machine.move_location_extend(
8599 sized,
8600 signed,
8601 Location::Memory(GPR::X0, 10),
8602 Size::S64,
8603 Location::GPR(GPR::X1),
8604 )?;
8605
8606 Ok(())
8607 }
8608
8609 fn test_binop_op(
8610 machine: &mut MachineARM64,
8611 op: fn(&mut MachineARM64, Location, Location, Location) -> Result<(), CompileError>,
8612 ) -> Result<(), CompileError> {
8613 op(
8614 machine,
8615 Location::GPR(GPR::X2),
8616 Location::GPR(GPR::X2),
8617 Location::GPR(GPR::X0),
8618 )?;
8619 op(
8620 machine,
8621 Location::GPR(GPR::X2),
8622 Location::Imm32(10),
8623 Location::GPR(GPR::X0),
8624 )?;
8625 op(
8626 machine,
8627 Location::GPR(GPR::X0),
8628 Location::GPR(GPR::X0),
8629 Location::GPR(GPR::X0),
8630 )?;
8631 op(
8632 machine,
8633 Location::Imm32(10),
8634 Location::GPR(GPR::X2),
8635 Location::GPR(GPR::X0),
8636 )?;
8637 op(
8638 machine,
8639 Location::GPR(GPR::X0),
8640 Location::GPR(GPR::X2),
8641 Location::Memory(GPR::X0, 10),
8642 )?;
8643 op(
8644 machine,
8645 Location::GPR(GPR::X0),
8646 Location::Memory(GPR::X2, 16),
8647 Location::Memory(GPR::X0, 10),
8648 )?;
8649 op(
8650 machine,
8651 Location::Memory(GPR::X0, 0),
8652 Location::Memory(GPR::X2, 16),
8653 Location::Memory(GPR::X0, 10),
8654 )?;
8655
8656 Ok(())
8657 }
8658
8659 fn test_float_binop_op(
8660 machine: &mut MachineARM64,
8661 op: fn(&mut MachineARM64, Location, Location, Location) -> Result<(), CompileError>,
8662 ) -> Result<(), CompileError> {
8663 op(
8664 machine,
8665 Location::SIMD(NEON::V3),
8666 Location::SIMD(NEON::V2),
8667 Location::SIMD(NEON::V0),
8668 )?;
8669 op(
8670 machine,
8671 Location::SIMD(NEON::V0),
8672 Location::SIMD(NEON::V2),
8673 Location::SIMD(NEON::V0),
8674 )?;
8675 op(
8676 machine,
8677 Location::SIMD(NEON::V0),
8678 Location::SIMD(NEON::V0),
8679 Location::SIMD(NEON::V0),
8680 )?;
8681 op(
8682 machine,
8683 Location::Memory(GPR::X0, 0),
8684 Location::SIMD(NEON::V2),
8685 Location::SIMD(NEON::V0),
8686 )?;
8687 op(
8688 machine,
8689 Location::Memory(GPR::X0, 0),
8690 Location::Memory(GPR::X1, 10),
8691 Location::SIMD(NEON::V0),
8692 )?;
8693 op(
8694 machine,
8695 Location::Memory(GPR::X0, 0),
8696 Location::Memory(GPR::X1, 16),
8697 Location::Memory(GPR::X2, 32),
8698 )?;
8699 op(
8700 machine,
8701 Location::SIMD(NEON::V0),
8702 Location::Memory(GPR::X1, 16),
8703 Location::Memory(GPR::X2, 32),
8704 )?;
8705 op(
8706 machine,
8707 Location::SIMD(NEON::V0),
8708 Location::SIMD(NEON::V1),
8709 Location::Memory(GPR::X2, 32),
8710 )?;
8711
8712 Ok(())
8713 }
8714
8715 fn test_float_cmp_op(
8716 machine: &mut MachineARM64,
8717 op: fn(&mut MachineARM64, Location, Location, Location) -> Result<(), CompileError>,
8718 ) -> Result<(), CompileError> {
8719 op(
8720 machine,
8721 Location::SIMD(NEON::V3),
8722 Location::SIMD(NEON::V2),
8723 Location::GPR(GPR::X0),
8724 )?;
8725 op(
8726 machine,
8727 Location::SIMD(NEON::V0),
8728 Location::SIMD(NEON::V0),
8729 Location::GPR(GPR::X0),
8730 )?;
8731 op(
8732 machine,
8733 Location::Memory(GPR::X1, 0),
8734 Location::SIMD(NEON::V2),
8735 Location::GPR(GPR::X0),
8736 )?;
8737 op(
8738 machine,
8739 Location::Memory(GPR::X1, 0),
8740 Location::Memory(GPR::X2, 10),
8741 Location::GPR(GPR::X0),
8742 )?;
8743 op(
8744 machine,
8745 Location::Memory(GPR::X1, 0),
8746 Location::Memory(GPR::X2, 16),
8747 Location::Memory(GPR::X0, 32),
8748 )?;
8749 op(
8750 machine,
8751 Location::SIMD(NEON::V0),
8752 Location::Memory(GPR::X2, 16),
8753 Location::Memory(GPR::X0, 32),
8754 )?;
8755 op(
8756 machine,
8757 Location::SIMD(NEON::V0),
8758 Location::SIMD(NEON::V1),
8759 Location::Memory(GPR::X0, 32),
8760 )?;
8761
8762 Ok(())
8763 }
8764
8765 #[test]
8766 fn tests_arm64() -> Result<(), CompileError> {
8767 let mut machine = MachineARM64::new(None);
8768
8769 test_move_location(&mut machine, Size::S32)?;
8770 test_move_location(&mut machine, Size::S64)?;
8771 test_move_location_extended(&mut machine, false, Size::S8)?;
8772 test_move_location_extended(&mut machine, false, Size::S16)?;
8773 test_move_location_extended(&mut machine, false, Size::S32)?;
8774 test_move_location_extended(&mut machine, true, Size::S8)?;
8775 test_move_location_extended(&mut machine, true, Size::S16)?;
8776 test_move_location_extended(&mut machine, true, Size::S32)?;
8777 test_binop_op(&mut machine, MachineARM64::emit_binop_add32)?;
8778 test_binop_op(&mut machine, MachineARM64::emit_binop_add64)?;
8779 test_binop_op(&mut machine, MachineARM64::emit_binop_sub32)?;
8780 test_binop_op(&mut machine, MachineARM64::emit_binop_sub64)?;
8781 test_binop_op(&mut machine, MachineARM64::emit_binop_and32)?;
8782 test_binop_op(&mut machine, MachineARM64::emit_binop_and64)?;
8783 test_binop_op(&mut machine, MachineARM64::emit_binop_xor32)?;
8784 test_binop_op(&mut machine, MachineARM64::emit_binop_xor64)?;
8785 test_binop_op(&mut machine, MachineARM64::emit_binop_or32)?;
8786 test_binop_op(&mut machine, MachineARM64::emit_binop_or64)?;
8787 test_binop_op(&mut machine, MachineARM64::emit_binop_mul32)?;
8788 test_binop_op(&mut machine, MachineARM64::emit_binop_mul64)?;
8789 test_float_binop_op(&mut machine, MachineARM64::f32_add)?;
8790 test_float_binop_op(&mut machine, MachineARM64::f32_sub)?;
8791 test_float_binop_op(&mut machine, MachineARM64::f32_mul)?;
8792 test_float_binop_op(&mut machine, MachineARM64::f32_div)?;
8793 test_float_cmp_op(&mut machine, MachineARM64::f32_cmp_eq)?;
8794 test_float_cmp_op(&mut machine, MachineARM64::f32_cmp_lt)?;
8795 test_float_cmp_op(&mut machine, MachineARM64::f32_cmp_le)?;
8796
8797 Ok(())
8798 }
8799}