1use std::collections::HashMap;
2
3use dynasmrt::{VecAssembler, aarch64::Aarch64Relocation};
4use fixedbitset::FixedBitSet;
5#[cfg(feature = "unwind")]
6use gimli::{AArch64, write::CallFrameInstruction};
7
8use wasmer_compiler::{
9 types::{
10 address_map::InstructionAddressMap,
11 function::FunctionBody,
12 relocation::{Relocation, RelocationKind, RelocationTarget},
13 section::CustomSection,
14 },
15 wasmparser::MemArg,
16};
17use wasmer_types::{
18 CompilationProgressCallback, CompileError, FunctionIndex, FunctionType, SourceLoc, TrapCode,
19 TrapInformation, VMOffsets,
20 target::{CallingConvention, CpuFeature, Target},
21};
22
23use crate::{
24 arm64_decl::{GPR, NEON},
25 codegen_error,
26 common_decl::*,
27 emitter_arm64::*,
28 location::{Location as AbstractLocation, Reg},
29 machine::*,
30 unwind::{UnwindInstructions, UnwindOps, UnwindRegister},
31};
32
33type Assembler = VecAssembler<Aarch64Relocation>;
34type Location = AbstractLocation<GPR, NEON>;
35
36pub struct MachineARM64 {
37 assembler: Assembler,
38 used_gprs: FixedBitSet,
39 used_simd: FixedBitSet,
40 trap_table: TrapTable,
41 instructions_address_map: Vec<InstructionAddressMap>,
44 src_loc: u32,
46 pushed: bool,
48 unwind_ops: Vec<(usize, UnwindOps<GPR, NEON>)>,
50 has_neon: bool,
52}
53
54pub(crate) const ARM64_RETURN_VALUE_REGISTERS: [GPR; 8] = [
57 GPR::X0,
58 GPR::X1,
59 GPR::X2,
60 GPR::X3,
61 GPR::X4,
62 GPR::X5,
63 GPR::X6,
64 GPR::X7,
65];
66
67#[allow(dead_code)]
68#[derive(PartialEq)]
69enum ImmType {
70 None,
71 NoneXzr,
72 Bits8,
73 Bits12,
74 Shift32,
75 Shift32No0,
76 Shift64,
77 Shift64No0,
78 Logical32,
79 Logical64,
80 UnscaledOffset,
81 OffsetByte,
82 OffsetHWord,
83 OffsetWord,
84 OffsetDWord,
85}
86
87const SCRATCH_REG: GPR = GPR::X17;
88
89#[allow(dead_code)]
90impl MachineARM64 {
91 pub fn new(target: Option<Target>) -> Self {
92 let has_neon = match target {
96 Some(ref target) => target.cpu_features().contains(CpuFeature::NEON),
97 None => false,
98 };
99
100 MachineARM64 {
101 assembler: Assembler::new(0),
102 used_gprs: FixedBitSet::with_capacity(32),
103 used_simd: FixedBitSet::with_capacity(32),
104 trap_table: TrapTable::default(),
105 instructions_address_map: vec![],
106 src_loc: 0,
107 pushed: false,
108 unwind_ops: vec![],
109 has_neon,
110 }
111 }
112 fn compatible_imm(&self, imm: i64, ty: ImmType) -> bool {
113 match ty {
114 ImmType::None => false,
115 ImmType::NoneXzr => false,
116 ImmType::Bits8 => (0..256).contains(&imm),
117 ImmType::Bits12 => (0..0x1000).contains(&imm),
118 ImmType::Shift32 => (0..32).contains(&imm),
119 ImmType::Shift32No0 => (1..32).contains(&imm),
120 ImmType::Shift64 => (0..64).contains(&imm),
121 ImmType::Shift64No0 => (1..64).contains(&imm),
122 ImmType::Logical32 => encode_logical_immediate_32bit(imm as u32).is_some(),
123 ImmType::Logical64 => encode_logical_immediate_64bit(imm as u64).is_some(),
124 ImmType::UnscaledOffset => (imm > -256) && (imm < 256),
125 ImmType::OffsetByte => (0..0x1000).contains(&imm),
126 ImmType::OffsetHWord => (imm & 1 == 0) && (0..0x2000).contains(&imm),
127 ImmType::OffsetWord => (imm & 3 == 0) && (0..0x4000).contains(&imm),
128 ImmType::OffsetDWord => (imm & 7 == 0) && (0..0x8000).contains(&imm),
129 }
130 }
131
132 fn location_to_reg(
133 &mut self,
134 sz: Size,
135 src: Location,
136 temps: &mut Vec<GPR>,
137 allow_imm: ImmType,
138 read_val: bool,
139 wanted: Option<GPR>,
140 ) -> Result<Location, CompileError> {
141 match src {
142 Location::GPR(_) | Location::SIMD(_) => Ok(src),
143 Location::Imm8(val) => {
144 if allow_imm == ImmType::NoneXzr && val == 0 {
145 Ok(Location::GPR(GPR::XzrSp))
146 } else if self.compatible_imm(val as i64, allow_imm) {
147 Ok(src)
148 } else {
149 let tmp = if let Some(wanted) = wanted {
150 wanted
151 } else {
152 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
153 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
154 })?;
155 temps.push(tmp);
156 tmp
157 };
158 self.assembler
159 .emit_mov_imm(Location::GPR(tmp), val as u64)?;
160 Ok(Location::GPR(tmp))
161 }
162 }
163 Location::Imm32(val) => {
164 if allow_imm == ImmType::NoneXzr && val == 0 {
165 Ok(Location::GPR(GPR::XzrSp))
166 } else if self.compatible_imm(val as i64, allow_imm) {
167 Ok(src)
168 } else {
169 let tmp = if let Some(wanted) = wanted {
170 wanted
171 } else {
172 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
173 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
174 })?;
175 temps.push(tmp);
176 tmp
177 };
178 self.assembler
179 .emit_mov_imm(Location::GPR(tmp), (val as i64) as u64)?;
180 Ok(Location::GPR(tmp))
181 }
182 }
183 Location::Imm64(val) => {
184 if allow_imm == ImmType::NoneXzr && val == 0 {
185 Ok(Location::GPR(GPR::XzrSp))
186 } else if self.compatible_imm(val as i64, allow_imm) {
187 Ok(src)
188 } else {
189 let tmp = if let Some(wanted) = wanted {
190 wanted
191 } else {
192 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
193 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
194 })?;
195 temps.push(tmp);
196 tmp
197 };
198 self.assembler
199 .emit_mov_imm(Location::GPR(tmp), val as u64)?;
200 Ok(Location::GPR(tmp))
201 }
202 }
203 Location::Memory(reg, val) => {
204 let tmp = if let Some(wanted) = wanted {
205 wanted
206 } else {
207 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
208 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
209 })?;
210 temps.push(tmp);
211 tmp
212 };
213 if read_val {
214 let offsize = match sz {
215 Size::S8 => ImmType::OffsetByte,
216 Size::S16 => ImmType::OffsetHWord,
217 Size::S32 => ImmType::OffsetWord,
218 Size::S64 => ImmType::OffsetDWord,
219 };
220 if sz == Size::S8 {
221 if self.compatible_imm(val as i64, offsize) {
222 self.assembler.emit_ldrb(
223 sz,
224 Location::GPR(tmp),
225 Location::Memory(reg, val as _),
226 )?;
227 } else {
228 if reg == tmp {
229 codegen_error!("singlepass reg==tmp unreachable");
230 }
231 self.assembler
232 .emit_mov_imm(Location::GPR(tmp), (val as i64) as u64)?;
233 self.assembler.emit_ldrb(
234 sz,
235 Location::GPR(tmp),
236 Location::Memory2(reg, tmp, Multiplier::One, 0),
237 )?;
238 }
239 } else if sz == Size::S16 {
240 if self.compatible_imm(val as i64, offsize) {
241 self.assembler.emit_ldrh(
242 sz,
243 Location::GPR(tmp),
244 Location::Memory(reg, val as _),
245 )?;
246 } else {
247 if reg == tmp {
248 codegen_error!("singlepass reg==tmp unreachable");
249 }
250 self.assembler
251 .emit_mov_imm(Location::GPR(tmp), (val as i64) as u64)?;
252 self.assembler.emit_ldrh(
253 sz,
254 Location::GPR(tmp),
255 Location::Memory2(reg, tmp, Multiplier::One, 0),
256 )?;
257 }
258 } else if self.compatible_imm(val as i64, offsize) {
259 self.assembler.emit_ldr(
260 sz,
261 Location::GPR(tmp),
262 Location::Memory(reg, val as _),
263 )?;
264 } else if self.compatible_imm(val as i64, ImmType::UnscaledOffset) {
265 self.assembler.emit_ldur(sz, Location::GPR(tmp), reg, val)?;
266 } else {
267 if reg == tmp {
268 codegen_error!("singlepass reg == tmp unreachable");
269 }
270 self.assembler
271 .emit_mov_imm(Location::GPR(tmp), (val as i64) as u64)?;
272 self.assembler.emit_ldr(
273 sz,
274 Location::GPR(tmp),
275 Location::Memory2(reg, tmp, Multiplier::One, 0),
276 )?;
277 }
278 }
279 Ok(Location::GPR(tmp))
280 }
281 _ => codegen_error!("singlepass can't emit location_to_reg {:?} {:?}", sz, src),
282 }
283 }
284 fn location_to_neon(
285 &mut self,
286 sz: Size,
287 src: Location,
288 temps: &mut Vec<NEON>,
289 allow_imm: ImmType,
290 read_val: bool,
291 ) -> Result<Location, CompileError> {
292 match src {
293 Location::SIMD(_) => Ok(src),
294 Location::GPR(_) => {
295 let tmp = self.acquire_temp_simd().ok_or_else(|| {
296 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
297 })?;
298 temps.push(tmp);
299 if read_val {
300 self.assembler.emit_mov(sz, src, Location::SIMD(tmp))?;
301 }
302 Ok(Location::SIMD(tmp))
303 }
304 Location::Imm8(val) => {
305 if self.compatible_imm(val as i64, allow_imm) {
306 Ok(src)
307 } else {
308 let gpr = self.acquire_temp_gpr().ok_or_else(|| {
309 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
310 })?;
311 let tmp = self.acquire_temp_simd().ok_or_else(|| {
312 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
313 })?;
314 temps.push(tmp);
315 self.assembler
316 .emit_mov_imm(Location::GPR(gpr), val as u64)?;
317 self.assembler
318 .emit_mov(sz, Location::GPR(gpr), Location::SIMD(tmp))?;
319 self.release_gpr(gpr);
320 Ok(Location::SIMD(tmp))
321 }
322 }
323 Location::Imm32(val) => {
324 if self.compatible_imm(val as i64, allow_imm) {
325 Ok(src)
326 } else {
327 let gpr = self.acquire_temp_gpr().ok_or_else(|| {
328 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
329 })?;
330 let tmp = self.acquire_temp_simd().ok_or_else(|| {
331 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
332 })?;
333 temps.push(tmp);
334 self.assembler
335 .emit_mov_imm(Location::GPR(gpr), (val as i64) as u64)?;
336 self.assembler
337 .emit_mov(sz, Location::GPR(gpr), Location::SIMD(tmp))?;
338 self.release_gpr(gpr);
339 Ok(Location::SIMD(tmp))
340 }
341 }
342 Location::Imm64(val) => {
343 if self.compatible_imm(val as i64, allow_imm) {
344 Ok(src)
345 } else {
346 let gpr = self.acquire_temp_gpr().ok_or_else(|| {
347 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
348 })?;
349 let tmp = self.acquire_temp_simd().ok_or_else(|| {
350 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
351 })?;
352 temps.push(tmp);
353 self.assembler
354 .emit_mov_imm(Location::GPR(gpr), val as u64)?;
355 self.assembler
356 .emit_mov(sz, Location::GPR(gpr), Location::SIMD(tmp))?;
357 self.release_gpr(gpr);
358 Ok(Location::SIMD(tmp))
359 }
360 }
361 Location::Memory(reg, val) => {
362 let tmp = self.acquire_temp_simd().ok_or_else(|| {
363 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
364 })?;
365 temps.push(tmp);
366 if read_val {
367 let offsize = if sz == Size::S32 {
368 ImmType::OffsetWord
369 } else {
370 ImmType::OffsetDWord
371 };
372 if self.compatible_imm(val as i64, offsize) {
373 self.assembler.emit_ldr(
374 sz,
375 Location::SIMD(tmp),
376 Location::Memory(reg, val as _),
377 )?;
378 } else if self.compatible_imm(val as i64, ImmType::UnscaledOffset) {
379 self.assembler
380 .emit_ldur(sz, Location::SIMD(tmp), reg, val)?;
381 } else {
382 let gpr = self.acquire_temp_gpr().ok_or_else(|| {
383 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
384 })?;
385 self.assembler
386 .emit_mov_imm(Location::GPR(gpr), (val as i64) as u64)?;
387 self.assembler.emit_ldr(
388 sz,
389 Location::SIMD(tmp),
390 Location::Memory2(reg, gpr, Multiplier::One, 0),
391 )?;
392 self.release_gpr(gpr);
393 }
394 }
395 Ok(Location::SIMD(tmp))
396 }
397 _ => codegen_error!("singlepass can't emit location_to_neon {:?} {:?}", sz, src),
398 }
399 }
400
401 fn emit_relaxed_binop(
402 &mut self,
403 op: fn(&mut Assembler, Size, Location, Location) -> Result<(), CompileError>,
404 sz: Size,
405 src: Location,
406 dst: Location,
407 putback: bool,
408 ) -> Result<(), CompileError> {
409 let mut temps = vec![];
410 let src_imm = if putback {
411 ImmType::None
412 } else {
413 ImmType::Bits12
414 };
415 let src = self.location_to_reg(sz, src, &mut temps, src_imm, true, None)?;
416 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, !putback, None)?;
417 op(&mut self.assembler, sz, src, dest)?;
418 if dst != dest && putback {
419 self.move_location(sz, dest, dst)?;
420 }
421 for r in temps {
422 self.release_gpr(r);
423 }
424 Ok(())
425 }
426 fn emit_relaxed_binop_neon(
427 &mut self,
428 op: fn(&mut Assembler, Size, Location, Location) -> Result<(), CompileError>,
429 sz: Size,
430 src: Location,
431 dst: Location,
432 putback: bool,
433 ) -> Result<(), CompileError> {
434 let mut temps = vec![];
435 let src = self.location_to_neon(sz, src, &mut temps, ImmType::None, true)?;
436 let dest = self.location_to_neon(sz, dst, &mut temps, ImmType::None, !putback)?;
437 op(&mut self.assembler, sz, src, dest)?;
438 if dst != dest && putback {
439 self.move_location(sz, dest, dst)?;
440 }
441 for r in temps {
442 self.release_simd(r);
443 }
444 Ok(())
445 }
446 fn emit_relaxed_binop3(
447 &mut self,
448 op: fn(&mut Assembler, Size, Location, Location, Location) -> Result<(), CompileError>,
449 sz: Size,
450 src1: Location,
451 src2: Location,
452 dst: Location,
453 allow_imm: ImmType,
454 ) -> Result<(), CompileError> {
455 let mut temps = vec![];
456 let src1 = self.location_to_reg(sz, src1, &mut temps, ImmType::None, true, None)?;
457 let src2 = self.location_to_reg(sz, src2, &mut temps, allow_imm, true, None)?;
458 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
459 op(&mut self.assembler, sz, src1, src2, dest)?;
460 if dst != dest {
461 self.move_location(sz, dest, dst)?;
462 }
463 for r in temps {
464 self.release_gpr(r);
465 }
466 Ok(())
467 }
468 fn emit_relaxed_binop3_neon(
469 &mut self,
470 op: fn(&mut Assembler, Size, Location, Location, Location) -> Result<(), CompileError>,
471 sz: Size,
472 src1: Location,
473 src2: Location,
474 dst: Location,
475 allow_imm: ImmType,
476 ) -> Result<(), CompileError> {
477 let mut temps = vec![];
478 let src1 = self.location_to_neon(sz, src1, &mut temps, ImmType::None, true)?;
479 let src2 = self.location_to_neon(sz, src2, &mut temps, allow_imm, true)?;
480 let dest = self.location_to_neon(sz, dst, &mut temps, ImmType::None, false)?;
481 op(&mut self.assembler, sz, src1, src2, dest)?;
482 if dst != dest {
483 self.move_location(sz, dest, dst)?;
484 }
485 for r in temps {
486 self.release_simd(r);
487 }
488 Ok(())
489 }
490 fn emit_relaxed_ldr64(
491 &mut self,
492 sz: Size,
493 dst: Location,
494 src: Location,
495 ) -> Result<(), CompileError> {
496 let mut temps = vec![];
497 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
498 match src {
499 Location::Memory(addr, offset) => {
500 if self.compatible_imm(offset as i64, ImmType::OffsetDWord) {
501 self.assembler.emit_ldr(Size::S64, dest, src)?;
502 } else if self.compatible_imm(offset as i64, ImmType::UnscaledOffset) {
503 self.assembler.emit_ldur(Size::S64, dest, addr, offset)?;
504 } else {
505 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
506 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
507 })?;
508 self.assembler
509 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
510 self.assembler.emit_ldr(
511 Size::S64,
512 dest,
513 Location::Memory2(addr, tmp, Multiplier::One, 0),
514 )?;
515 temps.push(tmp);
516 }
517 }
518 _ => codegen_error!("singlepass emit_relaxed_ldr64 unreachable"),
519 }
520 if dst != dest {
521 self.move_location(sz, dest, dst)?;
522 }
523 for r in temps {
524 self.release_gpr(r);
525 }
526 Ok(())
527 }
528 fn emit_relaxed_ldr32(
529 &mut self,
530 sz: Size,
531 dst: Location,
532 src: Location,
533 ) -> Result<(), CompileError> {
534 let mut temps = vec![];
535 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
536 match src {
537 Location::Memory(addr, offset) => {
538 if self.compatible_imm(offset as i64, ImmType::OffsetWord) {
539 self.assembler.emit_ldr(Size::S32, dest, src)?;
540 } else if self.compatible_imm(offset as i64, ImmType::UnscaledOffset) {
541 self.assembler.emit_ldur(Size::S32, dest, addr, offset)?;
542 } else {
543 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
544 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
545 })?;
546 self.assembler
547 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
548 self.assembler.emit_ldr(
549 Size::S32,
550 dest,
551 Location::Memory2(addr, tmp, Multiplier::One, 0),
552 )?;
553 temps.push(tmp);
554 }
555 }
556 _ => codegen_error!("singlepass emit_relaxed_ldr32 unreachable"),
557 }
558 if dst != dest {
559 self.move_location(sz, dest, dst)?;
560 }
561 for r in temps {
562 self.release_gpr(r);
563 }
564 Ok(())
565 }
566 fn emit_relaxed_ldr32s(
567 &mut self,
568 sz: Size,
569 dst: Location,
570 src: Location,
571 ) -> Result<(), CompileError> {
572 let mut temps = vec![];
573 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
574 match src {
575 Location::Memory(addr, offset) => {
576 if self.compatible_imm(offset as i64, ImmType::OffsetWord) {
577 self.assembler.emit_ldrsw(Size::S64, dest, src)?;
578 } else {
579 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
580 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
581 })?;
582 self.assembler
583 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
584 self.assembler.emit_ldrsw(
585 Size::S64,
586 dest,
587 Location::Memory2(addr, tmp, Multiplier::One, 0),
588 )?;
589 temps.push(tmp);
590 }
591 }
592 _ => codegen_error!("singplepass emit_relaxed_ldr32s unreachable"),
593 }
594 if dst != dest {
595 self.move_location(sz, dest, dst)?;
596 }
597 for r in temps {
598 self.release_gpr(r);
599 }
600 Ok(())
601 }
602 fn emit_relaxed_ldr16(
603 &mut self,
604 sz: Size,
605 dst: Location,
606 src: Location,
607 ) -> Result<(), CompileError> {
608 let mut temps = vec![];
609 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
610 match src {
611 Location::Memory(addr, offset) => {
612 if self.compatible_imm(offset as i64, ImmType::OffsetHWord) {
613 self.assembler.emit_ldrh(Size::S32, dest, src)?;
614 } else {
615 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
616 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
617 })?;
618 self.assembler
619 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
620 self.assembler.emit_ldrh(
621 Size::S32,
622 dest,
623 Location::Memory2(addr, tmp, Multiplier::One, 0),
624 )?;
625 temps.push(tmp);
626 }
627 }
628 _ => codegen_error!("singlpass emit_relaxed_ldr16 unreachable"),
629 }
630 if dst != dest {
631 self.move_location(sz, dest, dst)?;
632 }
633 for r in temps {
634 self.release_gpr(r);
635 }
636 Ok(())
637 }
638 fn emit_relaxed_ldr16s(
639 &mut self,
640 sz: Size,
641 dst: Location,
642 src: Location,
643 ) -> Result<(), CompileError> {
644 let mut temps = vec![];
645 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
646 match src {
647 Location::Memory(addr, offset) => {
648 if self.compatible_imm(offset as i64, ImmType::OffsetHWord) {
649 self.assembler.emit_ldrsh(sz, dest, src)?;
650 } else {
651 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
652 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
653 })?;
654 self.assembler
655 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
656 self.assembler.emit_ldrsh(
657 sz,
658 dest,
659 Location::Memory2(addr, tmp, Multiplier::One, 0),
660 )?;
661 temps.push(tmp);
662 }
663 }
664 _ => codegen_error!("singlepass emit_relaxed_ldr16s unreachable"),
665 }
666 if dst != dest {
667 self.move_location(sz, dest, dst)?;
668 }
669 for r in temps {
670 self.release_gpr(r);
671 }
672 Ok(())
673 }
674 fn emit_relaxed_ldr8(
675 &mut self,
676 sz: Size,
677 dst: Location,
678 src: Location,
679 ) -> Result<(), CompileError> {
680 let mut temps = vec![];
681 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
682 match src {
683 Location::Memory(addr, offset) => {
684 if self.compatible_imm(offset as i64, ImmType::OffsetByte) {
685 self.assembler.emit_ldrb(Size::S32, dest, src)?;
686 } else {
687 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
688 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
689 })?;
690 self.assembler
691 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
692 self.assembler.emit_ldrb(
693 Size::S32,
694 dest,
695 Location::Memory2(addr, tmp, Multiplier::One, 0),
696 )?;
697 temps.push(tmp);
698 }
699 }
700 _ => codegen_error!("singplepass emit_relaxed_ldr8 unreachable"),
701 }
702 if dst != dest {
703 self.move_location(sz, dest, dst)?;
704 }
705 for r in temps {
706 self.release_gpr(r);
707 }
708 Ok(())
709 }
710 fn emit_relaxed_ldr8s(
711 &mut self,
712 sz: Size,
713 dst: Location,
714 src: Location,
715 ) -> Result<(), CompileError> {
716 let mut temps = vec![];
717 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
718 match src {
719 Location::Memory(addr, offset) => {
720 if self.compatible_imm(offset as i64, ImmType::OffsetByte) {
721 self.assembler.emit_ldrsb(sz, dest, src)?;
722 } else {
723 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
724 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
725 })?;
726 self.assembler
727 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
728 self.assembler.emit_ldrsb(
729 sz,
730 dest,
731 Location::Memory2(addr, tmp, Multiplier::One, 0),
732 )?;
733 temps.push(tmp);
734 }
735 }
736 _ => codegen_error!("singlepass emit_relaxed_ldr8s unreachable"),
737 }
738 if dst != dest {
739 self.move_location(sz, dest, dst)?;
740 }
741 for r in temps {
742 self.release_gpr(r);
743 }
744 Ok(())
745 }
746 fn emit_relaxed_str64(&mut self, dst: Location, src: Location) -> Result<(), CompileError> {
747 let mut temps = vec![];
748 let dst = self.location_to_reg(Size::S64, dst, &mut temps, ImmType::NoneXzr, true, None)?;
749 match src {
750 Location::Memory(addr, offset) => {
751 if self.compatible_imm(offset as i64, ImmType::OffsetDWord) {
752 self.assembler.emit_str(Size::S64, dst, src)?;
753 } else if self.compatible_imm(offset as i64, ImmType::UnscaledOffset) {
754 self.assembler.emit_stur(Size::S64, dst, addr, offset)?;
755 } else {
756 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
757 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
758 })?;
759 self.assembler
760 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
761 self.assembler.emit_str(
762 Size::S64,
763 dst,
764 Location::Memory2(addr, tmp, Multiplier::One, 0),
765 )?;
766 temps.push(tmp);
767 }
768 }
769 _ => codegen_error!("singlepass can't emit str64 {:?} {:?}", dst, src),
770 }
771 for r in temps {
772 self.release_gpr(r);
773 }
774 Ok(())
775 }
776 fn emit_relaxed_str32(&mut self, dst: Location, src: Location) -> Result<(), CompileError> {
777 let mut temps = vec![];
778 let dst = self.location_to_reg(Size::S64, dst, &mut temps, ImmType::NoneXzr, true, None)?;
779 match src {
780 Location::Memory(addr, offset) => {
781 if self.compatible_imm(offset as i64, ImmType::OffsetWord) {
782 self.assembler.emit_str(Size::S32, dst, src)?;
783 } else if self.compatible_imm(offset as i64, ImmType::UnscaledOffset) {
784 self.assembler.emit_stur(Size::S32, dst, addr, offset)?;
785 } else {
786 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
787 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
788 })?;
789 self.assembler
790 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
791 self.assembler.emit_str(
792 Size::S32,
793 dst,
794 Location::Memory2(addr, tmp, Multiplier::One, 0),
795 )?;
796 temps.push(tmp);
797 }
798 }
799 _ => codegen_error!("singplepass emit_relaxed_str32 unreachable"),
800 }
801 for r in temps {
802 self.release_gpr(r);
803 }
804 Ok(())
805 }
806 fn emit_relaxed_str16(&mut self, dst: Location, src: Location) -> Result<(), CompileError> {
807 let mut temps = vec![];
808 let dst = self.location_to_reg(Size::S64, dst, &mut temps, ImmType::NoneXzr, true, None)?;
809 match src {
810 Location::Memory(addr, offset) => {
811 if self.compatible_imm(offset as i64, ImmType::OffsetHWord) {
812 self.assembler.emit_strh(Size::S32, dst, src)?;
813 } else {
814 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
815 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
816 })?;
817 self.assembler
818 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
819 self.assembler.emit_strh(
820 Size::S32,
821 dst,
822 Location::Memory2(addr, tmp, Multiplier::One, 0),
823 )?;
824 temps.push(tmp);
825 }
826 }
827 _ => codegen_error!("singlepass emit_relaxed_str16 unreachable"),
828 }
829 for r in temps {
830 self.release_gpr(r);
831 }
832 Ok(())
833 }
834 fn emit_relaxed_str8(&mut self, dst: Location, src: Location) -> Result<(), CompileError> {
835 let mut temps = vec![];
836 let dst = self.location_to_reg(Size::S64, dst, &mut temps, ImmType::NoneXzr, true, None)?;
837 match src {
838 Location::Memory(addr, offset) => {
839 if self.compatible_imm(offset as i64, ImmType::OffsetByte) {
840 self.assembler
841 .emit_strb(Size::S32, dst, Location::Memory(addr, offset))?;
842 } else {
843 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
844 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
845 })?;
846 self.assembler
847 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
848 self.assembler.emit_strb(
849 Size::S32,
850 dst,
851 Location::Memory2(addr, tmp, Multiplier::One, 0),
852 )?;
853 temps.push(tmp);
854 }
855 }
856 _ => codegen_error!("singlepass emit_relaxed_str8 unreachable"),
857 }
858 for r in temps {
859 self.release_gpr(r);
860 }
861 Ok(())
862 }
863 fn emit_cmpop_i64_dynamic_b(
865 &mut self,
866 c: Condition,
867 loc_a: Location,
868 loc_b: Location,
869 ret: Location,
870 ) -> Result<(), CompileError> {
871 match ret {
872 Location::GPR(_) => {
873 self.emit_relaxed_cmp(Size::S64, loc_b, loc_a)?;
874 self.assembler.emit_cset(Size::S32, ret, c)?;
875 }
876 Location::Memory(_, _) => {
877 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
878 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
879 })?;
880 self.emit_relaxed_cmp(Size::S64, loc_b, loc_a)?;
881 self.assembler.emit_cset(Size::S32, Location::GPR(tmp), c)?;
882 self.move_location(Size::S32, Location::GPR(tmp), ret)?;
883 self.release_gpr(tmp);
884 }
885 _ => {
886 codegen_error!("singlepass emit_compop_i64_dynamic_b unreachable");
887 }
888 }
889 Ok(())
890 }
891 fn emit_cmpop_i32_dynamic_b(
893 &mut self,
894 c: Condition,
895 loc_a: Location,
896 loc_b: Location,
897 ret: Location,
898 ) -> Result<(), CompileError> {
899 match ret {
900 Location::GPR(_) => {
901 self.emit_relaxed_cmp(Size::S32, loc_b, loc_a)?;
902 self.assembler.emit_cset(Size::S32, ret, c)?;
903 }
904 Location::Memory(_, _) => {
905 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
906 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
907 })?;
908 self.emit_relaxed_cmp(Size::S32, loc_b, loc_a)?;
909 self.assembler.emit_cset(Size::S32, Location::GPR(tmp), c)?;
910 self.move_location(Size::S32, Location::GPR(tmp), ret)?;
911 self.release_gpr(tmp);
912 }
913 _ => {
914 codegen_error!("singlepass emit_cmpop_i32_dynamic_b unreachable");
915 }
916 }
917 Ok(())
918 }
919
920 #[allow(clippy::too_many_arguments)]
921 fn memory_op<F: FnOnce(&mut Self, GPR) -> Result<(), CompileError>>(
922 &mut self,
923 addr: Location,
924 memarg: &MemArg,
925 check_alignment: bool,
926 value_size: usize,
927 need_check: bool,
928 imported_memories: bool,
929 offset: i32,
930 heap_access_oob: Label,
931 unaligned_atomic: Label,
932 cb: F,
933 ) -> Result<(), CompileError> {
934 let tmp_addr = self.acquire_temp_gpr().ok_or_else(|| {
935 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
936 })?;
937
938 let (base_loc, bound_loc) = if imported_memories {
940 self.emit_relaxed_binop(
942 Assembler::emit_mov,
943 Size::S64,
944 Location::Memory(self.get_vmctx_reg(), offset),
945 Location::GPR(tmp_addr),
946 true,
947 )?;
948 (Location::Memory(tmp_addr, 0), Location::Memory(tmp_addr, 8))
949 } else {
950 (
951 Location::Memory(self.get_vmctx_reg(), offset),
952 Location::Memory(self.get_vmctx_reg(), offset + 8),
953 )
954 };
955
956 let tmp_base = self.acquire_temp_gpr().ok_or_else(|| {
957 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
958 })?;
959 let tmp_bound = self.acquire_temp_gpr().ok_or_else(|| {
960 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
961 })?;
962
963 self.emit_relaxed_ldr64(Size::S64, Location::GPR(tmp_base), base_loc)?;
965
966 if need_check {
968 self.emit_relaxed_ldr64(Size::S64, Location::GPR(tmp_bound), bound_loc)?;
969
970 self.assembler.emit_add(
974 Size::S64,
975 Location::GPR(tmp_bound),
976 Location::GPR(tmp_base),
977 Location::GPR(tmp_bound),
978 )?;
979 if self.compatible_imm(value_size as _, ImmType::Bits12) {
980 self.assembler.emit_sub(
981 Size::S64,
982 Location::GPR(tmp_bound),
983 Location::Imm32(value_size as _),
984 Location::GPR(tmp_bound),
985 )?;
986 } else {
987 let tmp2 = self.acquire_temp_gpr().ok_or_else(|| {
988 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
989 })?;
990 self.assembler
991 .emit_mov_imm(Location::GPR(tmp2), value_size as u64)?;
992 self.assembler.emit_sub(
993 Size::S64,
994 Location::GPR(tmp_bound),
995 Location::GPR(tmp2),
996 Location::GPR(tmp_bound),
997 )?;
998 self.release_gpr(tmp2);
999 }
1000 }
1001
1002 self.move_location(Size::S32, addr, Location::GPR(tmp_addr))?;
1006
1007 if memarg.offset != 0 {
1009 if self.compatible_imm(memarg.offset as _, ImmType::Bits12) {
1010 self.assembler.emit_adds(
1011 Size::S32,
1012 Location::Imm32(memarg.offset as u32),
1013 Location::GPR(tmp_addr),
1014 Location::GPR(tmp_addr),
1015 )?;
1016 } else {
1017 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
1018 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1019 })?;
1020 self.assembler
1021 .emit_mov_imm(Location::GPR(tmp), memarg.offset as _)?;
1022 self.assembler.emit_adds(
1023 Size::S32,
1024 Location::GPR(tmp_addr),
1025 Location::GPR(tmp),
1026 Location::GPR(tmp_addr),
1027 )?;
1028 self.release_gpr(tmp);
1029 }
1030
1031 self.assembler
1033 .emit_bcond_label_far(Condition::Cs, heap_access_oob)?;
1034 }
1035
1036 self.assembler.emit_add(
1038 Size::S64,
1039 Location::GPR(tmp_base),
1040 Location::GPR(tmp_addr),
1041 Location::GPR(tmp_addr),
1042 )?;
1043
1044 if need_check {
1045 self.assembler.emit_cmp(
1047 Size::S64,
1048 Location::GPR(tmp_bound),
1049 Location::GPR(tmp_addr),
1050 )?;
1051
1052 self.assembler
1054 .emit_bcond_label_far(Condition::Hi, heap_access_oob)?;
1055 }
1056
1057 self.release_gpr(tmp_bound);
1058 self.release_gpr(tmp_base);
1059
1060 let align = value_size as u32;
1061 if check_alignment && align != 1 {
1062 self.assembler.emit_tst(
1063 Size::S64,
1064 Location::Imm32(align - 1),
1065 Location::GPR(tmp_addr),
1066 )?;
1067 self.assembler
1068 .emit_bcond_label_far(Condition::Ne, unaligned_atomic)?;
1069 }
1070 let begin = self.assembler.get_offset().0;
1071 cb(self, tmp_addr)?;
1072 let end = self.assembler.get_offset().0;
1073 self.mark_address_range_with_trap_code(TrapCode::HeapAccessOutOfBounds, begin, end);
1074
1075 self.release_gpr(tmp_addr);
1076 Ok(())
1077 }
1078
1079 fn offset_is_ok(&self, size: Size, offset: i32) -> bool {
1080 if offset < 0 {
1081 return false;
1082 }
1083 let shift = size.bytes().trailing_zeros() as i32;
1084 if offset >= 0x1000 << shift {
1085 return false;
1086 }
1087 if (offset & ((1 << shift) - 1)) != 0 {
1088 return false;
1089 }
1090 true
1091 }
1092
1093 fn emit_push(&mut self, sz: Size, src: Location) -> Result<(), CompileError> {
1094 match (sz, src) {
1095 (Size::S64, Location::GPR(_)) | (Size::S64, Location::SIMD(_)) => {
1096 let offset = if self.pushed {
1097 0
1098 } else {
1099 self.assembler.emit_sub(
1100 Size::S64,
1101 Location::GPR(GPR::XzrSp),
1102 Location::Imm8(16),
1103 Location::GPR(GPR::XzrSp),
1104 )?;
1105 8
1106 };
1107 self.assembler
1108 .emit_stur(Size::S64, src, GPR::XzrSp, offset)?;
1109 self.pushed = !self.pushed;
1110 }
1111 (Size::S64, _) => {
1112 let mut temps = vec![];
1113 let src = self.location_to_reg(sz, src, &mut temps, ImmType::None, true, None)?;
1114 let offset = if self.pushed {
1115 0
1116 } else {
1117 self.assembler.emit_sub(
1118 Size::S64,
1119 Location::GPR(GPR::XzrSp),
1120 Location::Imm8(16),
1121 Location::GPR(GPR::XzrSp),
1122 )?;
1123 8
1124 };
1125 self.assembler
1126 .emit_stur(Size::S64, src, GPR::XzrSp, offset)?;
1127 self.pushed = !self.pushed;
1128 for r in temps {
1129 self.release_gpr(r);
1130 }
1131 }
1132 _ => codegen_error!("singlepass can't emit PUSH {:?} {:?}", sz, src),
1133 }
1134 Ok(())
1135 }
1136 fn emit_double_push(
1137 &mut self,
1138 sz: Size,
1139 src1: Location,
1140 src2: Location,
1141 ) -> Result<(), CompileError> {
1142 if !self.pushed {
1143 match (sz, src1, src2) {
1144 (Size::S64, Location::GPR(_), Location::GPR(_)) => {
1145 self.assembler
1146 .emit_stpdb(Size::S64, src1, src2, GPR::XzrSp, 16)?;
1147 }
1148 _ => {
1149 self.emit_push(sz, src1)?;
1150 self.emit_push(sz, src2)?;
1151 }
1152 }
1153 } else {
1154 self.emit_push(sz, src1)?;
1155 self.emit_push(sz, src2)?;
1156 }
1157 Ok(())
1158 }
1159 fn emit_pop(&mut self, sz: Size, dst: Location) -> Result<(), CompileError> {
1160 match (sz, dst) {
1161 (Size::S64, Location::GPR(_)) | (Size::S64, Location::SIMD(_)) => {
1162 let offset = if self.pushed { 8 } else { 0 };
1163 self.assembler
1164 .emit_ldur(Size::S64, dst, GPR::XzrSp, offset)?;
1165 if self.pushed {
1166 self.assembler.emit_add(
1167 Size::S64,
1168 Location::GPR(GPR::XzrSp),
1169 Location::Imm8(16),
1170 Location::GPR(GPR::XzrSp),
1171 )?;
1172 }
1173 self.pushed = !self.pushed;
1174 }
1175 _ => codegen_error!("singlepass can't emit POP {:?} {:?}", sz, dst),
1176 }
1177 Ok(())
1178 }
1179 fn emit_double_pop(
1180 &mut self,
1181 sz: Size,
1182 dst1: Location,
1183 dst2: Location,
1184 ) -> Result<(), CompileError> {
1185 if !self.pushed {
1186 match (sz, dst1, dst2) {
1187 (Size::S64, Location::GPR(_), Location::GPR(_)) => {
1188 self.assembler
1189 .emit_ldpia(Size::S64, dst1, dst2, GPR::XzrSp, 16)?;
1190 }
1191 _ => {
1192 self.emit_pop(sz, dst2)?;
1193 self.emit_pop(sz, dst1)?;
1194 }
1195 }
1196 } else {
1197 self.emit_pop(sz, dst2)?;
1198 self.emit_pop(sz, dst1)?;
1199 }
1200 Ok(())
1201 }
1202
1203 fn set_default_nan(&mut self, temps: &mut Vec<GPR>) -> Result<GPR, CompileError> {
1204 let old_fpcr = self.acquire_temp_gpr().ok_or_else(|| {
1206 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1207 })?;
1208 temps.push(old_fpcr);
1209 self.assembler.emit_read_fpcr(old_fpcr)?;
1210 let new_fpcr = self.acquire_temp_gpr().ok_or_else(|| {
1211 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1212 })?;
1213 temps.push(new_fpcr);
1214 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
1215 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1216 })?;
1217 temps.push(tmp);
1218 self.assembler
1219 .emit_mov(Size::S32, Location::Imm32(1), Location::GPR(tmp))?;
1220 self.assembler
1221 .emit_mov(Size::S64, Location::GPR(old_fpcr), Location::GPR(new_fpcr))?;
1222 self.assembler.emit_bfi(
1224 Size::S64,
1225 Location::GPR(tmp),
1226 25,
1227 1,
1228 Location::GPR(new_fpcr),
1229 )?;
1230 self.assembler.emit_write_fpcr(new_fpcr)?;
1231 Ok(old_fpcr)
1232 }
1233 fn set_trap_enabled(&mut self, temps: &mut Vec<GPR>) -> Result<GPR, CompileError> {
1234 let old_fpcr = self.acquire_temp_gpr().ok_or_else(|| {
1236 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1237 })?;
1238 temps.push(old_fpcr);
1239 self.assembler.emit_read_fpcr(old_fpcr)?;
1240 let new_fpcr = self.acquire_temp_gpr().ok_or_else(|| {
1241 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1242 })?;
1243 temps.push(new_fpcr);
1244 self.assembler
1245 .emit_mov(Size::S64, Location::GPR(old_fpcr), Location::GPR(new_fpcr))?;
1246 self.assembler
1248 .emit_bfc(Size::S64, 8, 1, Location::GPR(new_fpcr))?;
1249 self.assembler.emit_write_fpcr(new_fpcr)?;
1250 Ok(old_fpcr)
1251 }
1252 fn restore_fpcr(&mut self, old_fpcr: GPR) -> Result<(), CompileError> {
1253 self.assembler.emit_write_fpcr(old_fpcr)
1254 }
1255
1256 fn reset_exception_fpsr(&mut self) -> Result<(), CompileError> {
1257 let fpsr = self.acquire_temp_gpr().ok_or_else(|| {
1259 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1260 })?;
1261 self.assembler.emit_read_fpsr(fpsr)?;
1262 self.assembler
1264 .emit_bfc(Size::S64, 0, 1, Location::GPR(fpsr))?;
1265 self.assembler.emit_write_fpsr(fpsr)?;
1266 self.release_gpr(fpsr);
1267 Ok(())
1268 }
1269 fn read_fpsr(&mut self) -> Result<GPR, CompileError> {
1270 let fpsr = self.acquire_temp_gpr().ok_or_else(|| {
1271 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1272 })?;
1273 self.assembler.emit_read_fpsr(fpsr)?;
1274 Ok(fpsr)
1275 }
1276
1277 fn trap_float_conversion_errors(
1278 &mut self,
1279 old_fpcr: GPR,
1280 sz: Size,
1281 f: Location,
1282 temps: &mut Vec<GPR>,
1283 ) -> Result<(), CompileError> {
1284 let trap_badconv = self.assembler.get_label();
1285 let end = self.assembler.get_label();
1286
1287 let fpsr = self.read_fpsr()?;
1288 temps.push(fpsr);
1289 self.assembler
1291 .emit_tbz_label(Size::S32, Location::GPR(fpsr), 0, end)?;
1292 self.assembler
1294 .emit_bfc(Size::S64, 0, 4, Location::GPR(fpsr))?;
1295 self.restore_fpcr(old_fpcr)?;
1296 self.assembler.emit_fcmp(sz, f, f)?;
1297 self.assembler
1298 .emit_bcond_label(Condition::Vs, trap_badconv)?;
1299 self.emit_illegal_op_internal(TrapCode::IntegerOverflow)?;
1301
1302 self.emit_label(trap_badconv)?;
1303 self.emit_illegal_op_internal(TrapCode::BadConversionToInteger)?;
1304
1305 self.emit_label(end)?;
1306 self.restore_fpcr(old_fpcr)
1307 }
1308
1309 fn used_gprs_contains(&self, r: &GPR) -> bool {
1310 self.used_gprs.contains(r.into_index())
1311 }
1312 fn used_simd_contains(&self, r: &NEON) -> bool {
1313 self.used_simd.contains(r.into_index())
1314 }
1315 fn used_gprs_insert(&mut self, r: GPR) {
1316 self.used_gprs.insert(r.into_index());
1317 }
1318 fn used_simd_insert(&mut self, r: NEON) {
1319 self.used_simd.insert(r.into_index());
1320 }
1321 fn used_gprs_remove(&mut self, r: &GPR) -> bool {
1322 let ret = self.used_gprs_contains(r);
1323 self.used_gprs.set(r.into_index(), false);
1324 ret
1325 }
1326 fn used_simd_remove(&mut self, r: &NEON) -> bool {
1327 let ret = self.used_simd_contains(r);
1328 self.used_simd.set(r.into_index(), false);
1329 ret
1330 }
1331 fn emit_unwind_op(&mut self, op: UnwindOps<GPR, NEON>) {
1332 self.unwind_ops.push((self.get_offset().0, op));
1333 }
1334 fn emit_illegal_op_internal(&mut self, trap: TrapCode) -> Result<(), CompileError> {
1335 self.assembler.emit_udf(0xc0 | (trap as u8) as u16)
1336 }
1337}
1338
1339impl Machine for MachineARM64 {
1340 type GPR = GPR;
1341 type SIMD = NEON;
1342
1343 const STACK_ALIGNMENT: usize = 16;
1344
1345 fn assembler_get_offset(&self) -> Offset {
1346 self.assembler.get_offset()
1347 }
1348
1349 fn get_vmctx_reg(&self) -> GPR {
1350 GPR::X28
1351 }
1352
1353 fn get_used_gprs(&self) -> Vec<GPR> {
1354 GPR::iterator()
1355 .filter(|x| self.used_gprs.contains(x.into_index()))
1356 .cloned()
1357 .collect()
1358 }
1359
1360 fn get_used_simd(&self) -> Vec<NEON> {
1361 NEON::iterator()
1362 .filter(|x| self.used_simd.contains(x.into_index()))
1363 .cloned()
1364 .collect()
1365 }
1366
1367 fn pick_gpr(&self) -> Option<GPR> {
1368 use GPR::*;
1369 static REGS: &[GPR] = &[X9, X10, X11, X12, X13, X14, X15];
1370 for r in REGS {
1371 if !self.used_gprs_contains(r) {
1372 return Some(*r);
1373 }
1374 }
1375 None
1376 }
1377
1378 fn pick_temp_gpr(&self) -> Option<GPR> {
1379 use GPR::*;
1380 static REGS: &[GPR] = &[X8, X7, X6, X5, X4, X3, X2, X1];
1381 for r in REGS {
1382 if !self.used_gprs_contains(r) {
1383 return Some(*r);
1384 }
1385 }
1386 None
1387 }
1388
1389 fn acquire_temp_gpr(&mut self) -> Option<GPR> {
1390 let gpr = self.pick_temp_gpr();
1391 if let Some(x) = gpr {
1392 self.used_gprs_insert(x);
1393 }
1394 gpr
1395 }
1396
1397 fn release_gpr(&mut self, gpr: GPR) {
1398 assert!(self.used_gprs_remove(&gpr));
1399 }
1400
1401 fn reserve_unused_temp_gpr(&mut self, gpr: GPR) -> GPR {
1402 assert!(!self.used_gprs_contains(&gpr));
1403 self.used_gprs_insert(gpr);
1404 gpr
1405 }
1406
1407 fn reserve_gpr(&mut self, gpr: GPR) {
1408 self.used_gprs_insert(gpr);
1409 }
1410
1411 fn push_used_gpr(&mut self, used_gprs: &[GPR]) -> Result<usize, CompileError> {
1412 if used_gprs.len() % 2 == 1 {
1413 self.emit_push(Size::S64, Location::GPR(GPR::XzrSp))?;
1414 }
1415 for r in used_gprs.iter() {
1416 self.emit_push(Size::S64, Location::GPR(*r))?;
1417 }
1418 Ok(used_gprs.len().div_ceil(2) * 16)
1419 }
1420
1421 fn pop_used_gpr(&mut self, used_gprs: &[GPR]) -> Result<(), CompileError> {
1422 for r in used_gprs.iter().rev() {
1423 self.emit_pop(Size::S64, Location::GPR(*r))?;
1424 }
1425 if used_gprs.len() % 2 == 1 {
1426 self.emit_pop(Size::S64, Location::GPR(GPR::XzrSp))?;
1427 }
1428 Ok(())
1429 }
1430
1431 fn pick_simd(&self) -> Option<NEON> {
1432 use NEON::*;
1433 static REGS: &[NEON] = &[V8, V9, V10, V11, V12];
1434 for r in REGS {
1435 if !self.used_simd_contains(r) {
1436 return Some(*r);
1437 }
1438 }
1439 None
1440 }
1441
1442 fn pick_temp_simd(&self) -> Option<NEON> {
1443 use NEON::*;
1444 static REGS: &[NEON] = &[V0, V1, V2, V3, V4, V5, V6, V7];
1445 for r in REGS {
1446 if !self.used_simd_contains(r) {
1447 return Some(*r);
1448 }
1449 }
1450 None
1451 }
1452
1453 fn acquire_temp_simd(&mut self) -> Option<NEON> {
1454 let simd = self.pick_temp_simd();
1455 if let Some(x) = simd {
1456 self.used_simd_insert(x);
1457 }
1458 simd
1459 }
1460
1461 fn reserve_simd(&mut self, simd: NEON) {
1462 self.used_simd_insert(simd);
1463 }
1464
1465 fn release_simd(&mut self, simd: NEON) {
1466 assert!(self.used_simd_remove(&simd));
1467 }
1468
1469 fn push_used_simd(&mut self, used_neons: &[NEON]) -> Result<usize, CompileError> {
1470 let stack_adjust = if used_neons.len() % 2 == 1 {
1471 (used_neons.len() * 8) as u32 + 8
1472 } else {
1473 (used_neons.len() * 8) as u32
1474 };
1475 self.extend_stack(stack_adjust)?;
1476
1477 for (i, r) in used_neons.iter().enumerate() {
1478 self.assembler.emit_str(
1479 Size::S64,
1480 Location::SIMD(*r),
1481 Location::Memory(GPR::XzrSp, (i * 8) as i32),
1482 )?;
1483 }
1484 Ok(stack_adjust as usize)
1485 }
1486
1487 fn pop_used_simd(&mut self, used_neons: &[NEON]) -> Result<(), CompileError> {
1488 for (i, r) in used_neons.iter().enumerate() {
1489 self.assembler.emit_ldr(
1490 Size::S64,
1491 Location::SIMD(*r),
1492 Location::Memory(GPR::XzrSp, (i * 8) as i32),
1493 )?;
1494 }
1495 let stack_adjust = if used_neons.len() % 2 == 1 {
1496 (used_neons.len() * 8) as u32 + 8
1497 } else {
1498 (used_neons.len() * 8) as u32
1499 };
1500 self.assembler.emit_add(
1501 Size::S64,
1502 Location::GPR(GPR::XzrSp),
1503 Location::Imm32(stack_adjust as _),
1504 Location::GPR(GPR::XzrSp),
1505 )
1506 }
1507
1508 fn set_srcloc(&mut self, offset: u32) {
1509 self.src_loc = offset;
1510 }
1511
1512 fn mark_address_range_with_trap_code(&mut self, code: TrapCode, begin: usize, end: usize) {
1513 for i in begin..end {
1514 self.trap_table.offset_to_code.insert(i, code);
1515 }
1516 self.mark_instruction_address_end(begin);
1517 }
1518
1519 fn mark_address_with_trap_code(&mut self, code: TrapCode) {
1520 let offset = self.assembler.get_offset().0;
1521 self.trap_table.offset_to_code.insert(offset, code);
1522 self.mark_instruction_address_end(offset);
1523 }
1524
1525 fn mark_instruction_with_trap_code(&mut self, code: TrapCode) -> usize {
1526 let offset = self.assembler.get_offset().0;
1527 self.trap_table.offset_to_code.insert(offset, code);
1528 offset
1529 }
1530
1531 fn mark_instruction_address_end(&mut self, begin: usize) {
1532 self.instructions_address_map.push(InstructionAddressMap {
1533 srcloc: SourceLoc::new(self.src_loc),
1534 code_offset: begin,
1535 code_len: self.assembler.get_offset().0 - begin,
1536 });
1537 }
1538
1539 fn insert_stackoverflow(&mut self) {
1540 let offset = 0;
1541 self.trap_table
1542 .offset_to_code
1543 .insert(offset, TrapCode::StackOverflow);
1544 self.mark_instruction_address_end(offset);
1545 }
1546
1547 fn collect_trap_information(&self) -> Vec<TrapInformation> {
1548 self.trap_table
1549 .offset_to_code
1550 .clone()
1551 .into_iter()
1552 .map(|(offset, code)| TrapInformation {
1553 code_offset: offset as u32,
1554 trap_code: code,
1555 })
1556 .collect()
1557 }
1558
1559 fn instructions_address_map(&self) -> Vec<InstructionAddressMap> {
1560 self.instructions_address_map.clone()
1561 }
1562
1563 fn local_on_stack(&mut self, stack_offset: i32) -> Location {
1564 Location::Memory(GPR::X29, -stack_offset)
1565 }
1566
1567 fn extend_stack(&mut self, delta_stack_offset: u32) -> Result<(), CompileError> {
1568 let delta = if self.compatible_imm(delta_stack_offset as _, ImmType::Bits12) {
1569 Location::Imm32(delta_stack_offset as _)
1570 } else {
1571 let tmp = SCRATCH_REG;
1572 self.assembler
1573 .emit_mov_imm(Location::GPR(tmp), delta_stack_offset as u64)?;
1574 Location::GPR(tmp)
1575 };
1576 self.assembler.emit_sub(
1577 Size::S64,
1578 Location::GPR(GPR::XzrSp),
1579 delta,
1580 Location::GPR(GPR::XzrSp),
1581 )
1582 }
1583
1584 fn truncate_stack(&mut self, delta_stack_offset: u32) -> Result<(), CompileError> {
1585 let delta = if self.compatible_imm(delta_stack_offset as _, ImmType::Bits12) {
1586 Location::Imm32(delta_stack_offset as _)
1587 } else {
1588 let tmp = SCRATCH_REG;
1589 self.assembler
1590 .emit_mov_imm(Location::GPR(tmp), delta_stack_offset as u64)?;
1591 Location::GPR(tmp)
1592 };
1593 self.assembler.emit_add(
1594 Size::S64,
1595 Location::GPR(GPR::XzrSp),
1596 delta,
1597 Location::GPR(GPR::XzrSp),
1598 )
1599 }
1600
1601 fn move_location_for_native(
1602 &mut self,
1603 size: Size,
1604 loc: Location,
1605 dest: Location,
1606 ) -> Result<(), CompileError> {
1607 match loc {
1608 Location::Imm64(_)
1609 | Location::Imm32(_)
1610 | Location::Imm8(_)
1611 | Location::Memory(_, _)
1612 | Location::Memory2(_, _, _, _) => {
1613 self.move_location(size, loc, Location::GPR(SCRATCH_REG))?;
1614 self.move_location(size, Location::GPR(SCRATCH_REG), dest)
1615 }
1616 _ => self.move_location(size, loc, dest),
1617 }
1618 }
1619
1620 fn zero_location(&mut self, size: Size, location: Location) -> Result<(), CompileError> {
1621 self.move_location(size, Location::GPR(GPR::XzrSp), location)
1622 }
1623
1624 fn local_pointer(&self) -> GPR {
1625 GPR::X29
1626 }
1627
1628 fn is_local_on_stack(&self, idx: usize) -> bool {
1629 idx > 7
1630 }
1631
1632 fn get_local_location(&self, idx: usize, callee_saved_regs_size: usize) -> Location {
1633 match idx {
1635 0 => Location::GPR(GPR::X19),
1636 1 => Location::GPR(GPR::X20),
1637 2 => Location::GPR(GPR::X21),
1638 3 => Location::GPR(GPR::X22),
1639 4 => Location::GPR(GPR::X23),
1640 5 => Location::GPR(GPR::X24),
1641 6 => Location::GPR(GPR::X25),
1642 7 => Location::GPR(GPR::X26),
1643 _ => Location::Memory(GPR::X29, -(((idx - 7) * 8 + callee_saved_regs_size) as i32)),
1644 }
1645 }
1646
1647 fn move_local(&mut self, stack_offset: i32, location: Location) -> Result<(), CompileError> {
1648 if stack_offset < 256 {
1649 self.assembler
1650 .emit_stur(Size::S64, location, GPR::X29, -stack_offset)?;
1651 } else {
1652 let tmp = SCRATCH_REG;
1653 if stack_offset < 0x1_0000 {
1654 self.assembler
1655 .emit_mov_imm(Location::GPR(tmp), (-stack_offset as i64) as u64)?;
1656 self.assembler.emit_str(
1657 Size::S64,
1658 location,
1659 Location::Memory2(GPR::X29, tmp, Multiplier::One, 0),
1660 )?;
1661 } else {
1662 self.assembler
1663 .emit_mov_imm(Location::GPR(tmp), (stack_offset as i64) as u64)?;
1664 self.assembler.emit_sub(
1665 Size::S64,
1666 Location::GPR(GPR::X29),
1667 Location::GPR(tmp),
1668 Location::GPR(tmp),
1669 )?;
1670 self.assembler
1671 .emit_str(Size::S64, location, Location::GPR(tmp))?;
1672 }
1673 }
1674 match location {
1675 Location::GPR(x) => self.emit_unwind_op(UnwindOps::SaveRegister {
1676 reg: UnwindRegister::<GPR, NEON>::GPR(x),
1677 bp_neg_offset: stack_offset,
1678 }),
1679 Location::SIMD(x) => self.emit_unwind_op(UnwindOps::SaveRegister {
1680 reg: UnwindRegister::FPR(x),
1681 bp_neg_offset: stack_offset,
1682 }),
1683 _ => (),
1684 }
1685 Ok(())
1686 }
1687
1688 fn get_param_registers(&self) -> &'static [Self::GPR] {
1689 &[
1690 GPR::X0,
1691 GPR::X1,
1692 GPR::X2,
1693 GPR::X3,
1694 GPR::X4,
1695 GPR::X5,
1696 GPR::X6,
1697 GPR::X7,
1698 ]
1699 }
1700
1701 fn get_param_location(
1702 &self,
1703 idx: usize,
1704 sz: Size,
1705 stack_args: &mut usize,
1706 calling_convention: CallingConvention,
1707 ) -> Location {
1708 let register_params = self.get_param_registers();
1709 match calling_convention {
1710 CallingConvention::AppleAarch64 => register_params.get(idx).map_or_else(
1711 || {
1712 let sz = sz.bytes() as usize;
1714 *stack_args = (*stack_args).next_multiple_of(sz);
1715 let loc = Location::Memory(GPR::XzrSp, *stack_args as i32);
1716 *stack_args += sz;
1717 loc
1718 },
1719 |reg| Location::GPR(*reg),
1720 ),
1721 _ => {
1722 if let Some(reg) = register_params.get(idx) {
1723 Location::GPR(*reg)
1724 } else {
1725 let loc = Location::Memory(GPR::XzrSp, *stack_args as i32);
1726 *stack_args += 8;
1727 loc
1728 }
1729 }
1730 }
1731 }
1732
1733 fn get_call_param_location(
1734 &self,
1735 return_slots: usize,
1736 idx: usize,
1737 sz: Size,
1738 stack_args: &mut usize,
1739 calling_convention: CallingConvention,
1740 ) -> Location {
1741 let register_params = self.get_param_registers();
1742 let return_values_memory_size =
1743 8 * return_slots.saturating_sub(ARM64_RETURN_VALUE_REGISTERS.len()) as i32;
1744
1745 match calling_convention {
1746 CallingConvention::AppleAarch64 => register_params.get(idx).map_or_else(
1747 || {
1748 let sz = sz.bytes() as usize;
1749 *stack_args = (*stack_args).next_multiple_of(sz);
1751 let loc = Location::Memory(
1752 GPR::X29,
1753 16 * 2 + return_values_memory_size + *stack_args as i32,
1754 );
1755 *stack_args += sz;
1756 loc
1757 },
1758 |reg| Location::GPR(*reg),
1759 ),
1760 _ => register_params.get(idx).map_or_else(
1761 || {
1762 let loc = Location::Memory(
1763 GPR::X29,
1764 16 * 2 + return_values_memory_size + *stack_args as i32,
1765 );
1766 *stack_args += 8;
1767 loc
1768 },
1769 |reg| Location::GPR(*reg),
1770 ),
1771 }
1772 }
1773
1774 fn get_simple_param_location(&self, idx: usize) -> Self::GPR {
1775 self.get_param_registers()[idx]
1776 }
1777
1778 fn adjust_gpr_param_location(
1779 &mut self,
1780 _register: Self::GPR,
1781 _size: Size,
1782 ) -> Result<(), CompileError> {
1783 Ok(())
1784 }
1785
1786 fn get_return_value_location(
1787 &self,
1788 idx: usize,
1789 stack_location: &mut usize,
1790 ) -> AbstractLocation<Self::GPR, Self::SIMD> {
1791 ARM64_RETURN_VALUE_REGISTERS.get(idx).map_or_else(
1792 || {
1793 let loc = Location::Memory(GPR::XzrSp, *stack_location as i32);
1794 *stack_location += 8;
1795 loc
1796 },
1797 |reg| Location::GPR(*reg),
1798 )
1799 }
1800
1801 fn get_call_return_value_location(
1802 &self,
1803 idx: usize,
1804 ) -> AbstractLocation<Self::GPR, Self::SIMD> {
1805 ARM64_RETURN_VALUE_REGISTERS.get(idx).map_or_else(
1806 || {
1807 Location::Memory(
1808 GPR::X29,
1809 (16 * 2 + (idx - ARM64_RETURN_VALUE_REGISTERS.len()) * 8) as i32,
1810 )
1811 },
1812 |reg| Location::GPR(*reg),
1813 )
1814 }
1815
1816 fn move_location(
1817 &mut self,
1818 size: Size,
1819 source: Location,
1820 dest: Location,
1821 ) -> Result<(), CompileError> {
1822 match source {
1823 Location::GPR(_) | Location::SIMD(_) => match dest {
1824 Location::GPR(_) | Location::SIMD(_) => self.assembler.emit_mov(size, source, dest),
1825 Location::Memory(addr, offs) => {
1826 if self.offset_is_ok(size, offs) {
1827 self.assembler.emit_str(size, source, dest)
1828 } else if self.compatible_imm(offs as i64, ImmType::UnscaledOffset) {
1829 self.assembler.emit_stur(size, source, addr, offs)
1830 } else {
1831 let tmp = SCRATCH_REG;
1832 if offs < 0 {
1833 self.assembler
1834 .emit_mov_imm(Location::GPR(tmp), (-offs) as u64)?;
1835 self.assembler.emit_sub(
1836 Size::S64,
1837 Location::GPR(addr),
1838 Location::GPR(tmp),
1839 Location::GPR(tmp),
1840 )?;
1841 } else {
1842 self.assembler
1843 .emit_mov_imm(Location::GPR(tmp), offs as u64)?;
1844 self.assembler.emit_add(
1845 Size::S64,
1846 Location::GPR(addr),
1847 Location::GPR(tmp),
1848 Location::GPR(tmp),
1849 )?;
1850 }
1851 self.assembler
1852 .emit_str(size, source, Location::Memory(tmp, 0))
1853 }
1854 }
1855 _ => codegen_error!(
1856 "singlepass can't emit move_location {:?} {:?} => {:?}",
1857 size,
1858 source,
1859 dest
1860 ),
1861 },
1862 Location::Imm8(_) => match dest {
1863 Location::GPR(_) => self.assembler.emit_mov(size, source, dest),
1864 Location::Memory(_, _) => match size {
1865 Size::S64 => self.emit_relaxed_str64(source, dest),
1866 Size::S32 => self.emit_relaxed_str32(source, dest),
1867 Size::S16 => self.emit_relaxed_str16(source, dest),
1868 Size::S8 => self.emit_relaxed_str8(source, dest),
1869 },
1870 _ => codegen_error!(
1871 "singlepass can't emit move_location {:?} {:?} => {:?}",
1872 size,
1873 source,
1874 dest
1875 ),
1876 },
1877 Location::Imm32(val) => match dest {
1878 Location::GPR(_) => self.assembler.emit_mov_imm(dest, val as u64),
1879 Location::Memory(_, _) => match size {
1880 Size::S64 => self.emit_relaxed_str64(source, dest),
1881 Size::S32 => self.emit_relaxed_str32(source, dest),
1882 Size::S16 => self.emit_relaxed_str16(source, dest),
1883 Size::S8 => self.emit_relaxed_str8(source, dest),
1884 },
1885 _ => codegen_error!(
1886 "singlepass can't emit move_location {:?} {:?} => {:?}",
1887 size,
1888 source,
1889 dest
1890 ),
1891 },
1892 Location::Imm64(val) => match dest {
1893 Location::GPR(_) => self.assembler.emit_mov_imm(dest, val),
1894 Location::Memory(_, _) => match size {
1895 Size::S64 => self.emit_relaxed_str64(source, dest),
1896 Size::S32 => self.emit_relaxed_str32(source, dest),
1897 Size::S16 => self.emit_relaxed_str16(source, dest),
1898 Size::S8 => self.emit_relaxed_str8(source, dest),
1899 },
1900 _ => codegen_error!(
1901 "singlepass can't emit move_location {:?} {:?} => {:?}",
1902 size,
1903 source,
1904 dest
1905 ),
1906 },
1907 Location::Memory(addr, offs) => match dest {
1908 Location::GPR(_) | Location::SIMD(_) => {
1909 if self.offset_is_ok(size, offs) {
1910 self.assembler.emit_ldr(size, dest, source)
1911 } else if offs > -256 && offs < 256 {
1912 self.assembler.emit_ldur(size, dest, addr, offs)
1913 } else {
1914 let tmp = SCRATCH_REG;
1915 if offs < 0 {
1916 self.assembler
1917 .emit_mov_imm(Location::GPR(tmp), (-offs) as u64)?;
1918 self.assembler.emit_sub(
1919 Size::S64,
1920 Location::GPR(addr),
1921 Location::GPR(tmp),
1922 Location::GPR(tmp),
1923 )?;
1924 } else {
1925 self.assembler
1926 .emit_mov_imm(Location::GPR(tmp), offs as u64)?;
1927 self.assembler.emit_add(
1928 Size::S64,
1929 Location::GPR(addr),
1930 Location::GPR(tmp),
1931 Location::GPR(tmp),
1932 )?;
1933 }
1934 self.assembler
1935 .emit_ldr(size, dest, Location::Memory(tmp, 0))
1936 }
1937 }
1938 _ => {
1939 let mut temps = vec![];
1940 let src =
1941 self.location_to_reg(size, source, &mut temps, ImmType::None, true, None)?;
1942 self.move_location(size, src, dest)?;
1943 for r in temps {
1944 self.release_gpr(r);
1945 }
1946 Ok(())
1947 }
1948 },
1949 _ => codegen_error!(
1950 "singlepass can't emit move_location {:?} {:?} => {:?}",
1951 size,
1952 source,
1953 dest
1954 ),
1955 }
1956 }
1957
1958 fn move_location_extend(
1959 &mut self,
1960 size_val: Size,
1961 signed: bool,
1962 source: Location,
1963 size_op: Size,
1964 dest: Location,
1965 ) -> Result<(), CompileError> {
1966 if size_op != Size::S64 {
1967 codegen_error!("singlepass move_location_extend unreachable");
1968 }
1969 let mut temps = vec![];
1970 let dst = self.location_to_reg(size_op, dest, &mut temps, ImmType::None, false, None)?;
1971 let src = match (size_val, signed, source) {
1972 (Size::S64, _, _) => source,
1973 (Size::S32, false, Location::GPR(_)) => {
1974 self.assembler.emit_mov(size_val, source, dst)?;
1975 dst
1976 }
1977 (Size::S8, false, Location::GPR(_)) => {
1978 self.assembler.emit_uxtb(size_op, source, dst)?;
1979 dst
1980 }
1981 (Size::S16, false, Location::GPR(_)) => {
1982 self.assembler.emit_uxth(size_op, source, dst)?;
1983 dst
1984 }
1985 (Size::S8, true, Location::GPR(_)) => {
1986 self.assembler.emit_sxtb(size_op, source, dst)?;
1987 dst
1988 }
1989 (Size::S16, true, Location::GPR(_)) => {
1990 self.assembler.emit_sxth(size_op, source, dst)?;
1991 dst
1992 }
1993 (Size::S32, true, Location::GPR(_)) => {
1994 self.assembler.emit_sxtw(size_op, source, dst)?;
1995 dst
1996 }
1997 (Size::S32, false, Location::Memory(_, _)) => {
1998 self.emit_relaxed_ldr32(size_op, dst, source)?;
1999 dst
2000 }
2001 (Size::S32, true, Location::Memory(_, _)) => {
2002 self.emit_relaxed_ldr32s(size_op, dst, source)?;
2003 dst
2004 }
2005 (Size::S16, false, Location::Memory(_, _)) => {
2006 self.emit_relaxed_ldr16(size_op, dst, source)?;
2007 dst
2008 }
2009 (Size::S16, true, Location::Memory(_, _)) => {
2010 self.emit_relaxed_ldr16s(size_op, dst, source)?;
2011 dst
2012 }
2013 (Size::S8, false, Location::Memory(_, _)) => {
2014 self.emit_relaxed_ldr8(size_op, dst, source)?;
2015 dst
2016 }
2017 (Size::S8, true, Location::Memory(_, _)) => {
2018 self.emit_relaxed_ldr8s(size_op, dst, source)?;
2019 dst
2020 }
2021 _ => codegen_error!(
2022 "singlepass can't emit move_location_extend {:?} {:?} {:?} => {:?} {:?}",
2023 size_val,
2024 signed,
2025 source,
2026 size_op,
2027 dest
2028 ),
2029 };
2030 if src != dst {
2031 self.move_location(size_op, src, dst)?;
2032 }
2033 if dst != dest {
2034 self.move_location(size_op, dst, dest)?;
2035 }
2036 for r in temps {
2037 self.release_gpr(r);
2038 }
2039 Ok(())
2040 }
2041
2042 fn init_stack_loc(
2043 &mut self,
2044 init_stack_loc_cnt: u64,
2045 last_stack_loc: Location,
2046 ) -> Result<(), CompileError> {
2047 let label = self.assembler.get_label();
2048 let mut temps = vec![];
2049 let dest = self.acquire_temp_gpr().ok_or_else(|| {
2050 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2051 })?;
2052 temps.push(dest);
2053 let cnt = self.location_to_reg(
2054 Size::S64,
2055 Location::Imm64(init_stack_loc_cnt),
2056 &mut temps,
2057 ImmType::None,
2058 true,
2059 None,
2060 )?;
2061 let dest = match last_stack_loc {
2062 Location::GPR(_) => codegen_error!("singlepass init_stack_loc unreachable"),
2063 Location::SIMD(_) => codegen_error!("singlepass init_stack_loc unreachable"),
2064 Location::Memory(reg, offset) => {
2065 if offset < 0 {
2066 let offset = (-offset) as u32;
2067 if self.compatible_imm(offset as i64, ImmType::Bits12) {
2068 self.assembler.emit_sub(
2069 Size::S64,
2070 Location::GPR(reg),
2071 Location::Imm32(offset),
2072 Location::GPR(dest),
2073 )?;
2074 } else {
2075 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2076 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2077 })?;
2078 self.assembler
2079 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
2080 self.assembler.emit_sub(
2081 Size::S64,
2082 Location::GPR(reg),
2083 Location::GPR(tmp),
2084 Location::GPR(dest),
2085 )?;
2086 temps.push(tmp);
2087 }
2088 dest
2089 } else {
2090 let offset = offset as u32;
2091 if self.compatible_imm(offset as i64, ImmType::Bits12) {
2092 self.assembler.emit_add(
2093 Size::S64,
2094 Location::GPR(reg),
2095 Location::Imm32(offset),
2096 Location::GPR(dest),
2097 )?;
2098 } else {
2099 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2100 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2101 })?;
2102 self.assembler
2103 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
2104 self.assembler.emit_add(
2105 Size::S64,
2106 Location::GPR(reg),
2107 Location::GPR(tmp),
2108 Location::GPR(dest),
2109 )?;
2110 temps.push(tmp);
2111 }
2112 dest
2113 }
2114 }
2115 _ => codegen_error!("singlepass can't emit init_stack_loc {:?}", last_stack_loc),
2116 };
2117 self.assembler.emit_label(label)?;
2118 self.assembler
2119 .emit_stria(Size::S64, Location::GPR(GPR::XzrSp), dest, 8)?;
2120 self.assembler
2121 .emit_sub(Size::S64, cnt, Location::Imm8(1), cnt)?;
2122 self.assembler.emit_cbnz_label(Size::S64, cnt, label)?;
2123 for r in temps {
2124 self.release_gpr(r);
2125 }
2126 Ok(())
2127 }
2128
2129 fn restore_saved_area(&mut self, saved_area_offset: i32) -> Result<(), CompileError> {
2130 let real_delta = if saved_area_offset & 15 != 0 {
2131 self.pushed = true;
2132 saved_area_offset + 8
2133 } else {
2134 self.pushed = false;
2135 saved_area_offset
2136 };
2137 if self.compatible_imm(real_delta as _, ImmType::Bits12) {
2138 self.assembler.emit_sub(
2139 Size::S64,
2140 Location::GPR(GPR::X29),
2141 Location::Imm32(real_delta as _),
2142 Location::GPR(GPR::XzrSp),
2143 )?;
2144 } else {
2145 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2146 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2147 })?;
2148 self.assembler
2149 .emit_mov_imm(Location::GPR(tmp), real_delta as u64)?;
2150 self.assembler.emit_sub(
2151 Size::S64,
2152 Location::GPR(GPR::X29),
2153 Location::GPR(tmp),
2154 Location::GPR(GPR::XzrSp),
2155 )?;
2156 self.release_gpr(tmp);
2157 }
2158 Ok(())
2159 }
2160
2161 fn pop_location(&mut self, location: Location) -> Result<(), CompileError> {
2162 self.emit_pop(Size::S64, location)
2163 }
2164
2165 fn assembler_finalize(
2166 self,
2167 assembly_comments: HashMap<usize, AssemblyComment>,
2168 ) -> Result<FinalizedAssembly, CompileError> {
2169 Ok(FinalizedAssembly {
2170 body: self.assembler.finalize().map_err(|e| {
2171 CompileError::Codegen(format!("Assembler failed finalization with: {e:?}"))
2172 })?,
2173 assembly_comments,
2174 })
2175 }
2176
2177 fn get_offset(&self) -> Offset {
2178 self.assembler.get_offset()
2179 }
2180
2181 fn finalize_function(&mut self) -> Result<(), CompileError> {
2182 self.assembler.finalize_function();
2183 Ok(())
2184 }
2185
2186 fn emit_function_prolog(&mut self) -> Result<(), CompileError> {
2187 self.emit_double_push(Size::S64, Location::GPR(GPR::X29), Location::GPR(GPR::X30))?; self.emit_unwind_op(UnwindOps::Push2Regs {
2189 reg1: UnwindRegister::GPR(GPR::X29),
2190 reg2: UnwindRegister::GPR(GPR::X30),
2191 up_to_sp: 16,
2192 });
2193 self.emit_double_push(Size::S64, Location::GPR(GPR::X27), Location::GPR(GPR::X28))?;
2194 self.emit_unwind_op(UnwindOps::Push2Regs {
2195 reg1: UnwindRegister::GPR(GPR::X27),
2196 reg2: UnwindRegister::GPR(GPR::X28),
2197 up_to_sp: 32,
2198 });
2199 self.assembler.emit_add(
2201 Size::S64,
2202 Location::GPR(GPR::XzrSp),
2203 Location::Imm8(0),
2204 Location::GPR(GPR::X29),
2205 )?;
2206 self.emit_unwind_op(UnwindOps::DefineNewFrame);
2207 Ok(())
2208 }
2209
2210 fn emit_function_epilog(&mut self) -> Result<(), CompileError> {
2211 self.assembler.emit_add(
2213 Size::S64,
2214 Location::GPR(GPR::X29),
2215 Location::Imm8(0),
2216 Location::GPR(GPR::XzrSp),
2217 )?;
2218 self.pushed = false; self.emit_double_pop(Size::S64, Location::GPR(GPR::X27), Location::GPR(GPR::X28))?;
2220 self.emit_double_pop(Size::S64, Location::GPR(GPR::X29), Location::GPR(GPR::X30))?;
2221 Ok(())
2222 }
2223
2224 fn emit_function_return_float(&mut self) -> Result<(), CompileError> {
2225 self.assembler
2226 .emit_mov(Size::S64, Location::GPR(GPR::X0), Location::SIMD(NEON::V0))
2227 }
2228
2229 fn canonicalize_nan(
2230 &mut self,
2231 sz: Size,
2232 input: Location,
2233 output: Location,
2234 ) -> Result<(), CompileError> {
2235 let mut tempn = vec![];
2236 let mut temps = vec![];
2237 let old_fpcr = self.set_default_nan(&mut temps)?;
2238 match (sz, input, output) {
2240 (Size::S32, Location::SIMD(_), Location::SIMD(_)) => {
2241 self.assembler.emit_fmax(sz, input, input, output)?;
2242 }
2243 (Size::S64, Location::SIMD(_), Location::SIMD(_)) => {
2244 self.assembler.emit_fmax(sz, input, input, output)?;
2245 }
2246 (Size::S32, Location::SIMD(_), _) | (Size::S64, Location::SIMD(_), _) => {
2247 let tmp = self.location_to_neon(sz, output, &mut tempn, ImmType::None, false)?;
2248 self.assembler.emit_fmax(sz, input, input, tmp)?;
2249 self.move_location(sz, tmp, output)?;
2250 }
2251 (Size::S32, Location::Memory(_, _), _) | (Size::S64, Location::Memory(_, _), _) => {
2252 let src = self.location_to_neon(sz, input, &mut tempn, ImmType::None, true)?;
2253 let tmp = self.location_to_neon(sz, output, &mut tempn, ImmType::None, false)?;
2254 self.assembler.emit_fmax(sz, src, src, tmp)?;
2255 if tmp != output {
2256 self.move_location(sz, tmp, output)?;
2257 }
2258 }
2259 _ => codegen_error!(
2260 "singlepass can't emit canonicalize_nan {:?} {:?} {:?}",
2261 sz,
2262 input,
2263 output
2264 ),
2265 }
2266
2267 self.restore_fpcr(old_fpcr)?;
2268 for r in temps {
2269 self.release_gpr(r);
2270 }
2271 for r in tempn {
2272 self.release_simd(r);
2273 }
2274 Ok(())
2275 }
2276
2277 fn emit_illegal_op(&mut self, trap: TrapCode) -> Result<(), CompileError> {
2278 let offset = self.assembler.get_offset().0;
2279 self.assembler.emit_udf(0xc0 | (trap as u8) as u16)?;
2280 self.mark_instruction_address_end(offset);
2281 Ok(())
2282 }
2283
2284 fn get_label(&mut self) -> Label {
2285 self.assembler.new_dynamic_label()
2286 }
2287
2288 fn emit_label(&mut self, label: Label) -> Result<(), CompileError> {
2289 self.assembler.emit_label(label)
2290 }
2291
2292 fn get_gpr_for_call(&self) -> GPR {
2293 GPR::X27
2294 }
2295
2296 fn emit_call_register(&mut self, reg: GPR) -> Result<(), CompileError> {
2297 self.assembler.emit_call_register(reg)
2298 }
2299
2300 fn emit_call_label(&mut self, label: Label) -> Result<(), CompileError> {
2301 self.assembler.emit_call_label(label)
2302 }
2303
2304 fn arch_emit_indirect_call_with_trampoline(
2305 &mut self,
2306 location: Location,
2307 ) -> Result<(), CompileError> {
2308 self.assembler
2309 .arch_emit_indirect_call_with_trampoline(location)
2310 }
2311
2312 fn emit_debug_breakpoint(&mut self) -> Result<(), CompileError> {
2313 self.assembler.emit_brk()
2314 }
2315
2316 fn emit_call_location(&mut self, location: Location) -> Result<(), CompileError> {
2317 let mut temps = vec![];
2318 let loc = self.location_to_reg(
2319 Size::S64,
2320 location,
2321 &mut temps,
2322 ImmType::None,
2323 true,
2324 Some(self.get_gpr_for_call()),
2325 )?;
2326 match loc {
2327 Location::GPR(reg) => self.assembler.emit_call_register(reg),
2328 _ => codegen_error!("singlepass can't emit CALL Location"),
2329 }?;
2330 for r in temps {
2331 self.release_gpr(r);
2332 }
2333 Ok(())
2334 }
2335
2336 fn location_add(
2337 &mut self,
2338 size: Size,
2339 source: Location,
2340 dest: Location,
2341 flags: bool,
2342 ) -> Result<(), CompileError> {
2343 let mut temps = vec![];
2344 let src = self.location_to_reg(size, source, &mut temps, ImmType::Bits12, true, None)?;
2345 let dst = self.location_to_reg(size, dest, &mut temps, ImmType::None, true, None)?;
2346 if flags {
2347 self.assembler.emit_adds(size, dst, src, dst)?;
2348 } else {
2349 self.assembler.emit_add(size, dst, src, dst)?;
2350 }
2351 if dst != dest {
2352 self.move_location(size, dst, dest)?;
2353 }
2354 for r in temps {
2355 self.release_gpr(r);
2356 }
2357 Ok(())
2358 }
2359
2360 fn location_cmp(
2361 &mut self,
2362 size: Size,
2363 source: Location,
2364 dest: Location,
2365 ) -> Result<(), CompileError> {
2366 self.emit_relaxed_binop(Assembler::emit_cmp, size, source, dest, false)
2367 }
2368
2369 fn jmp_unconditional(&mut self, label: Label) -> Result<(), CompileError> {
2370 self.assembler.emit_b_label(label)
2371 }
2372
2373 fn jmp_on_condition(
2374 &mut self,
2375 cond: UnsignedCondition,
2376 size: Size,
2377 loc_a: AbstractLocation<Self::GPR, Self::SIMD>,
2378 loc_b: AbstractLocation<Self::GPR, Self::SIMD>,
2379 label: Label,
2380 ) -> Result<(), CompileError> {
2381 self.emit_relaxed_binop(Assembler::emit_cmp, size, loc_b, loc_a, false)?;
2382 let cond = match cond {
2383 UnsignedCondition::Equal => Condition::Eq,
2384 UnsignedCondition::NotEqual => Condition::Ne,
2385 UnsignedCondition::Above => Condition::Hi,
2386 UnsignedCondition::AboveEqual => Condition::Cs,
2387 UnsignedCondition::Below => Condition::Cc,
2388 UnsignedCondition::BelowEqual => Condition::Ls,
2389 };
2390 self.assembler.emit_bcond_label_far(cond, label)
2391 }
2392
2393 fn emit_jmp_to_jumptable(&mut self, label: Label, cond: Location) -> Result<(), CompileError> {
2394 let tmp1 = self.acquire_temp_gpr().ok_or_else(|| {
2395 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2396 })?;
2397 let tmp2 = self.acquire_temp_gpr().ok_or_else(|| {
2398 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2399 })?;
2400
2401 self.assembler.emit_load_label(tmp1, label)?;
2402 self.move_location(Size::S32, cond, Location::GPR(tmp2))?;
2403
2404 self.assembler.emit_add_lsl(
2405 Size::S64,
2406 Location::GPR(tmp1),
2407 Location::GPR(tmp2),
2408 2,
2409 Location::GPR(tmp2),
2410 )?;
2411 self.assembler.emit_b_register(tmp2)?;
2412 self.release_gpr(tmp2);
2413 self.release_gpr(tmp1);
2414 Ok(())
2415 }
2416
2417 fn align_for_loop(&mut self) -> Result<(), CompileError> {
2418 Ok(())
2420 }
2421
2422 fn emit_ret(&mut self) -> Result<(), CompileError> {
2423 self.assembler.emit_ret()
2424 }
2425
2426 fn emit_push(&mut self, size: Size, loc: Location) -> Result<(), CompileError> {
2427 self.emit_push(size, loc)
2428 }
2429
2430 fn emit_pop(&mut self, size: Size, loc: Location) -> Result<(), CompileError> {
2431 self.emit_pop(size, loc)
2432 }
2433
2434 fn emit_memory_fence(&mut self) -> Result<(), CompileError> {
2435 self.assembler.emit_dmb()
2436 }
2437
2438 fn emit_imul_imm32(&mut self, size: Size, imm32: u32, gpr: GPR) -> Result<(), CompileError> {
2439 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2440 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2441 })?;
2442 self.assembler
2443 .emit_mov_imm(Location::GPR(tmp), imm32 as u64)?;
2444 self.assembler.emit_mul(
2445 size,
2446 Location::GPR(gpr),
2447 Location::GPR(tmp),
2448 Location::GPR(gpr),
2449 )?;
2450 self.release_gpr(tmp);
2451 Ok(())
2452 }
2453
2454 fn emit_relaxed_mov(
2455 &mut self,
2456 sz: Size,
2457 src: Location,
2458 dst: Location,
2459 ) -> Result<(), CompileError> {
2460 self.emit_relaxed_binop(Assembler::emit_mov, sz, src, dst, true)
2461 }
2462
2463 fn emit_relaxed_cmp(
2464 &mut self,
2465 sz: Size,
2466 src: Location,
2467 dst: Location,
2468 ) -> Result<(), CompileError> {
2469 self.emit_relaxed_binop(Assembler::emit_cmp, sz, src, dst, false)
2470 }
2471
2472 fn emit_relaxed_sign_extension(
2473 &mut self,
2474 sz_src: Size,
2475 src: Location,
2476 sz_dst: Size,
2477 dst: Location,
2478 ) -> Result<(), CompileError> {
2479 match (src, dst) {
2480 (Location::Memory(_, _), Location::GPR(_)) => match sz_src {
2481 Size::S8 => self.emit_relaxed_ldr8s(sz_dst, dst, src),
2482 Size::S16 => self.emit_relaxed_ldr16s(sz_dst, dst, src),
2483 Size::S32 => self.emit_relaxed_ldr32s(sz_dst, dst, src),
2484 _ => codegen_error!("singlepass emit_relaxed_sign_extension unreachable"),
2485 },
2486 _ => {
2487 let mut temps = vec![];
2488 let src =
2489 self.location_to_reg(sz_src, src, &mut temps, ImmType::None, true, None)?;
2490 let dest =
2491 self.location_to_reg(sz_dst, dst, &mut temps, ImmType::None, false, None)?;
2492 match sz_src {
2493 Size::S8 => self.assembler.emit_sxtb(sz_dst, src, dest),
2494 Size::S16 => self.assembler.emit_sxth(sz_dst, src, dest),
2495 Size::S32 => self.assembler.emit_sxtw(sz_dst, src, dest),
2496 _ => codegen_error!("singlepass emit_relaxed_sign_extension unreachable"),
2497 }?;
2498 if dst != dest {
2499 self.move_location(sz_dst, dest, dst)?;
2500 }
2501 for r in temps {
2502 self.release_gpr(r);
2503 }
2504 Ok(())
2505 }
2506 }
2507 }
2508
2509 fn emit_binop_add32(
2510 &mut self,
2511 loc_a: Location,
2512 loc_b: Location,
2513 ret: Location,
2514 ) -> Result<(), CompileError> {
2515 self.emit_relaxed_binop3(
2516 Assembler::emit_add,
2517 Size::S32,
2518 loc_a,
2519 loc_b,
2520 ret,
2521 ImmType::Bits12,
2522 )
2523 }
2524
2525 fn emit_binop_sub32(
2526 &mut self,
2527 loc_a: Location,
2528 loc_b: Location,
2529 ret: Location,
2530 ) -> Result<(), CompileError> {
2531 self.emit_relaxed_binop3(
2532 Assembler::emit_sub,
2533 Size::S32,
2534 loc_a,
2535 loc_b,
2536 ret,
2537 ImmType::Bits12,
2538 )
2539 }
2540
2541 fn emit_binop_mul32(
2542 &mut self,
2543 loc_a: Location,
2544 loc_b: Location,
2545 ret: Location,
2546 ) -> Result<(), CompileError> {
2547 self.emit_relaxed_binop3(
2548 Assembler::emit_mul,
2549 Size::S32,
2550 loc_a,
2551 loc_b,
2552 ret,
2553 ImmType::None,
2554 )
2555 }
2556
2557 fn emit_binop_udiv32(
2558 &mut self,
2559 loc_a: Location,
2560 loc_b: Location,
2561 ret: Location,
2562 integer_division_by_zero: Label,
2563 ) -> Result<usize, CompileError> {
2564 let mut temps = vec![];
2565 let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
2566 let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
2567 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
2568
2569 self.assembler
2570 .emit_cbz_label_far(Size::S32, src2, integer_division_by_zero)?;
2571 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
2572 self.assembler.emit_udiv(Size::S32, src1, src2, dest)?;
2573 if ret != dest {
2574 self.move_location(Size::S32, dest, ret)?;
2575 }
2576 for r in temps {
2577 self.release_gpr(r);
2578 }
2579 Ok(offset)
2580 }
2581
2582 fn emit_binop_sdiv32(
2583 &mut self,
2584 loc_a: Location,
2585 loc_b: Location,
2586 ret: Location,
2587 integer_division_by_zero: Label,
2588 integer_overflow: Label,
2589 ) -> Result<usize, CompileError> {
2590 let mut temps = vec![];
2591 let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
2592 let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
2593 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
2594
2595 self.assembler
2596 .emit_cbz_label_far(Size::S32, src2, integer_division_by_zero)?;
2597 let label_nooverflow = self.assembler.get_label();
2598 let tmp = self.location_to_reg(
2599 Size::S32,
2600 Location::Imm32(0x80000000),
2601 &mut temps,
2602 ImmType::None,
2603 true,
2604 None,
2605 )?;
2606 self.assembler.emit_cmp(Size::S32, tmp, src1)?;
2607 self.assembler
2608 .emit_bcond_label(Condition::Ne, label_nooverflow)?;
2609 self.assembler.emit_movn(Size::S32, tmp, 0)?;
2610 self.assembler.emit_cmp(Size::S32, tmp, src2)?;
2611 self.assembler
2612 .emit_bcond_label_far(Condition::Eq, integer_overflow)?;
2613 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
2614 self.assembler.emit_label(label_nooverflow)?;
2615 self.assembler.emit_sdiv(Size::S32, src1, src2, dest)?;
2616 if ret != dest {
2617 self.move_location(Size::S32, dest, ret)?;
2618 }
2619 for r in temps {
2620 self.release_gpr(r);
2621 }
2622 Ok(offset)
2623 }
2624
2625 fn emit_binop_urem32(
2626 &mut self,
2627 loc_a: Location,
2628 loc_b: Location,
2629 ret: Location,
2630 integer_division_by_zero: Label,
2631 ) -> Result<usize, CompileError> {
2632 let mut temps = vec![];
2633 let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
2634 let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
2635 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
2636 let dest = if dest == src1 || dest == src2 {
2637 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2638 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2639 })?;
2640 temps.push(tmp);
2641 self.assembler
2642 .emit_mov(Size::S32, dest, Location::GPR(tmp))?;
2643 Location::GPR(tmp)
2644 } else {
2645 dest
2646 };
2647 self.assembler
2648 .emit_cbz_label_far(Size::S32, src2, integer_division_by_zero)?;
2649 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
2650 self.assembler.emit_udiv(Size::S32, src1, src2, dest)?;
2651 self.assembler
2653 .emit_msub(Size::S32, dest, src2, src1, dest)?;
2654 if ret != dest {
2655 self.move_location(Size::S32, dest, ret)?;
2656 }
2657 for r in temps {
2658 self.release_gpr(r);
2659 }
2660 Ok(offset)
2661 }
2662
2663 fn emit_binop_srem32(
2664 &mut self,
2665 loc_a: Location,
2666 loc_b: Location,
2667 ret: Location,
2668 integer_division_by_zero: Label,
2669 ) -> Result<usize, CompileError> {
2670 let mut temps = vec![];
2671 let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
2672 let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
2673 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
2674 let dest = if dest == src1 || dest == src2 {
2675 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2676 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2677 })?;
2678 temps.push(tmp);
2679 self.assembler
2680 .emit_mov(Size::S32, dest, Location::GPR(tmp))?;
2681 Location::GPR(tmp)
2682 } else {
2683 dest
2684 };
2685 self.assembler
2686 .emit_cbz_label_far(Size::S32, src2, integer_division_by_zero)?;
2687 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
2688 self.assembler.emit_sdiv(Size::S32, src1, src2, dest)?;
2689 self.assembler
2691 .emit_msub(Size::S32, dest, src2, src1, dest)?;
2692 if ret != dest {
2693 self.move_location(Size::S32, dest, ret)?;
2694 }
2695 for r in temps {
2696 self.release_gpr(r);
2697 }
2698 Ok(offset)
2699 }
2700
2701 fn emit_binop_and32(
2702 &mut self,
2703 loc_a: Location,
2704 loc_b: Location,
2705 ret: Location,
2706 ) -> Result<(), CompileError> {
2707 self.emit_relaxed_binop3(
2708 Assembler::emit_and,
2709 Size::S32,
2710 loc_a,
2711 loc_b,
2712 ret,
2713 ImmType::Logical32,
2714 )
2715 }
2716
2717 fn emit_binop_or32(
2718 &mut self,
2719 loc_a: Location,
2720 loc_b: Location,
2721 ret: Location,
2722 ) -> Result<(), CompileError> {
2723 self.emit_relaxed_binop3(
2724 Assembler::emit_or,
2725 Size::S32,
2726 loc_a,
2727 loc_b,
2728 ret,
2729 ImmType::Logical32,
2730 )
2731 }
2732
2733 fn emit_binop_xor32(
2734 &mut self,
2735 loc_a: Location,
2736 loc_b: Location,
2737 ret: Location,
2738 ) -> Result<(), CompileError> {
2739 self.emit_relaxed_binop3(
2740 Assembler::emit_eor,
2741 Size::S32,
2742 loc_a,
2743 loc_b,
2744 ret,
2745 ImmType::Logical32,
2746 )
2747 }
2748
2749 fn i32_cmp_ge_s(
2750 &mut self,
2751 loc_a: Location,
2752 loc_b: Location,
2753 ret: Location,
2754 ) -> Result<(), CompileError> {
2755 self.emit_cmpop_i32_dynamic_b(Condition::Ge, loc_a, loc_b, ret)
2756 }
2757
2758 fn i32_cmp_gt_s(
2759 &mut self,
2760 loc_a: Location,
2761 loc_b: Location,
2762 ret: Location,
2763 ) -> Result<(), CompileError> {
2764 self.emit_cmpop_i32_dynamic_b(Condition::Gt, loc_a, loc_b, ret)
2765 }
2766
2767 fn i32_cmp_le_s(
2768 &mut self,
2769 loc_a: Location,
2770 loc_b: Location,
2771 ret: Location,
2772 ) -> Result<(), CompileError> {
2773 self.emit_cmpop_i32_dynamic_b(Condition::Le, loc_a, loc_b, ret)
2774 }
2775
2776 fn i32_cmp_lt_s(
2777 &mut self,
2778 loc_a: Location,
2779 loc_b: Location,
2780 ret: Location,
2781 ) -> Result<(), CompileError> {
2782 self.emit_cmpop_i32_dynamic_b(Condition::Lt, loc_a, loc_b, ret)
2783 }
2784
2785 fn i32_cmp_ge_u(
2786 &mut self,
2787 loc_a: Location,
2788 loc_b: Location,
2789 ret: Location,
2790 ) -> Result<(), CompileError> {
2791 self.emit_cmpop_i32_dynamic_b(Condition::Cs, loc_a, loc_b, ret)
2792 }
2793
2794 fn i32_cmp_gt_u(
2795 &mut self,
2796 loc_a: Location,
2797 loc_b: Location,
2798 ret: Location,
2799 ) -> Result<(), CompileError> {
2800 self.emit_cmpop_i32_dynamic_b(Condition::Hi, loc_a, loc_b, ret)
2801 }
2802
2803 fn i32_cmp_le_u(
2804 &mut self,
2805 loc_a: Location,
2806 loc_b: Location,
2807 ret: Location,
2808 ) -> Result<(), CompileError> {
2809 self.emit_cmpop_i32_dynamic_b(Condition::Ls, loc_a, loc_b, ret)
2810 }
2811
2812 fn i32_cmp_lt_u(
2813 &mut self,
2814 loc_a: Location,
2815 loc_b: Location,
2816 ret: Location,
2817 ) -> Result<(), CompileError> {
2818 self.emit_cmpop_i32_dynamic_b(Condition::Cc, loc_a, loc_b, ret)
2819 }
2820
2821 fn i32_cmp_ne(
2822 &mut self,
2823 loc_a: Location,
2824 loc_b: Location,
2825 ret: Location,
2826 ) -> Result<(), CompileError> {
2827 self.emit_cmpop_i32_dynamic_b(Condition::Ne, loc_a, loc_b, ret)
2828 }
2829
2830 fn i32_cmp_eq(
2831 &mut self,
2832 loc_a: Location,
2833 loc_b: Location,
2834 ret: Location,
2835 ) -> Result<(), CompileError> {
2836 self.emit_cmpop_i32_dynamic_b(Condition::Eq, loc_a, loc_b, ret)
2837 }
2838
2839 fn i32_clz(&mut self, src: Location, dst: Location) -> Result<(), CompileError> {
2840 self.emit_relaxed_binop(Assembler::emit_clz, Size::S32, src, dst, true)
2841 }
2842
2843 fn i32_ctz(&mut self, src: Location, dst: Location) -> Result<(), CompileError> {
2844 let mut temps = vec![];
2845 let src = self.location_to_reg(Size::S32, src, &mut temps, ImmType::None, true, None)?;
2846 let dest = self.location_to_reg(Size::S32, dst, &mut temps, ImmType::None, false, None)?;
2847 self.assembler.emit_rbit(Size::S32, src, dest)?;
2848 self.assembler.emit_clz(Size::S32, dest, dest)?;
2849 if dst != dest {
2850 self.move_location(Size::S32, dest, dst)?;
2851 }
2852 for r in temps {
2853 self.release_gpr(r);
2854 }
2855 Ok(())
2856 }
2857
2858 fn i32_popcnt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
2859 if self.has_neon {
2860 let mut temps = vec![];
2861
2862 let src_gpr =
2863 self.location_to_reg(Size::S32, loc, &mut temps, ImmType::None, true, None)?;
2864 let dst_gpr =
2865 self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
2866
2867 let mut neon_temps = vec![];
2868 let neon_temp = self.acquire_temp_simd().ok_or_else(|| {
2869 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2870 })?;
2871 neon_temps.push(neon_temp);
2872
2873 self.assembler
2874 .emit_fmov(Size::S32, src_gpr, Size::S32, Location::SIMD(neon_temp))?;
2875 self.assembler.emit_cnt(neon_temp, neon_temp)?;
2876 self.assembler.emit_addv(neon_temp, neon_temp)?;
2877 self.assembler
2878 .emit_fmov(Size::S32, Location::SIMD(neon_temp), Size::S32, dst_gpr)?;
2879
2880 if ret != dst_gpr {
2881 self.move_location(Size::S32, dst_gpr, ret)?;
2882 }
2883
2884 for r in temps {
2885 self.release_gpr(r);
2886 }
2887
2888 for r in neon_temps {
2889 self.release_simd(r);
2890 }
2891 } else {
2892 let mut temps = vec![];
2893 let src =
2894 self.location_to_reg(Size::S32, loc, &mut temps, ImmType::None, true, None)?;
2895 let dest =
2896 self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
2897 let src = if src == loc {
2898 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2899 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2900 })?;
2901 temps.push(tmp);
2902 self.assembler
2903 .emit_mov(Size::S32, src, Location::GPR(tmp))?;
2904 Location::GPR(tmp)
2905 } else {
2906 src
2907 };
2908 let tmp = {
2909 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2910 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2911 })?;
2912 temps.push(tmp);
2913 Location::GPR(tmp)
2914 };
2915 let label_loop = self.assembler.get_label();
2916 let label_exit = self.assembler.get_label();
2917 self.assembler
2918 .emit_mov(Size::S32, Location::GPR(GPR::XzrSp), dest)?; self.assembler.emit_cbz_label(Size::S32, src, label_exit)?; self.assembler.emit_label(label_loop)?; self.assembler
2922 .emit_add(Size::S32, dest, Location::Imm8(1), dest)?; self.assembler.emit_clz(Size::S32, src, tmp)?; self.assembler.emit_lsl(Size::S32, src, tmp, src)?; self.assembler
2926 .emit_lsl(Size::S32, src, Location::Imm8(1), src)?; self.assembler.emit_cbnz_label(Size::S32, src, label_loop)?; self.assembler.emit_label(label_exit)?;
2929 if ret != dest {
2930 self.move_location(Size::S32, dest, ret)?;
2931 }
2932 for r in temps {
2933 self.release_gpr(r);
2934 }
2935 }
2936 Ok(())
2937 }
2938
2939 fn i32_shl(
2940 &mut self,
2941 loc_a: Location,
2942 loc_b: Location,
2943 ret: Location,
2944 ) -> Result<(), CompileError> {
2945 self.emit_relaxed_binop3(
2946 Assembler::emit_lsl,
2947 Size::S32,
2948 loc_a,
2949 loc_b,
2950 ret,
2951 ImmType::Shift32No0,
2952 )
2953 }
2954
2955 fn i32_shr(
2956 &mut self,
2957 loc_a: Location,
2958 loc_b: Location,
2959 ret: Location,
2960 ) -> Result<(), CompileError> {
2961 self.emit_relaxed_binop3(
2962 Assembler::emit_lsr,
2963 Size::S32,
2964 loc_a,
2965 loc_b,
2966 ret,
2967 ImmType::Shift32No0,
2968 )
2969 }
2970
2971 fn i32_sar(
2972 &mut self,
2973 loc_a: Location,
2974 loc_b: Location,
2975 ret: Location,
2976 ) -> Result<(), CompileError> {
2977 self.emit_relaxed_binop3(
2978 Assembler::emit_asr,
2979 Size::S32,
2980 loc_a,
2981 loc_b,
2982 ret,
2983 ImmType::Shift32No0,
2984 )
2985 }
2986
2987 fn i32_rol(
2988 &mut self,
2989 loc_a: Location,
2990 loc_b: Location,
2991 ret: Location,
2992 ) -> Result<(), CompileError> {
2993 let mut temps = vec![];
2994 let src2 = match loc_b {
2995 Location::Imm8(imm) => Location::Imm8(32 - (imm & 31)),
2996 Location::Imm32(imm) => Location::Imm8(32 - (imm & 31) as u8),
2997 Location::Imm64(imm) => Location::Imm8(32 - (imm & 31) as u8),
2998 _ => {
2999 let tmp1 = self.location_to_reg(
3000 Size::S32,
3001 Location::Imm32(32),
3002 &mut temps,
3003 ImmType::None,
3004 true,
3005 None,
3006 )?;
3007 let tmp2 =
3008 self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
3009 self.assembler.emit_sub(Size::S32, tmp1, tmp2, tmp1)?;
3010 tmp1
3011 }
3012 };
3013 self.emit_relaxed_binop3(
3014 Assembler::emit_ror,
3015 Size::S32,
3016 loc_a,
3017 src2,
3018 ret,
3019 ImmType::Shift32No0,
3020 )?;
3021 for r in temps {
3022 self.release_gpr(r);
3023 }
3024 Ok(())
3025 }
3026
3027 fn i32_ror(
3028 &mut self,
3029 loc_a: Location,
3030 loc_b: Location,
3031 ret: Location,
3032 ) -> Result<(), CompileError> {
3033 self.emit_relaxed_binop3(
3034 Assembler::emit_ror,
3035 Size::S32,
3036 loc_a,
3037 loc_b,
3038 ret,
3039 ImmType::Shift32No0,
3040 )
3041 }
3042
3043 fn i32_load(
3044 &mut self,
3045 addr: Location,
3046 memarg: &MemArg,
3047 ret: Location,
3048 need_check: bool,
3049 imported_memories: bool,
3050 offset: i32,
3051 heap_access_oob: Label,
3052 unaligned_atomic: Label,
3053 ) -> Result<(), CompileError> {
3054 self.memory_op(
3055 addr,
3056 memarg,
3057 false,
3058 4,
3059 need_check,
3060 imported_memories,
3061 offset,
3062 heap_access_oob,
3063 unaligned_atomic,
3064 |this, addr| this.emit_relaxed_ldr32(Size::S32, ret, Location::Memory(addr, 0)),
3065 )
3066 }
3067
3068 fn i32_load_8u(
3069 &mut self,
3070 addr: Location,
3071 memarg: &MemArg,
3072 ret: Location,
3073 need_check: bool,
3074 imported_memories: bool,
3075 offset: i32,
3076 heap_access_oob: Label,
3077 unaligned_atomic: Label,
3078 ) -> Result<(), CompileError> {
3079 self.memory_op(
3080 addr,
3081 memarg,
3082 false,
3083 1,
3084 need_check,
3085 imported_memories,
3086 offset,
3087 heap_access_oob,
3088 unaligned_atomic,
3089 |this, addr| this.emit_relaxed_ldr8(Size::S32, ret, Location::Memory(addr, 0)),
3090 )
3091 }
3092
3093 fn i32_load_8s(
3094 &mut self,
3095 addr: Location,
3096 memarg: &MemArg,
3097 ret: Location,
3098 need_check: bool,
3099 imported_memories: bool,
3100 offset: i32,
3101 heap_access_oob: Label,
3102 unaligned_atomic: Label,
3103 ) -> Result<(), CompileError> {
3104 self.memory_op(
3105 addr,
3106 memarg,
3107 false,
3108 1,
3109 need_check,
3110 imported_memories,
3111 offset,
3112 heap_access_oob,
3113 unaligned_atomic,
3114 |this, addr| this.emit_relaxed_ldr8s(Size::S32, ret, Location::Memory(addr, 0)),
3115 )
3116 }
3117
3118 fn i32_load_16u(
3119 &mut self,
3120 addr: Location,
3121 memarg: &MemArg,
3122 ret: Location,
3123 need_check: bool,
3124 imported_memories: bool,
3125 offset: i32,
3126 heap_access_oob: Label,
3127 unaligned_atomic: Label,
3128 ) -> Result<(), CompileError> {
3129 self.memory_op(
3130 addr,
3131 memarg,
3132 false,
3133 2,
3134 need_check,
3135 imported_memories,
3136 offset,
3137 heap_access_oob,
3138 unaligned_atomic,
3139 |this, addr| this.emit_relaxed_ldr16(Size::S32, ret, Location::Memory(addr, 0)),
3140 )
3141 }
3142
3143 fn i32_load_16s(
3144 &mut self,
3145 addr: Location,
3146 memarg: &MemArg,
3147 ret: Location,
3148 need_check: bool,
3149 imported_memories: bool,
3150 offset: i32,
3151 heap_access_oob: Label,
3152 unaligned_atomic: Label,
3153 ) -> Result<(), CompileError> {
3154 self.memory_op(
3155 addr,
3156 memarg,
3157 false,
3158 2,
3159 need_check,
3160 imported_memories,
3161 offset,
3162 heap_access_oob,
3163 unaligned_atomic,
3164 |this, addr| this.emit_relaxed_ldr16s(Size::S32, ret, Location::Memory(addr, 0)),
3165 )
3166 }
3167
3168 fn i32_atomic_load(
3169 &mut self,
3170 addr: Location,
3171 memarg: &MemArg,
3172 ret: Location,
3173 need_check: bool,
3174 imported_memories: bool,
3175 offset: i32,
3176 heap_access_oob: Label,
3177 unaligned_atomic: Label,
3178 ) -> Result<(), CompileError> {
3179 self.memory_op(
3180 addr,
3181 memarg,
3182 true,
3183 4,
3184 need_check,
3185 imported_memories,
3186 offset,
3187 heap_access_oob,
3188 unaligned_atomic,
3189 |this, addr| this.emit_relaxed_ldr32(Size::S32, ret, Location::Memory(addr, 0)),
3190 )
3191 }
3192
3193 fn i32_atomic_load_8u(
3194 &mut self,
3195 addr: Location,
3196 memarg: &MemArg,
3197 ret: Location,
3198 need_check: bool,
3199 imported_memories: bool,
3200 offset: i32,
3201 heap_access_oob: Label,
3202 unaligned_atomic: Label,
3203 ) -> Result<(), CompileError> {
3204 self.memory_op(
3205 addr,
3206 memarg,
3207 true,
3208 1,
3209 need_check,
3210 imported_memories,
3211 offset,
3212 heap_access_oob,
3213 unaligned_atomic,
3214 |this, addr| this.emit_relaxed_ldr8(Size::S32, ret, Location::Memory(addr, 0)),
3215 )
3216 }
3217
3218 fn i32_atomic_load_16u(
3219 &mut self,
3220 addr: Location,
3221 memarg: &MemArg,
3222 ret: Location,
3223 need_check: bool,
3224 imported_memories: bool,
3225 offset: i32,
3226 heap_access_oob: Label,
3227 unaligned_atomic: Label,
3228 ) -> Result<(), CompileError> {
3229 self.memory_op(
3230 addr,
3231 memarg,
3232 true,
3233 2,
3234 need_check,
3235 imported_memories,
3236 offset,
3237 heap_access_oob,
3238 unaligned_atomic,
3239 |this, addr| this.emit_relaxed_ldr16(Size::S32, ret, Location::Memory(addr, 0)),
3240 )
3241 }
3242
3243 fn i32_save(
3244 &mut self,
3245 target_value: Location,
3246 memarg: &MemArg,
3247 target_addr: Location,
3248 need_check: bool,
3249 imported_memories: bool,
3250 offset: i32,
3251 heap_access_oob: Label,
3252 unaligned_atomic: Label,
3253 ) -> Result<(), CompileError> {
3254 self.memory_op(
3255 target_addr,
3256 memarg,
3257 false,
3258 4,
3259 need_check,
3260 imported_memories,
3261 offset,
3262 heap_access_oob,
3263 unaligned_atomic,
3264 |this, addr| this.emit_relaxed_str32(target_value, Location::Memory(addr, 0)),
3265 )
3266 }
3267
3268 fn i32_save_8(
3269 &mut self,
3270 target_value: Location,
3271 memarg: &MemArg,
3272 target_addr: Location,
3273 need_check: bool,
3274 imported_memories: bool,
3275 offset: i32,
3276 heap_access_oob: Label,
3277 unaligned_atomic: Label,
3278 ) -> Result<(), CompileError> {
3279 self.memory_op(
3280 target_addr,
3281 memarg,
3282 false,
3283 4,
3284 need_check,
3285 imported_memories,
3286 offset,
3287 heap_access_oob,
3288 unaligned_atomic,
3289 |this, addr| this.emit_relaxed_str8(target_value, Location::Memory(addr, 0)),
3290 )
3291 }
3292
3293 fn i32_save_16(
3294 &mut self,
3295 target_value: Location,
3296 memarg: &MemArg,
3297 target_addr: Location,
3298 need_check: bool,
3299 imported_memories: bool,
3300 offset: i32,
3301 heap_access_oob: Label,
3302 unaligned_atomic: Label,
3303 ) -> Result<(), CompileError> {
3304 self.memory_op(
3305 target_addr,
3306 memarg,
3307 false,
3308 4,
3309 need_check,
3310 imported_memories,
3311 offset,
3312 heap_access_oob,
3313 unaligned_atomic,
3314 |this, addr| this.emit_relaxed_str16(target_value, Location::Memory(addr, 0)),
3315 )
3316 }
3317
3318 fn i32_atomic_save(
3319 &mut self,
3320 target_value: Location,
3321 memarg: &MemArg,
3322 target_addr: Location,
3323 need_check: bool,
3324 imported_memories: bool,
3325 offset: i32,
3326 heap_access_oob: Label,
3327 unaligned_atomic: Label,
3328 ) -> Result<(), CompileError> {
3329 self.memory_op(
3330 target_addr,
3331 memarg,
3332 true,
3333 4,
3334 need_check,
3335 imported_memories,
3336 offset,
3337 heap_access_oob,
3338 unaligned_atomic,
3339 |this, addr| this.emit_relaxed_str32(target_value, Location::Memory(addr, 0)),
3340 )?;
3341 self.assembler.emit_dmb()
3342 }
3343
3344 fn i32_atomic_save_8(
3345 &mut self,
3346 target_value: Location,
3347 memarg: &MemArg,
3348 target_addr: Location,
3349 need_check: bool,
3350 imported_memories: bool,
3351 offset: i32,
3352 heap_access_oob: Label,
3353 unaligned_atomic: Label,
3354 ) -> Result<(), CompileError> {
3355 self.memory_op(
3356 target_addr,
3357 memarg,
3358 true,
3359 1,
3360 need_check,
3361 imported_memories,
3362 offset,
3363 heap_access_oob,
3364 unaligned_atomic,
3365 |this, addr| this.emit_relaxed_str8(target_value, Location::Memory(addr, 0)),
3366 )?;
3367 self.assembler.emit_dmb()
3368 }
3369
3370 fn i32_atomic_save_16(
3371 &mut self,
3372 target_value: Location,
3373 memarg: &MemArg,
3374 target_addr: Location,
3375 need_check: bool,
3376 imported_memories: bool,
3377 offset: i32,
3378 heap_access_oob: Label,
3379 unaligned_atomic: Label,
3380 ) -> Result<(), CompileError> {
3381 self.memory_op(
3382 target_addr,
3383 memarg,
3384 true,
3385 2,
3386 need_check,
3387 imported_memories,
3388 offset,
3389 heap_access_oob,
3390 unaligned_atomic,
3391 |this, addr| this.emit_relaxed_str16(target_value, Location::Memory(addr, 0)),
3392 )?;
3393 self.assembler.emit_dmb()
3394 }
3395
3396 fn i32_atomic_add(
3397 &mut self,
3398 loc: Location,
3399 target: Location,
3400 memarg: &MemArg,
3401 ret: Location,
3402 need_check: bool,
3403 imported_memories: bool,
3404 offset: i32,
3405 heap_access_oob: Label,
3406 unaligned_atomic: Label,
3407 ) -> Result<(), CompileError> {
3408 self.memory_op(
3409 target,
3410 memarg,
3411 true,
3412 4,
3413 need_check,
3414 imported_memories,
3415 offset,
3416 heap_access_oob,
3417 unaligned_atomic,
3418 |this, addr| {
3419 let mut temps = vec![];
3420 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3421 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3422 })?;
3423 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3424 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3425 })?;
3426 let dst =
3427 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3428 let reread = this.get_label();
3429
3430 this.emit_label(reread)?;
3431 this.assembler
3432 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
3433 this.emit_binop_add32(dst, loc, Location::GPR(tmp1))?;
3434 this.assembler.emit_stlxr(
3435 Size::S32,
3436 Location::GPR(tmp2),
3437 Location::GPR(tmp1),
3438 Location::GPR(addr),
3439 )?;
3440 this.assembler
3441 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3442 this.assembler.emit_dmb()?;
3443
3444 if dst != ret {
3445 this.move_location(Size::S32, ret, dst)?;
3446 }
3447 for r in temps {
3448 this.release_gpr(r);
3449 }
3450 this.release_gpr(tmp1);
3451 this.release_gpr(tmp2);
3452 Ok(())
3453 },
3454 )
3455 }
3456
3457 fn i32_atomic_add_8u(
3458 &mut self,
3459 loc: Location,
3460 target: Location,
3461 memarg: &MemArg,
3462 ret: Location,
3463 need_check: bool,
3464 imported_memories: bool,
3465 offset: i32,
3466 heap_access_oob: Label,
3467 unaligned_atomic: Label,
3468 ) -> Result<(), CompileError> {
3469 self.memory_op(
3470 target,
3471 memarg,
3472 true,
3473 1,
3474 need_check,
3475 imported_memories,
3476 offset,
3477 heap_access_oob,
3478 unaligned_atomic,
3479 |this, addr| {
3480 let mut temps = vec![];
3481 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3482 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3483 })?;
3484 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3485 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3486 })?;
3487 let dst =
3488 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3489 let reread = this.get_label();
3490
3491 this.emit_label(reread)?;
3492 this.assembler
3493 .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?;
3494 this.emit_binop_add32(dst, loc, Location::GPR(tmp1))?;
3495 this.assembler.emit_stlxrb(
3496 Size::S32,
3497 Location::GPR(tmp2),
3498 Location::GPR(tmp1),
3499 Location::GPR(addr),
3500 )?;
3501 this.assembler
3502 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3503 this.assembler.emit_dmb()?;
3504
3505 if dst != ret {
3506 this.move_location(Size::S32, ret, dst)?;
3507 }
3508 for r in temps {
3509 this.release_gpr(r);
3510 }
3511 this.release_gpr(tmp1);
3512 this.release_gpr(tmp2);
3513 Ok(())
3514 },
3515 )
3516 }
3517
3518 fn i32_atomic_add_16u(
3519 &mut self,
3520 loc: Location,
3521 target: Location,
3522 memarg: &MemArg,
3523 ret: Location,
3524 need_check: bool,
3525 imported_memories: bool,
3526 offset: i32,
3527 heap_access_oob: Label,
3528 unaligned_atomic: Label,
3529 ) -> Result<(), CompileError> {
3530 self.memory_op(
3531 target,
3532 memarg,
3533 true,
3534 2,
3535 need_check,
3536 imported_memories,
3537 offset,
3538 heap_access_oob,
3539 unaligned_atomic,
3540 |this, addr| {
3541 let mut temps = vec![];
3542 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3543 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3544 })?;
3545 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3546 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3547 })?;
3548 let dst =
3549 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3550 let reread = this.get_label();
3551
3552 this.emit_label(reread)?;
3553 this.assembler
3554 .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?;
3555 this.emit_binop_add32(dst, loc, Location::GPR(tmp1))?;
3556 this.assembler.emit_stlxrh(
3557 Size::S32,
3558 Location::GPR(tmp2),
3559 Location::GPR(tmp1),
3560 Location::GPR(addr),
3561 )?;
3562 this.assembler
3563 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3564 this.assembler.emit_dmb()?;
3565
3566 if dst != ret {
3567 this.move_location(Size::S32, ret, dst)?;
3568 }
3569 for r in temps {
3570 this.release_gpr(r);
3571 }
3572 this.release_gpr(tmp1);
3573 this.release_gpr(tmp2);
3574 Ok(())
3575 },
3576 )
3577 }
3578
3579 fn i32_atomic_sub(
3580 &mut self,
3581 loc: Location,
3582 target: Location,
3583 memarg: &MemArg,
3584 ret: Location,
3585 need_check: bool,
3586 imported_memories: bool,
3587 offset: i32,
3588 heap_access_oob: Label,
3589 unaligned_atomic: Label,
3590 ) -> Result<(), CompileError> {
3591 self.memory_op(
3592 target,
3593 memarg,
3594 true,
3595 4,
3596 need_check,
3597 imported_memories,
3598 offset,
3599 heap_access_oob,
3600 unaligned_atomic,
3601 |this, addr| {
3602 let mut temps = vec![];
3603 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3604 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3605 })?;
3606 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3607 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3608 })?;
3609 let dst =
3610 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3611 let reread = this.get_label();
3612
3613 this.emit_label(reread)?;
3614 this.assembler
3615 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
3616 this.emit_binop_sub32(dst, loc, Location::GPR(tmp1))?;
3617 this.assembler.emit_stlxr(
3618 Size::S32,
3619 Location::GPR(tmp2),
3620 Location::GPR(tmp1),
3621 Location::GPR(addr),
3622 )?;
3623 this.assembler
3624 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3625 this.assembler.emit_dmb()?;
3626
3627 if dst != ret {
3628 this.move_location(Size::S32, ret, dst)?;
3629 }
3630 for r in temps {
3631 this.release_gpr(r);
3632 }
3633 this.release_gpr(tmp1);
3634 this.release_gpr(tmp2);
3635 Ok(())
3636 },
3637 )
3638 }
3639
3640 fn i32_atomic_sub_8u(
3641 &mut self,
3642 loc: Location,
3643 target: Location,
3644 memarg: &MemArg,
3645 ret: Location,
3646 need_check: bool,
3647 imported_memories: bool,
3648 offset: i32,
3649 heap_access_oob: Label,
3650 unaligned_atomic: Label,
3651 ) -> Result<(), CompileError> {
3652 self.memory_op(
3653 target,
3654 memarg,
3655 true,
3656 1,
3657 need_check,
3658 imported_memories,
3659 offset,
3660 heap_access_oob,
3661 unaligned_atomic,
3662 |this, addr| {
3663 let mut temps = vec![];
3664 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3665 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3666 })?;
3667 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3668 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3669 })?;
3670 let dst =
3671 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3672 let reread = this.get_label();
3673
3674 this.emit_label(reread)?;
3675 this.assembler
3676 .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?;
3677 this.emit_binop_sub32(dst, loc, Location::GPR(tmp1))?;
3678 this.assembler.emit_stlxrb(
3679 Size::S32,
3680 Location::GPR(tmp2),
3681 Location::GPR(tmp1),
3682 Location::GPR(addr),
3683 )?;
3684 this.assembler
3685 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3686 this.assembler.emit_dmb()?;
3687
3688 if dst != ret {
3689 this.move_location(Size::S32, ret, dst)?;
3690 }
3691 for r in temps {
3692 this.release_gpr(r);
3693 }
3694 this.release_gpr(tmp1);
3695 this.release_gpr(tmp2);
3696 Ok(())
3697 },
3698 )
3699 }
3700
3701 fn i32_atomic_sub_16u(
3702 &mut self,
3703 loc: Location,
3704 target: Location,
3705 memarg: &MemArg,
3706 ret: Location,
3707 need_check: bool,
3708 imported_memories: bool,
3709 offset: i32,
3710 heap_access_oob: Label,
3711 unaligned_atomic: Label,
3712 ) -> Result<(), CompileError> {
3713 self.memory_op(
3714 target,
3715 memarg,
3716 true,
3717 2,
3718 need_check,
3719 imported_memories,
3720 offset,
3721 heap_access_oob,
3722 unaligned_atomic,
3723 |this, addr| {
3724 let mut temps = vec![];
3725 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3726 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3727 })?;
3728 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3729 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3730 })?;
3731 let dst =
3732 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3733 let reread = this.get_label();
3734
3735 this.emit_label(reread)?;
3736 this.assembler
3737 .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?;
3738 this.emit_binop_sub32(dst, loc, Location::GPR(tmp1))?;
3739 this.assembler.emit_stlxrh(
3740 Size::S32,
3741 Location::GPR(tmp2),
3742 Location::GPR(tmp1),
3743 Location::GPR(addr),
3744 )?;
3745 this.assembler
3746 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3747 this.assembler.emit_dmb()?;
3748
3749 if dst != ret {
3750 this.move_location(Size::S32, ret, dst)?;
3751 }
3752 for r in temps {
3753 this.release_gpr(r);
3754 }
3755 this.release_gpr(tmp1);
3756 this.release_gpr(tmp2);
3757 Ok(())
3758 },
3759 )
3760 }
3761
3762 fn i32_atomic_and(
3763 &mut self,
3764 loc: Location,
3765 target: Location,
3766 memarg: &MemArg,
3767 ret: Location,
3768 need_check: bool,
3769 imported_memories: bool,
3770 offset: i32,
3771 heap_access_oob: Label,
3772 unaligned_atomic: Label,
3773 ) -> Result<(), CompileError> {
3774 self.memory_op(
3775 target,
3776 memarg,
3777 true,
3778 4,
3779 need_check,
3780 imported_memories,
3781 offset,
3782 heap_access_oob,
3783 unaligned_atomic,
3784 |this, addr| {
3785 let mut temps = vec![];
3786 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3787 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3788 })?;
3789 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3790 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3791 })?;
3792 let dst =
3793 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3794 let reread = this.get_label();
3795
3796 this.emit_label(reread)?;
3797 this.assembler
3798 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
3799 this.emit_binop_and32(dst, loc, Location::GPR(tmp1))?;
3800 this.assembler.emit_stlxr(
3801 Size::S32,
3802 Location::GPR(tmp2),
3803 Location::GPR(tmp1),
3804 Location::GPR(addr),
3805 )?;
3806 this.assembler
3807 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3808 this.assembler.emit_dmb()?;
3809
3810 if dst != ret {
3811 this.move_location(Size::S32, ret, dst)?;
3812 }
3813 for r in temps {
3814 this.release_gpr(r);
3815 }
3816 this.release_gpr(tmp1);
3817 this.release_gpr(tmp2);
3818 Ok(())
3819 },
3820 )
3821 }
3822
3823 fn i32_atomic_and_8u(
3824 &mut self,
3825 loc: Location,
3826 target: Location,
3827 memarg: &MemArg,
3828 ret: Location,
3829 need_check: bool,
3830 imported_memories: bool,
3831 offset: i32,
3832 heap_access_oob: Label,
3833 unaligned_atomic: Label,
3834 ) -> Result<(), CompileError> {
3835 self.memory_op(
3836 target,
3837 memarg,
3838 true,
3839 1,
3840 need_check,
3841 imported_memories,
3842 offset,
3843 heap_access_oob,
3844 unaligned_atomic,
3845 |this, addr| {
3846 let mut temps = vec![];
3847 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3848 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3849 })?;
3850 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3851 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3852 })?;
3853 let dst =
3854 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3855 let reread = this.get_label();
3856
3857 this.emit_label(reread)?;
3858 this.assembler
3859 .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?;
3860 this.emit_binop_and32(dst, loc, Location::GPR(tmp1))?;
3861 this.assembler.emit_stlxrb(
3862 Size::S32,
3863 Location::GPR(tmp2),
3864 Location::GPR(tmp1),
3865 Location::GPR(addr),
3866 )?;
3867 this.assembler
3868 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3869 this.assembler.emit_dmb()?;
3870
3871 if dst != ret {
3872 this.move_location(Size::S32, ret, dst)?;
3873 }
3874 for r in temps {
3875 this.release_gpr(r);
3876 }
3877 this.release_gpr(tmp1);
3878 this.release_gpr(tmp2);
3879 Ok(())
3880 },
3881 )
3882 }
3883
3884 fn i32_atomic_and_16u(
3885 &mut self,
3886 loc: Location,
3887 target: Location,
3888 memarg: &MemArg,
3889 ret: Location,
3890 need_check: bool,
3891 imported_memories: bool,
3892 offset: i32,
3893 heap_access_oob: Label,
3894 unaligned_atomic: Label,
3895 ) -> Result<(), CompileError> {
3896 self.memory_op(
3897 target,
3898 memarg,
3899 true,
3900 2,
3901 need_check,
3902 imported_memories,
3903 offset,
3904 heap_access_oob,
3905 unaligned_atomic,
3906 |this, addr| {
3907 let mut temps = vec![];
3908 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3909 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3910 })?;
3911 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3912 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3913 })?;
3914 let dst =
3915 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3916 let reread = this.get_label();
3917
3918 this.emit_label(reread)?;
3919 this.assembler
3920 .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?;
3921 this.emit_binop_and32(dst, loc, Location::GPR(tmp1))?;
3922 this.assembler.emit_stlxrh(
3923 Size::S32,
3924 Location::GPR(tmp2),
3925 Location::GPR(tmp1),
3926 Location::GPR(addr),
3927 )?;
3928 this.assembler
3929 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3930 this.assembler.emit_dmb()?;
3931
3932 if dst != ret {
3933 this.move_location(Size::S32, ret, dst)?;
3934 }
3935 for r in temps {
3936 this.release_gpr(r);
3937 }
3938 this.release_gpr(tmp1);
3939 this.release_gpr(tmp2);
3940 Ok(())
3941 },
3942 )
3943 }
3944
3945 fn i32_atomic_or(
3946 &mut self,
3947 loc: Location,
3948 target: Location,
3949 memarg: &MemArg,
3950 ret: Location,
3951 need_check: bool,
3952 imported_memories: bool,
3953 offset: i32,
3954 heap_access_oob: Label,
3955 unaligned_atomic: Label,
3956 ) -> Result<(), CompileError> {
3957 self.memory_op(
3958 target,
3959 memarg,
3960 true,
3961 4,
3962 need_check,
3963 imported_memories,
3964 offset,
3965 heap_access_oob,
3966 unaligned_atomic,
3967 |this, addr| {
3968 let mut temps = vec![];
3969 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3970 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3971 })?;
3972 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3973 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3974 })?;
3975 let dst =
3976 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3977 let reread = this.get_label();
3978
3979 this.emit_label(reread)?;
3980 this.assembler
3981 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
3982 this.emit_binop_or32(dst, loc, Location::GPR(tmp1))?;
3983 this.assembler.emit_stlxr(
3984 Size::S32,
3985 Location::GPR(tmp2),
3986 Location::GPR(tmp1),
3987 Location::GPR(addr),
3988 )?;
3989 this.assembler
3990 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3991 this.assembler.emit_dmb()?;
3992
3993 if dst != ret {
3994 this.move_location(Size::S32, ret, dst)?;
3995 }
3996 for r in temps {
3997 this.release_gpr(r);
3998 }
3999 this.release_gpr(tmp1);
4000 this.release_gpr(tmp2);
4001 Ok(())
4002 },
4003 )
4004 }
4005
4006 fn i32_atomic_or_8u(
4007 &mut self,
4008 loc: Location,
4009 target: Location,
4010 memarg: &MemArg,
4011 ret: Location,
4012 need_check: bool,
4013 imported_memories: bool,
4014 offset: i32,
4015 heap_access_oob: Label,
4016 unaligned_atomic: Label,
4017 ) -> Result<(), CompileError> {
4018 self.memory_op(
4019 target,
4020 memarg,
4021 true,
4022 1,
4023 need_check,
4024 imported_memories,
4025 offset,
4026 heap_access_oob,
4027 unaligned_atomic,
4028 |this, addr| {
4029 let mut temps = vec![];
4030 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
4031 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4032 })?;
4033 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
4034 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4035 })?;
4036 let dst =
4037 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4038 let reread = this.get_label();
4039
4040 this.emit_label(reread)?;
4041 this.assembler
4042 .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?;
4043 this.emit_binop_or32(dst, loc, Location::GPR(tmp1))?;
4044 this.assembler.emit_stlxrb(
4045 Size::S32,
4046 Location::GPR(tmp2),
4047 Location::GPR(tmp1),
4048 Location::GPR(addr),
4049 )?;
4050 this.assembler
4051 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
4052 this.assembler.emit_dmb()?;
4053
4054 if dst != ret {
4055 this.move_location(Size::S32, ret, dst)?;
4056 }
4057 for r in temps {
4058 this.release_gpr(r);
4059 }
4060 this.release_gpr(tmp1);
4061 this.release_gpr(tmp2);
4062 Ok(())
4063 },
4064 )
4065 }
4066
4067 fn i32_atomic_or_16u(
4068 &mut self,
4069 loc: Location,
4070 target: Location,
4071 memarg: &MemArg,
4072 ret: Location,
4073 need_check: bool,
4074 imported_memories: bool,
4075 offset: i32,
4076 heap_access_oob: Label,
4077 unaligned_atomic: Label,
4078 ) -> Result<(), CompileError> {
4079 self.memory_op(
4080 target,
4081 memarg,
4082 true,
4083 2,
4084 need_check,
4085 imported_memories,
4086 offset,
4087 heap_access_oob,
4088 unaligned_atomic,
4089 |this, addr| {
4090 let mut temps = vec![];
4091 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
4092 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4093 })?;
4094 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
4095 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4096 })?;
4097 let dst =
4098 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4099 let reread = this.get_label();
4100
4101 this.emit_label(reread)?;
4102 this.assembler
4103 .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?;
4104 this.emit_binop_or32(dst, loc, Location::GPR(tmp1))?;
4105 this.assembler.emit_stlxrh(
4106 Size::S32,
4107 Location::GPR(tmp2),
4108 Location::GPR(tmp1),
4109 Location::GPR(addr),
4110 )?;
4111 this.assembler
4112 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
4113 this.assembler.emit_dmb()?;
4114
4115 if dst != ret {
4116 this.move_location(Size::S32, ret, dst)?;
4117 }
4118 for r in temps {
4119 this.release_gpr(r);
4120 }
4121 this.release_gpr(tmp1);
4122 this.release_gpr(tmp2);
4123 Ok(())
4124 },
4125 )
4126 }
4127
4128 fn i32_atomic_xor(
4129 &mut self,
4130 loc: Location,
4131 target: Location,
4132 memarg: &MemArg,
4133 ret: Location,
4134 need_check: bool,
4135 imported_memories: bool,
4136 offset: i32,
4137 heap_access_oob: Label,
4138 unaligned_atomic: Label,
4139 ) -> Result<(), CompileError> {
4140 self.memory_op(
4141 target,
4142 memarg,
4143 true,
4144 4,
4145 need_check,
4146 imported_memories,
4147 offset,
4148 heap_access_oob,
4149 unaligned_atomic,
4150 |this, addr| {
4151 let mut temps = vec![];
4152 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
4153 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4154 })?;
4155 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
4156 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4157 })?;
4158 let dst =
4159 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4160 let reread = this.get_label();
4161
4162 this.emit_label(reread)?;
4163 this.assembler
4164 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
4165 this.emit_binop_xor32(dst, loc, Location::GPR(tmp1))?;
4166 this.assembler.emit_stlxr(
4167 Size::S32,
4168 Location::GPR(tmp2),
4169 Location::GPR(tmp1),
4170 Location::GPR(addr),
4171 )?;
4172 this.assembler
4173 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
4174 this.assembler.emit_dmb()?;
4175
4176 if dst != ret {
4177 this.move_location(Size::S32, ret, dst)?;
4178 }
4179 for r in temps {
4180 this.release_gpr(r);
4181 }
4182 this.release_gpr(tmp1);
4183 this.release_gpr(tmp2);
4184 Ok(())
4185 },
4186 )
4187 }
4188
4189 fn i32_atomic_xor_8u(
4190 &mut self,
4191 loc: Location,
4192 target: Location,
4193 memarg: &MemArg,
4194 ret: Location,
4195 need_check: bool,
4196 imported_memories: bool,
4197 offset: i32,
4198 heap_access_oob: Label,
4199 unaligned_atomic: Label,
4200 ) -> Result<(), CompileError> {
4201 self.memory_op(
4202 target,
4203 memarg,
4204 true,
4205 1,
4206 need_check,
4207 imported_memories,
4208 offset,
4209 heap_access_oob,
4210 unaligned_atomic,
4211 |this, addr| {
4212 let mut temps = vec![];
4213 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
4214 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4215 })?;
4216 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
4217 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4218 })?;
4219 let dst =
4220 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4221 let reread = this.get_label();
4222
4223 this.emit_label(reread)?;
4224 this.assembler
4225 .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?;
4226 this.emit_binop_xor32(dst, loc, Location::GPR(tmp1))?;
4227 this.assembler.emit_stlxrb(
4228 Size::S32,
4229 Location::GPR(tmp2),
4230 Location::GPR(tmp1),
4231 Location::GPR(addr),
4232 )?;
4233 this.assembler
4234 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
4235 this.assembler.emit_dmb()?;
4236
4237 if dst != ret {
4238 this.move_location(Size::S32, ret, dst)?;
4239 }
4240 for r in temps {
4241 this.release_gpr(r);
4242 }
4243 this.release_gpr(tmp1);
4244 this.release_gpr(tmp2);
4245 Ok(())
4246 },
4247 )
4248 }
4249
4250 fn i32_atomic_xor_16u(
4251 &mut self,
4252 loc: Location,
4253 target: Location,
4254 memarg: &MemArg,
4255 ret: Location,
4256 need_check: bool,
4257 imported_memories: bool,
4258 offset: i32,
4259 heap_access_oob: Label,
4260 unaligned_atomic: Label,
4261 ) -> Result<(), CompileError> {
4262 self.memory_op(
4263 target,
4264 memarg,
4265 true,
4266 2,
4267 need_check,
4268 imported_memories,
4269 offset,
4270 heap_access_oob,
4271 unaligned_atomic,
4272 |this, addr| {
4273 let mut temps = vec![];
4274 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
4275 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4276 })?;
4277 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
4278 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4279 })?;
4280 let dst =
4281 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4282 let reread = this.get_label();
4283
4284 this.emit_label(reread)?;
4285 this.assembler
4286 .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?;
4287 this.emit_binop_xor32(dst, loc, Location::GPR(tmp1))?;
4288 this.assembler.emit_stlxrh(
4289 Size::S32,
4290 Location::GPR(tmp2),
4291 Location::GPR(tmp1),
4292 Location::GPR(addr),
4293 )?;
4294 this.assembler
4295 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
4296 this.assembler.emit_dmb()?;
4297
4298 if dst != ret {
4299 this.move_location(Size::S32, ret, dst)?;
4300 }
4301 for r in temps {
4302 this.release_gpr(r);
4303 }
4304 this.release_gpr(tmp1);
4305 this.release_gpr(tmp2);
4306 Ok(())
4307 },
4308 )
4309 }
4310
4311 fn i32_atomic_xchg(
4312 &mut self,
4313 loc: Location,
4314 target: Location,
4315 memarg: &MemArg,
4316 ret: Location,
4317 need_check: bool,
4318 imported_memories: bool,
4319 offset: i32,
4320 heap_access_oob: Label,
4321 unaligned_atomic: Label,
4322 ) -> Result<(), CompileError> {
4323 self.memory_op(
4324 target,
4325 memarg,
4326 true,
4327 4,
4328 need_check,
4329 imported_memories,
4330 offset,
4331 heap_access_oob,
4332 unaligned_atomic,
4333 |this, addr| {
4334 let mut temps = vec![];
4335 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
4336 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4337 })?;
4338 let dst =
4339 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4340 let org =
4341 this.location_to_reg(Size::S32, loc, &mut temps, ImmType::None, false, None)?;
4342 let reread = this.get_label();
4343
4344 this.emit_label(reread)?;
4345 this.assembler
4346 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
4347 this.assembler.emit_stlxr(
4348 Size::S32,
4349 Location::GPR(tmp),
4350 org,
4351 Location::GPR(addr),
4352 )?;
4353 this.assembler
4354 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
4355 this.assembler.emit_dmb()?;
4356
4357 if dst != ret {
4358 this.move_location(Size::S32, ret, dst)?;
4359 }
4360 for r in temps {
4361 this.release_gpr(r);
4362 }
4363 this.release_gpr(tmp);
4364 Ok(())
4365 },
4366 )
4367 }
4368
4369 fn i32_atomic_xchg_8u(
4370 &mut self,
4371 loc: Location,
4372 target: Location,
4373 memarg: &MemArg,
4374 ret: Location,
4375 need_check: bool,
4376 imported_memories: bool,
4377 offset: i32,
4378 heap_access_oob: Label,
4379 unaligned_atomic: Label,
4380 ) -> Result<(), CompileError> {
4381 self.memory_op(
4382 target,
4383 memarg,
4384 true,
4385 1,
4386 need_check,
4387 imported_memories,
4388 offset,
4389 heap_access_oob,
4390 unaligned_atomic,
4391 |this, addr| {
4392 let mut temps = vec![];
4393 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
4394 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4395 })?;
4396 let dst =
4397 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4398 let org =
4399 this.location_to_reg(Size::S32, loc, &mut temps, ImmType::None, false, None)?;
4400 let reread = this.get_label();
4401
4402 this.emit_label(reread)?;
4403 this.assembler
4404 .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?;
4405 this.assembler.emit_stlxrb(
4406 Size::S32,
4407 Location::GPR(tmp),
4408 org,
4409 Location::GPR(addr),
4410 )?;
4411 this.assembler
4412 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
4413 this.assembler.emit_dmb()?;
4414
4415 if dst != ret {
4416 this.move_location(Size::S32, ret, dst)?;
4417 }
4418 for r in temps {
4419 this.release_gpr(r);
4420 }
4421 this.release_gpr(tmp);
4422 Ok(())
4423 },
4424 )
4425 }
4426
4427 fn i32_atomic_xchg_16u(
4428 &mut self,
4429 loc: Location,
4430 target: Location,
4431 memarg: &MemArg,
4432 ret: Location,
4433 need_check: bool,
4434 imported_memories: bool,
4435 offset: i32,
4436 heap_access_oob: Label,
4437 unaligned_atomic: Label,
4438 ) -> Result<(), CompileError> {
4439 self.memory_op(
4440 target,
4441 memarg,
4442 true,
4443 2,
4444 need_check,
4445 imported_memories,
4446 offset,
4447 heap_access_oob,
4448 unaligned_atomic,
4449 |this, addr| {
4450 let mut temps = vec![];
4451 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
4452 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4453 })?;
4454 let dst =
4455 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4456 let org =
4457 this.location_to_reg(Size::S32, loc, &mut temps, ImmType::None, false, None)?;
4458 let reread = this.get_label();
4459
4460 this.emit_label(reread)?;
4461 this.assembler
4462 .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?;
4463 this.assembler.emit_stlxrh(
4464 Size::S32,
4465 Location::GPR(tmp),
4466 org,
4467 Location::GPR(addr),
4468 )?;
4469 this.assembler
4470 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
4471 this.assembler.emit_dmb()?;
4472
4473 if dst != ret {
4474 this.move_location(Size::S32, ret, dst)?;
4475 }
4476 for r in temps {
4477 this.release_gpr(r);
4478 }
4479 this.release_gpr(tmp);
4480 Ok(())
4481 },
4482 )
4483 }
4484
4485 fn i32_atomic_cmpxchg(
4486 &mut self,
4487 new: Location,
4488 cmp: Location,
4489 target: Location,
4490 memarg: &MemArg,
4491 ret: Location,
4492 need_check: bool,
4493 imported_memories: bool,
4494 offset: i32,
4495 heap_access_oob: Label,
4496 unaligned_atomic: Label,
4497 ) -> Result<(), CompileError> {
4498 self.memory_op(
4499 target,
4500 memarg,
4501 true,
4502 4,
4503 need_check,
4504 imported_memories,
4505 offset,
4506 heap_access_oob,
4507 unaligned_atomic,
4508 |this, addr| {
4509 let mut temps = vec![];
4510 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
4511 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4512 })?;
4513 let dst =
4514 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4515 let org =
4516 this.location_to_reg(Size::S32, new, &mut temps, ImmType::None, false, None)?;
4517 let reread = this.get_label();
4518 let nosame = this.get_label();
4519
4520 this.emit_label(reread)?;
4521 this.assembler
4522 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
4523 this.emit_relaxed_cmp(Size::S32, dst, cmp)?;
4524 this.assembler.emit_bcond_label(Condition::Ne, nosame)?;
4525 this.assembler.emit_stlxr(
4526 Size::S32,
4527 Location::GPR(tmp),
4528 org,
4529 Location::GPR(addr),
4530 )?;
4531 this.assembler
4532 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
4533 this.assembler.emit_dmb()?;
4534
4535 this.emit_label(nosame)?;
4536 if dst != ret {
4537 this.move_location(Size::S32, ret, dst)?;
4538 }
4539 for r in temps {
4540 this.release_gpr(r);
4541 }
4542 this.release_gpr(tmp);
4543 Ok(())
4544 },
4545 )
4546 }
4547
4548 fn i32_atomic_cmpxchg_8u(
4549 &mut self,
4550 new: Location,
4551 cmp: Location,
4552 target: Location,
4553 memarg: &MemArg,
4554 ret: Location,
4555 need_check: bool,
4556 imported_memories: bool,
4557 offset: i32,
4558 heap_access_oob: Label,
4559 unaligned_atomic: Label,
4560 ) -> Result<(), CompileError> {
4561 self.memory_op(
4562 target,
4563 memarg,
4564 true,
4565 1,
4566 need_check,
4567 imported_memories,
4568 offset,
4569 heap_access_oob,
4570 unaligned_atomic,
4571 |this, addr| {
4572 let mut temps = vec![];
4573 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
4574 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4575 })?;
4576 let dst =
4577 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4578 let org =
4579 this.location_to_reg(Size::S32, new, &mut temps, ImmType::None, false, None)?;
4580 let reread = this.get_label();
4581 let nosame = this.get_label();
4582
4583 this.emit_label(reread)?;
4584 this.assembler
4585 .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?;
4586 this.emit_relaxed_cmp(Size::S32, dst, cmp)?;
4587 this.assembler.emit_bcond_label(Condition::Ne, nosame)?;
4588 this.assembler.emit_stlxrb(
4589 Size::S32,
4590 Location::GPR(tmp),
4591 org,
4592 Location::GPR(addr),
4593 )?;
4594 this.assembler
4595 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
4596 this.assembler.emit_dmb()?;
4597
4598 this.emit_label(nosame)?;
4599 if dst != ret {
4600 this.move_location(Size::S32, ret, dst)?;
4601 }
4602 for r in temps {
4603 this.release_gpr(r);
4604 }
4605 this.release_gpr(tmp);
4606 Ok(())
4607 },
4608 )
4609 }
4610
4611 fn i32_atomic_cmpxchg_16u(
4612 &mut self,
4613 new: Location,
4614 cmp: Location,
4615 target: Location,
4616 memarg: &MemArg,
4617 ret: Location,
4618 need_check: bool,
4619 imported_memories: bool,
4620 offset: i32,
4621 heap_access_oob: Label,
4622 unaligned_atomic: Label,
4623 ) -> Result<(), CompileError> {
4624 self.memory_op(
4625 target,
4626 memarg,
4627 true,
4628 2,
4629 need_check,
4630 imported_memories,
4631 offset,
4632 heap_access_oob,
4633 unaligned_atomic,
4634 |this, addr| {
4635 let mut temps = vec![];
4636 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
4637 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4638 })?;
4639 let dst =
4640 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4641 let org =
4642 this.location_to_reg(Size::S32, new, &mut temps, ImmType::None, false, None)?;
4643 let reread = this.get_label();
4644 let nosame = this.get_label();
4645
4646 this.emit_label(reread)?;
4647 this.assembler
4648 .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?;
4649 this.emit_relaxed_cmp(Size::S32, dst, cmp)?;
4650 this.assembler.emit_bcond_label(Condition::Ne, nosame)?;
4651 this.assembler.emit_stlxrh(
4652 Size::S32,
4653 Location::GPR(tmp),
4654 org,
4655 Location::GPR(addr),
4656 )?;
4657 this.assembler
4658 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
4659 this.assembler.emit_dmb()?;
4660
4661 this.emit_label(nosame)?;
4662 if dst != ret {
4663 this.move_location(Size::S32, ret, dst)?;
4664 }
4665 for r in temps {
4666 this.release_gpr(r);
4667 }
4668 this.release_gpr(tmp);
4669 Ok(())
4670 },
4671 )
4672 }
4673
4674 fn emit_call_with_reloc(
4675 &mut self,
4676 reloc_target: RelocationTarget,
4677 ) -> Result<Vec<Relocation>, CompileError> {
4678 let mut relocations = vec![];
4679 let next = self.get_label();
4680 let reloc_at = self.assembler.get_offset().0;
4681 self.emit_label(next)?; self.assembler.emit_call_label(next)?;
4683 relocations.push(Relocation {
4684 kind: RelocationKind::Arm64Call,
4685 reloc_target,
4686 offset: reloc_at as u32,
4687 addend: 0,
4688 });
4689 Ok(relocations)
4690 }
4691
4692 fn emit_binop_add64(
4693 &mut self,
4694 loc_a: Location,
4695 loc_b: Location,
4696 ret: Location,
4697 ) -> Result<(), CompileError> {
4698 self.emit_relaxed_binop3(
4699 Assembler::emit_add,
4700 Size::S64,
4701 loc_a,
4702 loc_b,
4703 ret,
4704 ImmType::Bits12,
4705 )
4706 }
4707
4708 fn emit_binop_sub64(
4709 &mut self,
4710 loc_a: Location,
4711 loc_b: Location,
4712 ret: Location,
4713 ) -> Result<(), CompileError> {
4714 self.emit_relaxed_binop3(
4715 Assembler::emit_sub,
4716 Size::S64,
4717 loc_a,
4718 loc_b,
4719 ret,
4720 ImmType::Bits12,
4721 )
4722 }
4723
4724 fn emit_binop_mul64(
4725 &mut self,
4726 loc_a: Location,
4727 loc_b: Location,
4728 ret: Location,
4729 ) -> Result<(), CompileError> {
4730 self.emit_relaxed_binop3(
4731 Assembler::emit_mul,
4732 Size::S64,
4733 loc_a,
4734 loc_b,
4735 ret,
4736 ImmType::None,
4737 )
4738 }
4739
4740 fn emit_binop_udiv64(
4741 &mut self,
4742 loc_a: Location,
4743 loc_b: Location,
4744 ret: Location,
4745 integer_division_by_zero: Label,
4746 ) -> Result<usize, CompileError> {
4747 let mut temps = vec![];
4748 let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4749 let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4750 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4751
4752 self.assembler
4753 .emit_cbz_label(Size::S64, src2, integer_division_by_zero)?;
4754 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
4755 self.assembler.emit_udiv(Size::S64, src1, src2, dest)?;
4756 if ret != dest {
4757 self.move_location(Size::S64, dest, ret)?;
4758 }
4759 for r in temps {
4760 self.release_gpr(r);
4761 }
4762 Ok(offset)
4763 }
4764
4765 fn emit_binop_sdiv64(
4766 &mut self,
4767 loc_a: Location,
4768 loc_b: Location,
4769 ret: Location,
4770 integer_division_by_zero: Label,
4771 integer_overflow: Label,
4772 ) -> Result<usize, CompileError> {
4773 let mut temps = vec![];
4774 let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4775 let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4776 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4777
4778 self.assembler
4779 .emit_cbz_label_far(Size::S64, src2, integer_division_by_zero)?;
4780 let label_nooverflow = self.assembler.get_label();
4781 let tmp = self.location_to_reg(
4782 Size::S64,
4783 Location::Imm64(0x8000000000000000),
4784 &mut temps,
4785 ImmType::None,
4786 true,
4787 None,
4788 )?;
4789 self.assembler.emit_cmp(Size::S64, tmp, src1)?;
4790 self.assembler
4791 .emit_bcond_label(Condition::Ne, label_nooverflow)?;
4792 self.assembler.emit_movn(Size::S64, tmp, 0)?;
4793 self.assembler.emit_cmp(Size::S64, tmp, src2)?;
4794 self.assembler
4795 .emit_bcond_label_far(Condition::Eq, integer_overflow)?;
4796 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
4797 self.assembler.emit_label(label_nooverflow)?;
4798 self.assembler.emit_sdiv(Size::S64, src1, src2, dest)?;
4799 if ret != dest {
4800 self.move_location(Size::S64, dest, ret)?;
4801 }
4802 for r in temps {
4803 self.release_gpr(r);
4804 }
4805 Ok(offset)
4806 }
4807
4808 fn emit_binop_urem64(
4809 &mut self,
4810 loc_a: Location,
4811 loc_b: Location,
4812 ret: Location,
4813 integer_division_by_zero: Label,
4814 ) -> Result<usize, CompileError> {
4815 let mut temps = vec![];
4816 let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4817 let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4818 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4819 let dest = if dest == src1 || dest == src2 {
4820 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
4821 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4822 })?;
4823 temps.push(tmp);
4824 self.assembler
4825 .emit_mov(Size::S32, dest, Location::GPR(tmp))?;
4826 Location::GPR(tmp)
4827 } else {
4828 dest
4829 };
4830 self.assembler
4831 .emit_cbz_label_far(Size::S64, src2, integer_division_by_zero)?;
4832 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
4833 self.assembler.emit_udiv(Size::S64, src1, src2, dest)?;
4834 self.assembler
4836 .emit_msub(Size::S64, dest, src2, src1, dest)?;
4837 if ret != dest {
4838 self.move_location(Size::S64, dest, ret)?;
4839 }
4840 for r in temps {
4841 self.release_gpr(r);
4842 }
4843 Ok(offset)
4844 }
4845
4846 fn emit_binop_srem64(
4847 &mut self,
4848 loc_a: Location,
4849 loc_b: Location,
4850 ret: Location,
4851 integer_division_by_zero: Label,
4852 ) -> Result<usize, CompileError> {
4853 let mut temps = vec![];
4854 let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4855 let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4856 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4857 let dest = if dest == src1 || dest == src2 {
4858 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
4859 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4860 })?;
4861 temps.push(tmp);
4862 self.assembler
4863 .emit_mov(Size::S64, dest, Location::GPR(tmp))?;
4864 Location::GPR(tmp)
4865 } else {
4866 dest
4867 };
4868 self.assembler
4869 .emit_cbz_label_far(Size::S64, src2, integer_division_by_zero)?;
4870 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
4871 self.assembler.emit_sdiv(Size::S64, src1, src2, dest)?;
4872 self.assembler
4874 .emit_msub(Size::S64, dest, src2, src1, dest)?;
4875 if ret != dest {
4876 self.move_location(Size::S64, dest, ret)?;
4877 }
4878 for r in temps {
4879 self.release_gpr(r);
4880 }
4881 Ok(offset)
4882 }
4883
4884 fn emit_binop_and64(
4885 &mut self,
4886 loc_a: Location,
4887 loc_b: Location,
4888 ret: Location,
4889 ) -> Result<(), CompileError> {
4890 self.emit_relaxed_binop3(
4891 Assembler::emit_and,
4892 Size::S64,
4893 loc_a,
4894 loc_b,
4895 ret,
4896 ImmType::Logical64,
4897 )
4898 }
4899
4900 fn emit_binop_or64(
4901 &mut self,
4902 loc_a: Location,
4903 loc_b: Location,
4904 ret: Location,
4905 ) -> Result<(), CompileError> {
4906 self.emit_relaxed_binop3(
4907 Assembler::emit_or,
4908 Size::S64,
4909 loc_a,
4910 loc_b,
4911 ret,
4912 ImmType::Logical64,
4913 )
4914 }
4915
4916 fn emit_binop_xor64(
4917 &mut self,
4918 loc_a: Location,
4919 loc_b: Location,
4920 ret: Location,
4921 ) -> Result<(), CompileError> {
4922 self.emit_relaxed_binop3(
4923 Assembler::emit_eor,
4924 Size::S64,
4925 loc_a,
4926 loc_b,
4927 ret,
4928 ImmType::Logical64,
4929 )
4930 }
4931
4932 fn i64_cmp_ge_s(
4933 &mut self,
4934 loc_a: Location,
4935 loc_b: Location,
4936 ret: Location,
4937 ) -> Result<(), CompileError> {
4938 self.emit_cmpop_i64_dynamic_b(Condition::Ge, loc_a, loc_b, ret)
4939 }
4940
4941 fn i64_cmp_gt_s(
4942 &mut self,
4943 loc_a: Location,
4944 loc_b: Location,
4945 ret: Location,
4946 ) -> Result<(), CompileError> {
4947 self.emit_cmpop_i64_dynamic_b(Condition::Gt, loc_a, loc_b, ret)
4948 }
4949
4950 fn i64_cmp_le_s(
4951 &mut self,
4952 loc_a: Location,
4953 loc_b: Location,
4954 ret: Location,
4955 ) -> Result<(), CompileError> {
4956 self.emit_cmpop_i64_dynamic_b(Condition::Le, loc_a, loc_b, ret)
4957 }
4958
4959 fn i64_cmp_lt_s(
4960 &mut self,
4961 loc_a: Location,
4962 loc_b: Location,
4963 ret: Location,
4964 ) -> Result<(), CompileError> {
4965 self.emit_cmpop_i64_dynamic_b(Condition::Lt, loc_a, loc_b, ret)
4966 }
4967
4968 fn i64_cmp_ge_u(
4969 &mut self,
4970 loc_a: Location,
4971 loc_b: Location,
4972 ret: Location,
4973 ) -> Result<(), CompileError> {
4974 self.emit_cmpop_i64_dynamic_b(Condition::Cs, loc_a, loc_b, ret)
4975 }
4976
4977 fn i64_cmp_gt_u(
4978 &mut self,
4979 loc_a: Location,
4980 loc_b: Location,
4981 ret: Location,
4982 ) -> Result<(), CompileError> {
4983 self.emit_cmpop_i64_dynamic_b(Condition::Hi, loc_a, loc_b, ret)
4984 }
4985
4986 fn i64_cmp_le_u(
4987 &mut self,
4988 loc_a: Location,
4989 loc_b: Location,
4990 ret: Location,
4991 ) -> Result<(), CompileError> {
4992 self.emit_cmpop_i64_dynamic_b(Condition::Ls, loc_a, loc_b, ret)
4993 }
4994
4995 fn i64_cmp_lt_u(
4996 &mut self,
4997 loc_a: Location,
4998 loc_b: Location,
4999 ret: Location,
5000 ) -> Result<(), CompileError> {
5001 self.emit_cmpop_i64_dynamic_b(Condition::Cc, loc_a, loc_b, ret)
5002 }
5003
5004 fn i64_cmp_ne(
5005 &mut self,
5006 loc_a: Location,
5007 loc_b: Location,
5008 ret: Location,
5009 ) -> Result<(), CompileError> {
5010 self.emit_cmpop_i64_dynamic_b(Condition::Ne, loc_a, loc_b, ret)
5011 }
5012
5013 fn i64_cmp_eq(
5014 &mut self,
5015 loc_a: Location,
5016 loc_b: Location,
5017 ret: Location,
5018 ) -> Result<(), CompileError> {
5019 self.emit_cmpop_i64_dynamic_b(Condition::Eq, loc_a, loc_b, ret)
5020 }
5021
5022 fn i64_clz(&mut self, src: Location, dst: Location) -> Result<(), CompileError> {
5023 self.emit_relaxed_binop(Assembler::emit_clz, Size::S64, src, dst, true)
5024 }
5025
5026 fn i64_ctz(&mut self, src: Location, dst: Location) -> Result<(), CompileError> {
5027 let mut temps = vec![];
5028 let src = self.location_to_reg(Size::S64, src, &mut temps, ImmType::None, true, None)?;
5029 let dest = self.location_to_reg(Size::S64, dst, &mut temps, ImmType::None, false, None)?;
5030 self.assembler.emit_rbit(Size::S64, src, dest)?;
5031 self.assembler.emit_clz(Size::S64, dest, dest)?;
5032 if dst != dest {
5033 self.move_location(Size::S64, dest, dst)?;
5034 }
5035 for r in temps {
5036 self.release_gpr(r);
5037 }
5038 Ok(())
5039 }
5040
5041 fn i64_popcnt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
5042 if self.has_neon {
5043 let mut temps = vec![];
5044
5045 let src_gpr =
5046 self.location_to_reg(Size::S64, loc, &mut temps, ImmType::None, true, None)?;
5047 let dst_gpr =
5048 self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
5049
5050 let mut neon_temps = vec![];
5051 let neon_temp = self.acquire_temp_simd().ok_or_else(|| {
5052 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5053 })?;
5054 neon_temps.push(neon_temp);
5055
5056 self.assembler
5057 .emit_fmov(Size::S64, src_gpr, Size::S64, Location::SIMD(neon_temp))?;
5058 self.assembler.emit_cnt(neon_temp, neon_temp)?;
5059 self.assembler.emit_addv(neon_temp, neon_temp)?;
5060 self.assembler
5061 .emit_fmov(Size::S64, Location::SIMD(neon_temp), Size::S64, dst_gpr)?;
5062
5063 if ret != dst_gpr {
5064 self.move_location(Size::S64, dst_gpr, ret)?;
5065 }
5066
5067 for r in temps {
5068 self.release_gpr(r);
5069 }
5070
5071 for r in neon_temps {
5072 self.release_simd(r);
5073 }
5074 } else {
5075 let mut temps = vec![];
5076 let src =
5077 self.location_to_reg(Size::S64, loc, &mut temps, ImmType::None, true, None)?;
5078 let dest =
5079 self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
5080 let src = if src == loc {
5081 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
5082 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5083 })?;
5084 temps.push(tmp);
5085 self.assembler
5086 .emit_mov(Size::S64, src, Location::GPR(tmp))?;
5087 Location::GPR(tmp)
5088 } else {
5089 src
5090 };
5091 let tmp = {
5092 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
5093 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5094 })?;
5095 temps.push(tmp);
5096 Location::GPR(tmp)
5097 };
5098 let label_loop = self.assembler.get_label();
5099 let label_exit = self.assembler.get_label();
5100 self.assembler
5101 .emit_mov(Size::S32, Location::GPR(GPR::XzrSp), dest)?; self.assembler.emit_cbz_label(Size::S64, src, label_exit)?; self.assembler.emit_label(label_loop)?;
5104 self.assembler
5105 .emit_add(Size::S32, dest, Location::Imm8(1), dest)?; self.assembler.emit_clz(Size::S64, src, tmp)?; self.assembler.emit_lsl(Size::S64, src, tmp, src)?; self.assembler
5109 .emit_lsl(Size::S64, src, Location::Imm8(1), src)?; self.assembler.emit_cbnz_label(Size::S64, src, label_loop)?; self.assembler.emit_label(label_exit)?;
5112 if ret != dest {
5113 self.move_location(Size::S64, dest, ret)?;
5114 }
5115 for r in temps {
5116 self.release_gpr(r);
5117 }
5118 }
5119
5120 Ok(())
5121 }
5122
5123 fn i64_shl(
5124 &mut self,
5125 loc_a: Location,
5126 loc_b: Location,
5127 ret: Location,
5128 ) -> Result<(), CompileError> {
5129 self.emit_relaxed_binop3(
5130 Assembler::emit_lsl,
5131 Size::S64,
5132 loc_a,
5133 loc_b,
5134 ret,
5135 ImmType::Shift64No0,
5136 )
5137 }
5138
5139 fn i64_shr(
5140 &mut self,
5141 loc_a: Location,
5142 loc_b: Location,
5143 ret: Location,
5144 ) -> Result<(), CompileError> {
5145 self.emit_relaxed_binop3(
5146 Assembler::emit_lsr,
5147 Size::S64,
5148 loc_a,
5149 loc_b,
5150 ret,
5151 ImmType::Shift64No0,
5152 )
5153 }
5154
5155 fn i64_sar(
5156 &mut self,
5157 loc_a: Location,
5158 loc_b: Location,
5159 ret: Location,
5160 ) -> Result<(), CompileError> {
5161 self.emit_relaxed_binop3(
5162 Assembler::emit_asr,
5163 Size::S64,
5164 loc_a,
5165 loc_b,
5166 ret,
5167 ImmType::Shift64No0,
5168 )
5169 }
5170
5171 fn i64_rol(
5172 &mut self,
5173 loc_a: Location,
5174 loc_b: Location,
5175 ret: Location,
5176 ) -> Result<(), CompileError> {
5177 let mut temps = vec![];
5179 let src2 = match loc_b {
5180 Location::Imm8(imm) => Location::Imm8(64 - (imm & 63)),
5181 Location::Imm32(imm) => Location::Imm8(64 - (imm & 63) as u8),
5182 Location::Imm64(imm) => Location::Imm8(64 - (imm & 63) as u8),
5183 _ => {
5184 let tmp1 = self.location_to_reg(
5185 Size::S64,
5186 Location::Imm32(64),
5187 &mut temps,
5188 ImmType::None,
5189 true,
5190 None,
5191 )?;
5192 let tmp2 =
5193 self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
5194 self.assembler.emit_sub(Size::S64, tmp1, tmp2, tmp1)?;
5195 tmp1
5196 }
5197 };
5198 self.emit_relaxed_binop3(
5199 Assembler::emit_ror,
5200 Size::S64,
5201 loc_a,
5202 src2,
5203 ret,
5204 ImmType::Shift64No0,
5205 )?;
5206 for r in temps {
5207 self.release_gpr(r);
5208 }
5209 Ok(())
5210 }
5211
5212 fn i64_ror(
5213 &mut self,
5214 loc_a: Location,
5215 loc_b: Location,
5216 ret: Location,
5217 ) -> Result<(), CompileError> {
5218 self.emit_relaxed_binop3(
5219 Assembler::emit_ror,
5220 Size::S64,
5221 loc_a,
5222 loc_b,
5223 ret,
5224 ImmType::Shift64No0,
5225 )
5226 }
5227
5228 fn i64_load(
5229 &mut self,
5230 addr: Location,
5231 memarg: &MemArg,
5232 ret: Location,
5233 need_check: bool,
5234 imported_memories: bool,
5235 offset: i32,
5236 heap_access_oob: Label,
5237 unaligned_atomic: Label,
5238 ) -> Result<(), CompileError> {
5239 self.memory_op(
5240 addr,
5241 memarg,
5242 false,
5243 8,
5244 need_check,
5245 imported_memories,
5246 offset,
5247 heap_access_oob,
5248 unaligned_atomic,
5249 |this, addr| this.emit_relaxed_ldr64(Size::S64, ret, Location::Memory(addr, 0)),
5250 )
5251 }
5252
5253 fn i64_load_8u(
5254 &mut self,
5255 addr: Location,
5256 memarg: &MemArg,
5257 ret: Location,
5258 need_check: bool,
5259 imported_memories: bool,
5260 offset: i32,
5261 heap_access_oob: Label,
5262 unaligned_atomic: Label,
5263 ) -> Result<(), CompileError> {
5264 self.memory_op(
5265 addr,
5266 memarg,
5267 false,
5268 1,
5269 need_check,
5270 imported_memories,
5271 offset,
5272 heap_access_oob,
5273 unaligned_atomic,
5274 |this, addr| this.emit_relaxed_ldr8(Size::S64, ret, Location::Memory(addr, 0)),
5275 )
5276 }
5277
5278 fn i64_load_8s(
5279 &mut self,
5280 addr: Location,
5281 memarg: &MemArg,
5282 ret: Location,
5283 need_check: bool,
5284 imported_memories: bool,
5285 offset: i32,
5286 heap_access_oob: Label,
5287 unaligned_atomic: Label,
5288 ) -> Result<(), CompileError> {
5289 self.memory_op(
5290 addr,
5291 memarg,
5292 false,
5293 1,
5294 need_check,
5295 imported_memories,
5296 offset,
5297 heap_access_oob,
5298 unaligned_atomic,
5299 |this, addr| this.emit_relaxed_ldr8s(Size::S64, ret, Location::Memory(addr, 0)),
5300 )
5301 }
5302
5303 fn i64_load_16u(
5304 &mut self,
5305 addr: Location,
5306 memarg: &MemArg,
5307 ret: Location,
5308 need_check: bool,
5309 imported_memories: bool,
5310 offset: i32,
5311 heap_access_oob: Label,
5312 unaligned_atomic: Label,
5313 ) -> Result<(), CompileError> {
5314 self.memory_op(
5315 addr,
5316 memarg,
5317 false,
5318 2,
5319 need_check,
5320 imported_memories,
5321 offset,
5322 heap_access_oob,
5323 unaligned_atomic,
5324 |this, addr| this.emit_relaxed_ldr16(Size::S64, ret, Location::Memory(addr, 0)),
5325 )
5326 }
5327
5328 fn i64_load_16s(
5329 &mut self,
5330 addr: Location,
5331 memarg: &MemArg,
5332 ret: Location,
5333 need_check: bool,
5334 imported_memories: bool,
5335 offset: i32,
5336 heap_access_oob: Label,
5337 unaligned_atomic: Label,
5338 ) -> Result<(), CompileError> {
5339 self.memory_op(
5340 addr,
5341 memarg,
5342 false,
5343 2,
5344 need_check,
5345 imported_memories,
5346 offset,
5347 heap_access_oob,
5348 unaligned_atomic,
5349 |this, addr| this.emit_relaxed_ldr16s(Size::S64, ret, Location::Memory(addr, 0)),
5350 )
5351 }
5352
5353 fn i64_load_32u(
5354 &mut self,
5355 addr: Location,
5356 memarg: &MemArg,
5357 ret: Location,
5358 need_check: bool,
5359 imported_memories: bool,
5360 offset: i32,
5361 heap_access_oob: Label,
5362 unaligned_atomic: Label,
5363 ) -> Result<(), CompileError> {
5364 self.memory_op(
5365 addr,
5366 memarg,
5367 false,
5368 4,
5369 need_check,
5370 imported_memories,
5371 offset,
5372 heap_access_oob,
5373 unaligned_atomic,
5374 |this, addr| this.emit_relaxed_ldr32(Size::S64, ret, Location::Memory(addr, 0)),
5375 )
5376 }
5377
5378 fn i64_load_32s(
5379 &mut self,
5380 addr: Location,
5381 memarg: &MemArg,
5382 ret: Location,
5383 need_check: bool,
5384 imported_memories: bool,
5385 offset: i32,
5386 heap_access_oob: Label,
5387 unaligned_atomic: Label,
5388 ) -> Result<(), CompileError> {
5389 self.memory_op(
5390 addr,
5391 memarg,
5392 false,
5393 4,
5394 need_check,
5395 imported_memories,
5396 offset,
5397 heap_access_oob,
5398 unaligned_atomic,
5399 |this, addr| this.emit_relaxed_ldr32s(Size::S64, ret, Location::Memory(addr, 0)),
5400 )
5401 }
5402
5403 fn i64_atomic_load(
5404 &mut self,
5405 addr: Location,
5406 memarg: &MemArg,
5407 ret: Location,
5408 need_check: bool,
5409 imported_memories: bool,
5410 offset: i32,
5411 heap_access_oob: Label,
5412 unaligned_atomic: Label,
5413 ) -> Result<(), CompileError> {
5414 self.memory_op(
5415 addr,
5416 memarg,
5417 true,
5418 8,
5419 need_check,
5420 imported_memories,
5421 offset,
5422 heap_access_oob,
5423 unaligned_atomic,
5424 |this, addr| this.emit_relaxed_ldr64(Size::S64, ret, Location::Memory(addr, 0)),
5425 )
5426 }
5427
5428 fn i64_atomic_load_8u(
5429 &mut self,
5430 addr: Location,
5431 memarg: &MemArg,
5432 ret: Location,
5433 need_check: bool,
5434 imported_memories: bool,
5435 offset: i32,
5436 heap_access_oob: Label,
5437 unaligned_atomic: Label,
5438 ) -> Result<(), CompileError> {
5439 self.memory_op(
5440 addr,
5441 memarg,
5442 true,
5443 1,
5444 need_check,
5445 imported_memories,
5446 offset,
5447 heap_access_oob,
5448 unaligned_atomic,
5449 |this, addr| this.emit_relaxed_ldr8(Size::S64, ret, Location::Memory(addr, 0)),
5450 )
5451 }
5452
5453 fn i64_atomic_load_16u(
5454 &mut self,
5455 addr: Location,
5456 memarg: &MemArg,
5457 ret: Location,
5458 need_check: bool,
5459 imported_memories: bool,
5460 offset: i32,
5461 heap_access_oob: Label,
5462 unaligned_atomic: Label,
5463 ) -> Result<(), CompileError> {
5464 self.memory_op(
5465 addr,
5466 memarg,
5467 true,
5468 2,
5469 need_check,
5470 imported_memories,
5471 offset,
5472 heap_access_oob,
5473 unaligned_atomic,
5474 |this, addr| this.emit_relaxed_ldr16(Size::S64, ret, Location::Memory(addr, 0)),
5475 )
5476 }
5477
5478 fn i64_atomic_load_32u(
5479 &mut self,
5480 addr: Location,
5481 memarg: &MemArg,
5482 ret: Location,
5483 need_check: bool,
5484 imported_memories: bool,
5485 offset: i32,
5486 heap_access_oob: Label,
5487 unaligned_atomic: Label,
5488 ) -> Result<(), CompileError> {
5489 self.memory_op(
5490 addr,
5491 memarg,
5492 true,
5493 4,
5494 need_check,
5495 imported_memories,
5496 offset,
5497 heap_access_oob,
5498 unaligned_atomic,
5499 |this, addr| this.emit_relaxed_ldr32(Size::S64, ret, Location::Memory(addr, 0)),
5500 )
5501 }
5502
5503 fn i64_save(
5504 &mut self,
5505 target_value: Location,
5506 memarg: &MemArg,
5507 target_addr: Location,
5508 need_check: bool,
5509 imported_memories: bool,
5510 offset: i32,
5511 heap_access_oob: Label,
5512 unaligned_atomic: Label,
5513 ) -> Result<(), CompileError> {
5514 self.memory_op(
5515 target_addr,
5516 memarg,
5517 false,
5518 8,
5519 need_check,
5520 imported_memories,
5521 offset,
5522 heap_access_oob,
5523 unaligned_atomic,
5524 |this, addr| this.emit_relaxed_str64(target_value, Location::Memory(addr, 0)),
5525 )
5526 }
5527
5528 fn i64_save_8(
5529 &mut self,
5530 target_value: Location,
5531 memarg: &MemArg,
5532 target_addr: Location,
5533 need_check: bool,
5534 imported_memories: bool,
5535 offset: i32,
5536 heap_access_oob: Label,
5537 unaligned_atomic: Label,
5538 ) -> Result<(), CompileError> {
5539 self.memory_op(
5540 target_addr,
5541 memarg,
5542 false,
5543 1,
5544 need_check,
5545 imported_memories,
5546 offset,
5547 heap_access_oob,
5548 unaligned_atomic,
5549 |this, addr| this.emit_relaxed_str8(target_value, Location::Memory(addr, 0)),
5550 )
5551 }
5552
5553 fn i64_save_16(
5554 &mut self,
5555 target_value: Location,
5556 memarg: &MemArg,
5557 target_addr: Location,
5558 need_check: bool,
5559 imported_memories: bool,
5560 offset: i32,
5561 heap_access_oob: Label,
5562 unaligned_atomic: Label,
5563 ) -> Result<(), CompileError> {
5564 self.memory_op(
5565 target_addr,
5566 memarg,
5567 false,
5568 2,
5569 need_check,
5570 imported_memories,
5571 offset,
5572 heap_access_oob,
5573 unaligned_atomic,
5574 |this, addr| this.emit_relaxed_str16(target_value, Location::Memory(addr, 0)),
5575 )
5576 }
5577
5578 fn i64_save_32(
5579 &mut self,
5580 target_value: Location,
5581 memarg: &MemArg,
5582 target_addr: Location,
5583 need_check: bool,
5584 imported_memories: bool,
5585 offset: i32,
5586 heap_access_oob: Label,
5587 unaligned_atomic: Label,
5588 ) -> Result<(), CompileError> {
5589 self.memory_op(
5590 target_addr,
5591 memarg,
5592 false,
5593 4,
5594 need_check,
5595 imported_memories,
5596 offset,
5597 heap_access_oob,
5598 unaligned_atomic,
5599 |this, addr| this.emit_relaxed_str32(target_value, Location::Memory(addr, 0)),
5600 )
5601 }
5602
5603 fn i64_atomic_save(
5604 &mut self,
5605 target_value: Location,
5606 memarg: &MemArg,
5607 target_addr: Location,
5608 need_check: bool,
5609 imported_memories: bool,
5610 offset: i32,
5611 heap_access_oob: Label,
5612 unaligned_atomic: Label,
5613 ) -> Result<(), CompileError> {
5614 self.memory_op(
5615 target_addr,
5616 memarg,
5617 true,
5618 8,
5619 need_check,
5620 imported_memories,
5621 offset,
5622 heap_access_oob,
5623 unaligned_atomic,
5624 |this, addr| this.emit_relaxed_str64(target_value, Location::Memory(addr, 0)),
5625 )?;
5626 self.assembler.emit_dmb()
5627 }
5628
5629 fn i64_atomic_save_8(
5630 &mut self,
5631 target_value: Location,
5632 memarg: &MemArg,
5633 target_addr: Location,
5634 need_check: bool,
5635 imported_memories: bool,
5636 offset: i32,
5637 heap_access_oob: Label,
5638 unaligned_atomic: Label,
5639 ) -> Result<(), CompileError> {
5640 self.memory_op(
5641 target_addr,
5642 memarg,
5643 true,
5644 1,
5645 need_check,
5646 imported_memories,
5647 offset,
5648 heap_access_oob,
5649 unaligned_atomic,
5650 |this, addr| this.emit_relaxed_str8(target_value, Location::Memory(addr, 0)),
5651 )?;
5652 self.assembler.emit_dmb()
5653 }
5654
5655 fn i64_atomic_save_16(
5656 &mut self,
5657 target_value: Location,
5658 memarg: &MemArg,
5659 target_addr: Location,
5660 need_check: bool,
5661 imported_memories: bool,
5662 offset: i32,
5663 heap_access_oob: Label,
5664 unaligned_atomic: Label,
5665 ) -> Result<(), CompileError> {
5666 self.memory_op(
5667 target_addr,
5668 memarg,
5669 true,
5670 2,
5671 need_check,
5672 imported_memories,
5673 offset,
5674 heap_access_oob,
5675 unaligned_atomic,
5676 |this, addr| this.emit_relaxed_str16(target_value, Location::Memory(addr, 0)),
5677 )?;
5678 self.assembler.emit_dmb()
5679 }
5680
5681 fn i64_atomic_save_32(
5682 &mut self,
5683 target_value: Location,
5684 memarg: &MemArg,
5685 target_addr: Location,
5686 need_check: bool,
5687 imported_memories: bool,
5688 offset: i32,
5689 heap_access_oob: Label,
5690 unaligned_atomic: Label,
5691 ) -> Result<(), CompileError> {
5692 self.memory_op(
5693 target_addr,
5694 memarg,
5695 true,
5696 4,
5697 need_check,
5698 imported_memories,
5699 offset,
5700 heap_access_oob,
5701 unaligned_atomic,
5702 |this, addr| this.emit_relaxed_str32(target_value, Location::Memory(addr, 0)),
5703 )?;
5704 self.assembler.emit_dmb()
5705 }
5706
5707 fn i64_atomic_add(
5708 &mut self,
5709 loc: Location,
5710 target: Location,
5711 memarg: &MemArg,
5712 ret: Location,
5713 need_check: bool,
5714 imported_memories: bool,
5715 offset: i32,
5716 heap_access_oob: Label,
5717 unaligned_atomic: Label,
5718 ) -> Result<(), CompileError> {
5719 self.memory_op(
5720 target,
5721 memarg,
5722 true,
5723 8,
5724 need_check,
5725 imported_memories,
5726 offset,
5727 heap_access_oob,
5728 unaligned_atomic,
5729 |this, addr| {
5730 let mut temps = vec![];
5731 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
5732 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5733 })?;
5734 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
5735 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5736 })?;
5737 let dst =
5738 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
5739 let reread = this.get_label();
5740
5741 this.emit_label(reread)?;
5742 this.assembler
5743 .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?;
5744 this.emit_binop_add64(dst, loc, Location::GPR(tmp1))?;
5745 this.assembler.emit_stlxr(
5746 Size::S64,
5747 Location::GPR(tmp2),
5748 Location::GPR(tmp1),
5749 Location::GPR(addr),
5750 )?;
5751 this.assembler
5752 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
5753 this.assembler.emit_dmb()?;
5754
5755 if dst != ret {
5756 this.move_location(Size::S64, ret, dst)?;
5757 }
5758 for r in temps {
5759 this.release_gpr(r);
5760 }
5761 this.release_gpr(tmp1);
5762 this.release_gpr(tmp2);
5763 Ok(())
5764 },
5765 )
5766 }
5767
5768 fn i64_atomic_add_8u(
5769 &mut self,
5770 loc: Location,
5771 target: Location,
5772 memarg: &MemArg,
5773 ret: Location,
5774 need_check: bool,
5775 imported_memories: bool,
5776 offset: i32,
5777 heap_access_oob: Label,
5778 unaligned_atomic: Label,
5779 ) -> Result<(), CompileError> {
5780 self.memory_op(
5781 target,
5782 memarg,
5783 true,
5784 1,
5785 need_check,
5786 imported_memories,
5787 offset,
5788 heap_access_oob,
5789 unaligned_atomic,
5790 |this, addr| {
5791 let mut temps = vec![];
5792 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
5793 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5794 })?;
5795 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
5796 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5797 })?;
5798 let dst =
5799 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
5800 let reread = this.get_label();
5801
5802 this.emit_label(reread)?;
5803 this.assembler
5804 .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?;
5805 this.emit_binop_add64(dst, loc, Location::GPR(tmp1))?;
5806 this.assembler.emit_stlxrb(
5807 Size::S64,
5808 Location::GPR(tmp2),
5809 Location::GPR(tmp1),
5810 Location::GPR(addr),
5811 )?;
5812 this.assembler
5813 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
5814 this.assembler.emit_dmb()?;
5815
5816 if dst != ret {
5817 this.move_location(Size::S64, ret, dst)?;
5818 }
5819 for r in temps {
5820 this.release_gpr(r);
5821 }
5822 this.release_gpr(tmp1);
5823 this.release_gpr(tmp2);
5824 Ok(())
5825 },
5826 )
5827 }
5828
5829 fn i64_atomic_add_16u(
5830 &mut self,
5831 loc: Location,
5832 target: Location,
5833 memarg: &MemArg,
5834 ret: Location,
5835 need_check: bool,
5836 imported_memories: bool,
5837 offset: i32,
5838 heap_access_oob: Label,
5839 unaligned_atomic: Label,
5840 ) -> Result<(), CompileError> {
5841 self.memory_op(
5842 target,
5843 memarg,
5844 true,
5845 2,
5846 need_check,
5847 imported_memories,
5848 offset,
5849 heap_access_oob,
5850 unaligned_atomic,
5851 |this, addr| {
5852 let mut temps = vec![];
5853 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
5854 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5855 })?;
5856 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
5857 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5858 })?;
5859 let dst =
5860 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
5861 let reread = this.get_label();
5862
5863 this.emit_label(reread)?;
5864 this.assembler
5865 .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?;
5866 this.emit_binop_add64(dst, loc, Location::GPR(tmp1))?;
5867 this.assembler.emit_stlxrh(
5868 Size::S64,
5869 Location::GPR(tmp2),
5870 Location::GPR(tmp1),
5871 Location::GPR(addr),
5872 )?;
5873 this.assembler
5874 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
5875 this.assembler.emit_dmb()?;
5876
5877 if dst != ret {
5878 this.move_location(Size::S64, ret, dst)?;
5879 }
5880 for r in temps {
5881 this.release_gpr(r);
5882 }
5883 this.release_gpr(tmp1);
5884 this.release_gpr(tmp2);
5885 Ok(())
5886 },
5887 )
5888 }
5889
5890 fn i64_atomic_add_32u(
5891 &mut self,
5892 loc: Location,
5893 target: Location,
5894 memarg: &MemArg,
5895 ret: Location,
5896 need_check: bool,
5897 imported_memories: bool,
5898 offset: i32,
5899 heap_access_oob: Label,
5900 unaligned_atomic: Label,
5901 ) -> Result<(), CompileError> {
5902 self.memory_op(
5903 target,
5904 memarg,
5905 true,
5906 4,
5907 need_check,
5908 imported_memories,
5909 offset,
5910 heap_access_oob,
5911 unaligned_atomic,
5912 |this, addr| {
5913 let mut temps = vec![];
5914 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
5915 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5916 })?;
5917 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
5918 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5919 })?;
5920 let dst =
5921 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
5922 let reread = this.get_label();
5923
5924 this.emit_label(reread)?;
5925 this.assembler
5926 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
5927 this.emit_binop_add64(dst, loc, Location::GPR(tmp1))?;
5928 this.assembler.emit_stlxr(
5929 Size::S32,
5930 Location::GPR(tmp2),
5931 Location::GPR(tmp1),
5932 Location::GPR(addr),
5933 )?;
5934 this.assembler
5935 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
5936 this.assembler.emit_dmb()?;
5937
5938 if dst != ret {
5939 this.move_location(Size::S64, ret, dst)?;
5940 }
5941 for r in temps {
5942 this.release_gpr(r);
5943 }
5944 this.release_gpr(tmp1);
5945 this.release_gpr(tmp2);
5946 Ok(())
5947 },
5948 )
5949 }
5950
5951 fn i64_atomic_sub(
5952 &mut self,
5953 loc: Location,
5954 target: Location,
5955 memarg: &MemArg,
5956 ret: Location,
5957 need_check: bool,
5958 imported_memories: bool,
5959 offset: i32,
5960 heap_access_oob: Label,
5961 unaligned_atomic: Label,
5962 ) -> Result<(), CompileError> {
5963 self.memory_op(
5964 target,
5965 memarg,
5966 true,
5967 8,
5968 need_check,
5969 imported_memories,
5970 offset,
5971 heap_access_oob,
5972 unaligned_atomic,
5973 |this, addr| {
5974 let mut temps = vec![];
5975 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
5976 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5977 })?;
5978 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
5979 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5980 })?;
5981 let dst =
5982 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
5983 let reread = this.get_label();
5984
5985 this.emit_label(reread)?;
5986 this.assembler
5987 .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?;
5988 this.emit_binop_sub64(dst, loc, Location::GPR(tmp1))?;
5989 this.assembler.emit_stlxr(
5990 Size::S64,
5991 Location::GPR(tmp2),
5992 Location::GPR(tmp1),
5993 Location::GPR(addr),
5994 )?;
5995 this.assembler
5996 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
5997 this.assembler.emit_dmb()?;
5998
5999 if dst != ret {
6000 this.move_location(Size::S64, ret, dst)?;
6001 }
6002 for r in temps {
6003 this.release_gpr(r);
6004 }
6005 this.release_gpr(tmp1);
6006 this.release_gpr(tmp2);
6007 Ok(())
6008 },
6009 )
6010 }
6011
6012 fn i64_atomic_sub_8u(
6013 &mut self,
6014 loc: Location,
6015 target: Location,
6016 memarg: &MemArg,
6017 ret: Location,
6018 need_check: bool,
6019 imported_memories: bool,
6020 offset: i32,
6021 heap_access_oob: Label,
6022 unaligned_atomic: Label,
6023 ) -> Result<(), CompileError> {
6024 self.memory_op(
6025 target,
6026 memarg,
6027 true,
6028 1,
6029 need_check,
6030 imported_memories,
6031 offset,
6032 heap_access_oob,
6033 unaligned_atomic,
6034 |this, addr| {
6035 let mut temps = vec![];
6036 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6037 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6038 })?;
6039 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6040 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6041 })?;
6042 let dst =
6043 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6044 let reread = this.get_label();
6045
6046 this.emit_label(reread)?;
6047 this.assembler
6048 .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?;
6049 this.emit_binop_sub64(dst, loc, Location::GPR(tmp1))?;
6050 this.assembler.emit_stlxrb(
6051 Size::S64,
6052 Location::GPR(tmp2),
6053 Location::GPR(tmp1),
6054 Location::GPR(addr),
6055 )?;
6056 this.assembler
6057 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6058 this.assembler.emit_dmb()?;
6059
6060 if dst != ret {
6061 this.move_location(Size::S64, ret, dst)?;
6062 }
6063 for r in temps {
6064 this.release_gpr(r);
6065 }
6066 this.release_gpr(tmp1);
6067 this.release_gpr(tmp2);
6068 Ok(())
6069 },
6070 )
6071 }
6072
6073 fn i64_atomic_sub_16u(
6074 &mut self,
6075 loc: Location,
6076 target: Location,
6077 memarg: &MemArg,
6078 ret: Location,
6079 need_check: bool,
6080 imported_memories: bool,
6081 offset: i32,
6082 heap_access_oob: Label,
6083 unaligned_atomic: Label,
6084 ) -> Result<(), CompileError> {
6085 self.memory_op(
6086 target,
6087 memarg,
6088 true,
6089 2,
6090 need_check,
6091 imported_memories,
6092 offset,
6093 heap_access_oob,
6094 unaligned_atomic,
6095 |this, addr| {
6096 let mut temps = vec![];
6097 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6098 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6099 })?;
6100 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6101 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6102 })?;
6103 let dst =
6104 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6105 let reread = this.get_label();
6106
6107 this.emit_label(reread)?;
6108 this.assembler
6109 .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?;
6110 this.emit_binop_sub64(dst, loc, Location::GPR(tmp1))?;
6111 this.assembler.emit_stlxrh(
6112 Size::S64,
6113 Location::GPR(tmp2),
6114 Location::GPR(tmp1),
6115 Location::GPR(addr),
6116 )?;
6117 this.assembler
6118 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6119 this.assembler.emit_dmb()?;
6120
6121 if dst != ret {
6122 this.move_location(Size::S64, ret, dst)?;
6123 }
6124 for r in temps {
6125 this.release_gpr(r);
6126 }
6127 this.release_gpr(tmp1);
6128 this.release_gpr(tmp2);
6129 Ok(())
6130 },
6131 )
6132 }
6133
6134 fn i64_atomic_sub_32u(
6135 &mut self,
6136 loc: Location,
6137 target: Location,
6138 memarg: &MemArg,
6139 ret: Location,
6140 need_check: bool,
6141 imported_memories: bool,
6142 offset: i32,
6143 heap_access_oob: Label,
6144 unaligned_atomic: Label,
6145 ) -> Result<(), CompileError> {
6146 self.memory_op(
6147 target,
6148 memarg,
6149 true,
6150 4,
6151 need_check,
6152 imported_memories,
6153 offset,
6154 heap_access_oob,
6155 unaligned_atomic,
6156 |this, addr| {
6157 let mut temps = vec![];
6158 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6159 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6160 })?;
6161 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6162 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6163 })?;
6164 let dst =
6165 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6166 let reread = this.get_label();
6167
6168 this.emit_label(reread)?;
6169 this.assembler
6170 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
6171 this.emit_binop_sub64(dst, loc, Location::GPR(tmp1))?;
6172 this.assembler.emit_stlxr(
6173 Size::S32,
6174 Location::GPR(tmp2),
6175 Location::GPR(tmp1),
6176 Location::GPR(addr),
6177 )?;
6178 this.assembler
6179 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6180 this.assembler.emit_dmb()?;
6181
6182 if dst != ret {
6183 this.move_location(Size::S64, ret, dst)?;
6184 }
6185 for r in temps {
6186 this.release_gpr(r);
6187 }
6188 this.release_gpr(tmp1);
6189 this.release_gpr(tmp2);
6190 Ok(())
6191 },
6192 )
6193 }
6194
6195 fn i64_atomic_and(
6196 &mut self,
6197 loc: Location,
6198 target: Location,
6199 memarg: &MemArg,
6200 ret: Location,
6201 need_check: bool,
6202 imported_memories: bool,
6203 offset: i32,
6204 heap_access_oob: Label,
6205 unaligned_atomic: Label,
6206 ) -> Result<(), CompileError> {
6207 self.memory_op(
6208 target,
6209 memarg,
6210 true,
6211 8,
6212 need_check,
6213 imported_memories,
6214 offset,
6215 heap_access_oob,
6216 unaligned_atomic,
6217 |this, addr| {
6218 let mut temps = vec![];
6219 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6220 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6221 })?;
6222 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6223 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6224 })?;
6225 let dst =
6226 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6227 let reread = this.get_label();
6228
6229 this.emit_label(reread)?;
6230 this.assembler
6231 .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?;
6232 this.emit_binop_and64(dst, loc, Location::GPR(tmp1))?;
6233 this.assembler.emit_stlxr(
6234 Size::S64,
6235 Location::GPR(tmp2),
6236 Location::GPR(tmp1),
6237 Location::GPR(addr),
6238 )?;
6239 this.assembler
6240 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6241 this.assembler.emit_dmb()?;
6242
6243 if dst != ret {
6244 this.move_location(Size::S64, ret, dst)?;
6245 }
6246 for r in temps {
6247 this.release_gpr(r);
6248 }
6249 this.release_gpr(tmp1);
6250 this.release_gpr(tmp2);
6251 Ok(())
6252 },
6253 )
6254 }
6255
6256 fn i64_atomic_and_8u(
6257 &mut self,
6258 loc: Location,
6259 target: Location,
6260 memarg: &MemArg,
6261 ret: Location,
6262 need_check: bool,
6263 imported_memories: bool,
6264 offset: i32,
6265 heap_access_oob: Label,
6266 unaligned_atomic: Label,
6267 ) -> Result<(), CompileError> {
6268 self.memory_op(
6269 target,
6270 memarg,
6271 true,
6272 1,
6273 need_check,
6274 imported_memories,
6275 offset,
6276 heap_access_oob,
6277 unaligned_atomic,
6278 |this, addr| {
6279 let mut temps = vec![];
6280 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6281 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6282 })?;
6283 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6284 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6285 })?;
6286 let dst =
6287 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6288 let reread = this.get_label();
6289
6290 this.emit_label(reread)?;
6291 this.assembler
6292 .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?;
6293 this.emit_binop_and64(dst, loc, Location::GPR(tmp1))?;
6294 this.assembler.emit_stlxrb(
6295 Size::S64,
6296 Location::GPR(tmp2),
6297 Location::GPR(tmp1),
6298 Location::GPR(addr),
6299 )?;
6300 this.assembler
6301 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6302 this.assembler.emit_dmb()?;
6303
6304 if dst != ret {
6305 this.move_location(Size::S64, ret, dst)?;
6306 }
6307 for r in temps {
6308 this.release_gpr(r);
6309 }
6310 this.release_gpr(tmp1);
6311 this.release_gpr(tmp2);
6312 Ok(())
6313 },
6314 )
6315 }
6316
6317 fn i64_atomic_and_16u(
6318 &mut self,
6319 loc: Location,
6320 target: Location,
6321 memarg: &MemArg,
6322 ret: Location,
6323 need_check: bool,
6324 imported_memories: bool,
6325 offset: i32,
6326 heap_access_oob: Label,
6327 unaligned_atomic: Label,
6328 ) -> Result<(), CompileError> {
6329 self.memory_op(
6330 target,
6331 memarg,
6332 true,
6333 2,
6334 need_check,
6335 imported_memories,
6336 offset,
6337 heap_access_oob,
6338 unaligned_atomic,
6339 |this, addr| {
6340 let mut temps = vec![];
6341 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6342 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6343 })?;
6344 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6345 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6346 })?;
6347 let dst =
6348 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6349 let reread = this.get_label();
6350
6351 this.emit_label(reread)?;
6352 this.assembler
6353 .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?;
6354 this.emit_binop_and64(dst, loc, Location::GPR(tmp1))?;
6355 this.assembler.emit_stlxrh(
6356 Size::S64,
6357 Location::GPR(tmp2),
6358 Location::GPR(tmp1),
6359 Location::GPR(addr),
6360 )?;
6361 this.assembler
6362 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6363 this.assembler.emit_dmb()?;
6364
6365 if dst != ret {
6366 this.move_location(Size::S64, ret, dst)?;
6367 }
6368 for r in temps {
6369 this.release_gpr(r);
6370 }
6371 this.release_gpr(tmp1);
6372 this.release_gpr(tmp2);
6373 Ok(())
6374 },
6375 )
6376 }
6377
6378 fn i64_atomic_and_32u(
6379 &mut self,
6380 loc: Location,
6381 target: Location,
6382 memarg: &MemArg,
6383 ret: Location,
6384 need_check: bool,
6385 imported_memories: bool,
6386 offset: i32,
6387 heap_access_oob: Label,
6388 unaligned_atomic: Label,
6389 ) -> Result<(), CompileError> {
6390 self.memory_op(
6391 target,
6392 memarg,
6393 true,
6394 4,
6395 need_check,
6396 imported_memories,
6397 offset,
6398 heap_access_oob,
6399 unaligned_atomic,
6400 |this, addr| {
6401 let mut temps = vec![];
6402 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6403 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6404 })?;
6405 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6406 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6407 })?;
6408 let dst =
6409 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6410 let reread = this.get_label();
6411
6412 this.emit_label(reread)?;
6413 this.assembler
6414 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
6415 this.emit_binop_and64(dst, loc, Location::GPR(tmp1))?;
6416 this.assembler.emit_stlxr(
6417 Size::S32,
6418 Location::GPR(tmp2),
6419 Location::GPR(tmp1),
6420 Location::GPR(addr),
6421 )?;
6422 this.assembler
6423 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6424 this.assembler.emit_dmb()?;
6425
6426 if dst != ret {
6427 this.move_location(Size::S64, ret, dst)?;
6428 }
6429 for r in temps {
6430 this.release_gpr(r);
6431 }
6432 this.release_gpr(tmp1);
6433 this.release_gpr(tmp2);
6434 Ok(())
6435 },
6436 )
6437 }
6438
6439 fn i64_atomic_or(
6440 &mut self,
6441 loc: Location,
6442 target: Location,
6443 memarg: &MemArg,
6444 ret: Location,
6445 need_check: bool,
6446 imported_memories: bool,
6447 offset: i32,
6448 heap_access_oob: Label,
6449 unaligned_atomic: Label,
6450 ) -> Result<(), CompileError> {
6451 self.memory_op(
6452 target,
6453 memarg,
6454 true,
6455 8,
6456 need_check,
6457 imported_memories,
6458 offset,
6459 heap_access_oob,
6460 unaligned_atomic,
6461 |this, addr| {
6462 let mut temps = vec![];
6463 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6464 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6465 })?;
6466 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6467 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6468 })?;
6469 let dst =
6470 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6471 let reread = this.get_label();
6472
6473 this.emit_label(reread)?;
6474 this.assembler
6475 .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?;
6476 this.emit_binop_or64(dst, loc, Location::GPR(tmp1))?;
6477 this.assembler.emit_stlxr(
6478 Size::S64,
6479 Location::GPR(tmp2),
6480 Location::GPR(tmp1),
6481 Location::GPR(addr),
6482 )?;
6483 this.assembler
6484 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6485 this.assembler.emit_dmb()?;
6486
6487 if dst != ret {
6488 this.move_location(Size::S64, ret, dst)?;
6489 }
6490 for r in temps {
6491 this.release_gpr(r);
6492 }
6493 this.release_gpr(tmp1);
6494 this.release_gpr(tmp2);
6495 Ok(())
6496 },
6497 )
6498 }
6499
6500 fn i64_atomic_or_8u(
6501 &mut self,
6502 loc: Location,
6503 target: Location,
6504 memarg: &MemArg,
6505 ret: Location,
6506 need_check: bool,
6507 imported_memories: bool,
6508 offset: i32,
6509 heap_access_oob: Label,
6510 unaligned_atomic: Label,
6511 ) -> Result<(), CompileError> {
6512 self.memory_op(
6513 target,
6514 memarg,
6515 true,
6516 1,
6517 need_check,
6518 imported_memories,
6519 offset,
6520 heap_access_oob,
6521 unaligned_atomic,
6522 |this, addr| {
6523 let mut temps = vec![];
6524 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6525 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6526 })?;
6527 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6528 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6529 })?;
6530 let dst =
6531 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6532 let reread = this.get_label();
6533
6534 this.emit_label(reread)?;
6535 this.assembler
6536 .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?;
6537 this.emit_binop_or64(dst, loc, Location::GPR(tmp1))?;
6538 this.assembler.emit_stlxrb(
6539 Size::S64,
6540 Location::GPR(tmp2),
6541 Location::GPR(tmp1),
6542 Location::GPR(addr),
6543 )?;
6544 this.assembler
6545 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6546 this.assembler.emit_dmb()?;
6547
6548 if dst != ret {
6549 this.move_location(Size::S64, ret, dst)?;
6550 }
6551 for r in temps {
6552 this.release_gpr(r);
6553 }
6554 this.release_gpr(tmp1);
6555 this.release_gpr(tmp2);
6556 Ok(())
6557 },
6558 )
6559 }
6560
6561 fn i64_atomic_or_16u(
6562 &mut self,
6563 loc: Location,
6564 target: Location,
6565 memarg: &MemArg,
6566 ret: Location,
6567 need_check: bool,
6568 imported_memories: bool,
6569 offset: i32,
6570 heap_access_oob: Label,
6571 unaligned_atomic: Label,
6572 ) -> Result<(), CompileError> {
6573 self.memory_op(
6574 target,
6575 memarg,
6576 true,
6577 2,
6578 need_check,
6579 imported_memories,
6580 offset,
6581 heap_access_oob,
6582 unaligned_atomic,
6583 |this, addr| {
6584 let mut temps = vec![];
6585 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6586 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6587 })?;
6588 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6589 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6590 })?;
6591 let dst =
6592 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6593 let reread = this.get_label();
6594
6595 this.emit_label(reread)?;
6596 this.assembler
6597 .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?;
6598 this.emit_binop_or64(dst, loc, Location::GPR(tmp1))?;
6599 this.assembler.emit_stlxrh(
6600 Size::S64,
6601 Location::GPR(tmp2),
6602 Location::GPR(tmp1),
6603 Location::GPR(addr),
6604 )?;
6605 this.assembler
6606 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6607 this.assembler.emit_dmb()?;
6608
6609 if dst != ret {
6610 this.move_location(Size::S64, ret, dst)?;
6611 }
6612 for r in temps {
6613 this.release_gpr(r);
6614 }
6615 this.release_gpr(tmp1);
6616 this.release_gpr(tmp2);
6617 Ok(())
6618 },
6619 )
6620 }
6621
6622 fn i64_atomic_or_32u(
6623 &mut self,
6624 loc: Location,
6625 target: Location,
6626 memarg: &MemArg,
6627 ret: Location,
6628 need_check: bool,
6629 imported_memories: bool,
6630 offset: i32,
6631 heap_access_oob: Label,
6632 unaligned_atomic: Label,
6633 ) -> Result<(), CompileError> {
6634 self.memory_op(
6635 target,
6636 memarg,
6637 true,
6638 4,
6639 need_check,
6640 imported_memories,
6641 offset,
6642 heap_access_oob,
6643 unaligned_atomic,
6644 |this, addr| {
6645 let mut temps = vec![];
6646 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6647 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6648 })?;
6649 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6650 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6651 })?;
6652 let dst =
6653 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6654 let reread = this.get_label();
6655
6656 this.emit_label(reread)?;
6657 this.assembler
6658 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
6659 this.emit_binop_or64(dst, loc, Location::GPR(tmp1))?;
6660 this.assembler.emit_stlxr(
6661 Size::S32,
6662 Location::GPR(tmp2),
6663 Location::GPR(tmp1),
6664 Location::GPR(addr),
6665 )?;
6666 this.assembler
6667 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6668 this.assembler.emit_dmb()?;
6669
6670 if dst != ret {
6671 this.move_location(Size::S64, ret, dst)?;
6672 }
6673 for r in temps {
6674 this.release_gpr(r);
6675 }
6676 this.release_gpr(tmp1);
6677 this.release_gpr(tmp2);
6678 Ok(())
6679 },
6680 )
6681 }
6682
6683 fn i64_atomic_xor(
6684 &mut self,
6685 loc: Location,
6686 target: Location,
6687 memarg: &MemArg,
6688 ret: Location,
6689 need_check: bool,
6690 imported_memories: bool,
6691 offset: i32,
6692 heap_access_oob: Label,
6693 unaligned_atomic: Label,
6694 ) -> Result<(), CompileError> {
6695 self.memory_op(
6696 target,
6697 memarg,
6698 true,
6699 8,
6700 need_check,
6701 imported_memories,
6702 offset,
6703 heap_access_oob,
6704 unaligned_atomic,
6705 |this, addr| {
6706 let mut temps = vec![];
6707 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6708 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6709 })?;
6710 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6711 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6712 })?;
6713 let dst =
6714 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6715 let reread = this.get_label();
6716
6717 this.emit_label(reread)?;
6718 this.assembler
6719 .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?;
6720 this.emit_binop_xor64(dst, loc, Location::GPR(tmp1))?;
6721 this.assembler.emit_stlxr(
6722 Size::S64,
6723 Location::GPR(tmp2),
6724 Location::GPR(tmp1),
6725 Location::GPR(addr),
6726 )?;
6727 this.assembler
6728 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6729 this.assembler.emit_dmb()?;
6730
6731 if dst != ret {
6732 this.move_location(Size::S64, ret, dst)?;
6733 }
6734 for r in temps {
6735 this.release_gpr(r);
6736 }
6737 this.release_gpr(tmp1);
6738 this.release_gpr(tmp2);
6739 Ok(())
6740 },
6741 )
6742 }
6743
6744 fn i64_atomic_xor_8u(
6745 &mut self,
6746 loc: Location,
6747 target: Location,
6748 memarg: &MemArg,
6749 ret: Location,
6750 need_check: bool,
6751 imported_memories: bool,
6752 offset: i32,
6753 heap_access_oob: Label,
6754 unaligned_atomic: Label,
6755 ) -> Result<(), CompileError> {
6756 self.memory_op(
6757 target,
6758 memarg,
6759 true,
6760 1,
6761 need_check,
6762 imported_memories,
6763 offset,
6764 heap_access_oob,
6765 unaligned_atomic,
6766 |this, addr| {
6767 let mut temps = vec![];
6768 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6769 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6770 })?;
6771 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6772 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6773 })?;
6774 let dst =
6775 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6776 let reread = this.get_label();
6777
6778 this.emit_label(reread)?;
6779 this.assembler
6780 .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?;
6781 this.emit_binop_xor64(dst, loc, Location::GPR(tmp1))?;
6782 this.assembler.emit_stlxrb(
6783 Size::S64,
6784 Location::GPR(tmp2),
6785 Location::GPR(tmp1),
6786 Location::GPR(addr),
6787 )?;
6788 this.assembler
6789 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6790 this.assembler.emit_dmb()?;
6791
6792 if dst != ret {
6793 this.move_location(Size::S64, ret, dst)?;
6794 }
6795 for r in temps {
6796 this.release_gpr(r);
6797 }
6798 this.release_gpr(tmp1);
6799 this.release_gpr(tmp2);
6800 Ok(())
6801 },
6802 )
6803 }
6804
6805 fn i64_atomic_xor_16u(
6806 &mut self,
6807 loc: Location,
6808 target: Location,
6809 memarg: &MemArg,
6810 ret: Location,
6811 need_check: bool,
6812 imported_memories: bool,
6813 offset: i32,
6814 heap_access_oob: Label,
6815 unaligned_atomic: Label,
6816 ) -> Result<(), CompileError> {
6817 self.memory_op(
6818 target,
6819 memarg,
6820 true,
6821 2,
6822 need_check,
6823 imported_memories,
6824 offset,
6825 heap_access_oob,
6826 unaligned_atomic,
6827 |this, addr| {
6828 let mut temps = vec![];
6829 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6830 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6831 })?;
6832 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6833 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6834 })?;
6835 let dst =
6836 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6837 let reread = this.get_label();
6838
6839 this.emit_label(reread)?;
6840 this.assembler
6841 .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?;
6842 this.emit_binop_xor64(dst, loc, Location::GPR(tmp1))?;
6843 this.assembler.emit_stlxrh(
6844 Size::S64,
6845 Location::GPR(tmp2),
6846 Location::GPR(tmp1),
6847 Location::GPR(addr),
6848 )?;
6849 this.assembler
6850 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6851 this.assembler.emit_dmb()?;
6852
6853 if dst != ret {
6854 this.move_location(Size::S64, ret, dst)?;
6855 }
6856 for r in temps {
6857 this.release_gpr(r);
6858 }
6859 this.release_gpr(tmp1);
6860 this.release_gpr(tmp2);
6861 Ok(())
6862 },
6863 )
6864 }
6865
6866 fn i64_atomic_xor_32u(
6867 &mut self,
6868 loc: Location,
6869 target: Location,
6870 memarg: &MemArg,
6871 ret: Location,
6872 need_check: bool,
6873 imported_memories: bool,
6874 offset: i32,
6875 heap_access_oob: Label,
6876 unaligned_atomic: Label,
6877 ) -> Result<(), CompileError> {
6878 self.memory_op(
6879 target,
6880 memarg,
6881 true,
6882 4,
6883 need_check,
6884 imported_memories,
6885 offset,
6886 heap_access_oob,
6887 unaligned_atomic,
6888 |this, addr| {
6889 let mut temps = vec![];
6890 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6891 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6892 })?;
6893 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6894 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6895 })?;
6896 let dst =
6897 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6898 let reread = this.get_label();
6899
6900 this.emit_label(reread)?;
6901 this.assembler
6902 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
6903 this.emit_binop_xor64(dst, loc, Location::GPR(tmp1))?;
6904 this.assembler.emit_stlxr(
6905 Size::S32,
6906 Location::GPR(tmp2),
6907 Location::GPR(tmp1),
6908 Location::GPR(addr),
6909 )?;
6910 this.assembler
6911 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6912 this.assembler.emit_dmb()?;
6913
6914 if dst != ret {
6915 this.move_location(Size::S64, ret, dst)?;
6916 }
6917 for r in temps {
6918 this.release_gpr(r);
6919 }
6920 this.release_gpr(tmp1);
6921 this.release_gpr(tmp2);
6922 Ok(())
6923 },
6924 )
6925 }
6926
6927 fn i64_atomic_xchg(
6928 &mut self,
6929 loc: Location,
6930 target: Location,
6931 memarg: &MemArg,
6932 ret: Location,
6933 need_check: bool,
6934 imported_memories: bool,
6935 offset: i32,
6936 heap_access_oob: Label,
6937 unaligned_atomic: Label,
6938 ) -> Result<(), CompileError> {
6939 self.memory_op(
6940 target,
6941 memarg,
6942 true,
6943 8,
6944 need_check,
6945 imported_memories,
6946 offset,
6947 heap_access_oob,
6948 unaligned_atomic,
6949 |this, addr| {
6950 let mut temps = vec![];
6951 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
6952 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6953 })?;
6954 let dst =
6955 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6956 let org =
6957 this.location_to_reg(Size::S64, loc, &mut temps, ImmType::None, false, None)?;
6958 let reread = this.get_label();
6959
6960 this.emit_label(reread)?;
6961 this.assembler
6962 .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?;
6963 this.assembler.emit_stlxr(
6964 Size::S64,
6965 Location::GPR(tmp),
6966 org,
6967 Location::GPR(addr),
6968 )?;
6969 this.assembler
6970 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
6971 this.assembler.emit_dmb()?;
6972
6973 if dst != ret {
6974 this.move_location(Size::S64, ret, dst)?;
6975 }
6976 for r in temps {
6977 this.release_gpr(r);
6978 }
6979 this.release_gpr(tmp);
6980 Ok(())
6981 },
6982 )
6983 }
6984
6985 fn i64_atomic_xchg_8u(
6986 &mut self,
6987 loc: Location,
6988 target: Location,
6989 memarg: &MemArg,
6990 ret: Location,
6991 need_check: bool,
6992 imported_memories: bool,
6993 offset: i32,
6994 heap_access_oob: Label,
6995 unaligned_atomic: Label,
6996 ) -> Result<(), CompileError> {
6997 self.memory_op(
6998 target,
6999 memarg,
7000 true,
7001 1,
7002 need_check,
7003 imported_memories,
7004 offset,
7005 heap_access_oob,
7006 unaligned_atomic,
7007 |this, addr| {
7008 let mut temps = vec![];
7009 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
7010 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7011 })?;
7012 let dst =
7013 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7014 let org =
7015 this.location_to_reg(Size::S64, loc, &mut temps, ImmType::None, false, None)?;
7016 let reread = this.get_label();
7017
7018 this.emit_label(reread)?;
7019 this.assembler
7020 .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?;
7021 this.assembler.emit_stlxrb(
7022 Size::S64,
7023 Location::GPR(tmp),
7024 org,
7025 Location::GPR(addr),
7026 )?;
7027 this.assembler
7028 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7029 this.assembler.emit_dmb()?;
7030
7031 if dst != ret {
7032 this.move_location(Size::S64, ret, dst)?;
7033 }
7034 for r in temps {
7035 this.release_gpr(r);
7036 }
7037 this.release_gpr(tmp);
7038 Ok(())
7039 },
7040 )
7041 }
7042
7043 fn i64_atomic_xchg_16u(
7044 &mut self,
7045 loc: Location,
7046 target: Location,
7047 memarg: &MemArg,
7048 ret: Location,
7049 need_check: bool,
7050 imported_memories: bool,
7051 offset: i32,
7052 heap_access_oob: Label,
7053 unaligned_atomic: Label,
7054 ) -> Result<(), CompileError> {
7055 self.memory_op(
7056 target,
7057 memarg,
7058 true,
7059 2,
7060 need_check,
7061 imported_memories,
7062 offset,
7063 heap_access_oob,
7064 unaligned_atomic,
7065 |this, addr| {
7066 let mut temps = vec![];
7067 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
7068 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7069 })?;
7070 let dst =
7071 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7072 let org =
7073 this.location_to_reg(Size::S64, loc, &mut temps, ImmType::None, false, None)?;
7074 let reread = this.get_label();
7075
7076 this.emit_label(reread)?;
7077 this.assembler
7078 .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?;
7079 this.assembler.emit_stlxrh(
7080 Size::S64,
7081 Location::GPR(tmp),
7082 org,
7083 Location::GPR(addr),
7084 )?;
7085 this.assembler
7086 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7087 this.assembler.emit_dmb()?;
7088
7089 if dst != ret {
7090 this.move_location(Size::S64, ret, dst)?;
7091 }
7092 for r in temps {
7093 this.release_gpr(r);
7094 }
7095 this.release_gpr(tmp);
7096 Ok(())
7097 },
7098 )
7099 }
7100
7101 fn i64_atomic_xchg_32u(
7102 &mut self,
7103 loc: Location,
7104 target: Location,
7105 memarg: &MemArg,
7106 ret: Location,
7107 need_check: bool,
7108 imported_memories: bool,
7109 offset: i32,
7110 heap_access_oob: Label,
7111 unaligned_atomic: Label,
7112 ) -> Result<(), CompileError> {
7113 self.memory_op(
7114 target,
7115 memarg,
7116 true,
7117 4,
7118 need_check,
7119 imported_memories,
7120 offset,
7121 heap_access_oob,
7122 unaligned_atomic,
7123 |this, addr| {
7124 let mut temps = vec![];
7125 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
7126 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7127 })?;
7128 let dst =
7129 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7130 let org =
7131 this.location_to_reg(Size::S64, loc, &mut temps, ImmType::None, false, None)?;
7132 let reread = this.get_label();
7133
7134 this.emit_label(reread)?;
7135 this.assembler
7136 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
7137 this.assembler.emit_stlxr(
7138 Size::S32,
7139 Location::GPR(tmp),
7140 org,
7141 Location::GPR(addr),
7142 )?;
7143 this.assembler
7144 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7145 this.assembler.emit_dmb()?;
7146
7147 if dst != ret {
7148 this.move_location(Size::S64, ret, dst)?;
7149 }
7150 for r in temps {
7151 this.release_gpr(r);
7152 }
7153 this.release_gpr(tmp);
7154 Ok(())
7155 },
7156 )
7157 }
7158
7159 fn i64_atomic_cmpxchg(
7160 &mut self,
7161 new: Location,
7162 cmp: Location,
7163 target: Location,
7164 memarg: &MemArg,
7165 ret: Location,
7166 need_check: bool,
7167 imported_memories: bool,
7168 offset: i32,
7169 heap_access_oob: Label,
7170 unaligned_atomic: Label,
7171 ) -> Result<(), CompileError> {
7172 self.memory_op(
7173 target,
7174 memarg,
7175 true,
7176 8,
7177 need_check,
7178 imported_memories,
7179 offset,
7180 heap_access_oob,
7181 unaligned_atomic,
7182 |this, addr| {
7183 let mut temps = vec![];
7184 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
7185 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7186 })?;
7187 let dst =
7188 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7189 let org =
7190 this.location_to_reg(Size::S64, new, &mut temps, ImmType::None, false, None)?;
7191 let reread = this.get_label();
7192 let nosame = this.get_label();
7193
7194 this.emit_label(reread)?;
7195 this.assembler
7196 .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?;
7197 this.emit_relaxed_cmp(Size::S64, dst, cmp)?;
7198 this.assembler.emit_bcond_label(Condition::Ne, nosame)?;
7199 this.assembler.emit_stlxr(
7200 Size::S64,
7201 Location::GPR(tmp),
7202 org,
7203 Location::GPR(addr),
7204 )?;
7205 this.assembler
7206 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7207 this.assembler.emit_dmb()?;
7208
7209 this.emit_label(nosame)?;
7210 if dst != ret {
7211 this.move_location(Size::S64, ret, dst)?;
7212 }
7213 for r in temps {
7214 this.release_gpr(r);
7215 }
7216 this.release_gpr(tmp);
7217 Ok(())
7218 },
7219 )
7220 }
7221
7222 fn i64_atomic_cmpxchg_8u(
7223 &mut self,
7224 new: Location,
7225 cmp: Location,
7226 target: Location,
7227 memarg: &MemArg,
7228 ret: Location,
7229 need_check: bool,
7230 imported_memories: bool,
7231 offset: i32,
7232 heap_access_oob: Label,
7233 unaligned_atomic: Label,
7234 ) -> Result<(), CompileError> {
7235 self.memory_op(
7236 target,
7237 memarg,
7238 true,
7239 1,
7240 need_check,
7241 imported_memories,
7242 offset,
7243 heap_access_oob,
7244 unaligned_atomic,
7245 |this, addr| {
7246 let mut temps = vec![];
7247 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
7248 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7249 })?;
7250 let dst =
7251 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7252 let org =
7253 this.location_to_reg(Size::S64, new, &mut temps, ImmType::None, false, None)?;
7254 let reread = this.get_label();
7255 let nosame = this.get_label();
7256
7257 this.emit_label(reread)?;
7258 this.assembler
7259 .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?;
7260 this.emit_relaxed_cmp(Size::S64, dst, cmp)?;
7261 this.assembler.emit_bcond_label(Condition::Ne, nosame)?;
7262 this.assembler.emit_stlxrb(
7263 Size::S64,
7264 Location::GPR(tmp),
7265 org,
7266 Location::GPR(addr),
7267 )?;
7268 this.assembler
7269 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7270 this.assembler.emit_dmb()?;
7271
7272 this.emit_label(nosame)?;
7273 if dst != ret {
7274 this.move_location(Size::S64, ret, dst)?;
7275 }
7276 for r in temps {
7277 this.release_gpr(r);
7278 }
7279 this.release_gpr(tmp);
7280 Ok(())
7281 },
7282 )
7283 }
7284
7285 fn i64_atomic_cmpxchg_16u(
7286 &mut self,
7287 new: Location,
7288 cmp: Location,
7289 target: Location,
7290 memarg: &MemArg,
7291 ret: Location,
7292 need_check: bool,
7293 imported_memories: bool,
7294 offset: i32,
7295 heap_access_oob: Label,
7296 unaligned_atomic: Label,
7297 ) -> Result<(), CompileError> {
7298 self.memory_op(
7299 target,
7300 memarg,
7301 true,
7302 2,
7303 need_check,
7304 imported_memories,
7305 offset,
7306 heap_access_oob,
7307 unaligned_atomic,
7308 |this, addr| {
7309 let mut temps = vec![];
7310 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
7311 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7312 })?;
7313 let dst =
7314 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7315 let org =
7316 this.location_to_reg(Size::S64, new, &mut temps, ImmType::None, false, None)?;
7317 let reread = this.get_label();
7318 let nosame = this.get_label();
7319
7320 this.emit_label(reread)?;
7321 this.assembler
7322 .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?;
7323 this.emit_relaxed_cmp(Size::S64, dst, cmp)?;
7324 this.assembler.emit_bcond_label(Condition::Ne, nosame)?;
7325 this.assembler.emit_stlxrh(
7326 Size::S64,
7327 Location::GPR(tmp),
7328 org,
7329 Location::GPR(addr),
7330 )?;
7331 this.assembler
7332 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7333 this.assembler.emit_dmb()?;
7334
7335 this.emit_label(nosame)?;
7336 if dst != ret {
7337 this.move_location(Size::S64, ret, dst)?;
7338 }
7339 for r in temps {
7340 this.release_gpr(r);
7341 }
7342 this.release_gpr(tmp);
7343 Ok(())
7344 },
7345 )
7346 }
7347
7348 fn i64_atomic_cmpxchg_32u(
7349 &mut self,
7350 new: Location,
7351 cmp: Location,
7352 target: Location,
7353 memarg: &MemArg,
7354 ret: Location,
7355 need_check: bool,
7356 imported_memories: bool,
7357 offset: i32,
7358 heap_access_oob: Label,
7359 unaligned_atomic: Label,
7360 ) -> Result<(), CompileError> {
7361 self.memory_op(
7362 target,
7363 memarg,
7364 true,
7365 4,
7366 need_check,
7367 imported_memories,
7368 offset,
7369 heap_access_oob,
7370 unaligned_atomic,
7371 |this, addr| {
7372 let mut temps = vec![];
7373 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
7374 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7375 })?;
7376 let dst =
7377 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7378 let org =
7379 this.location_to_reg(Size::S64, new, &mut temps, ImmType::None, false, None)?;
7380 let reread = this.get_label();
7381 let nosame = this.get_label();
7382
7383 this.emit_label(reread)?;
7384 this.assembler
7385 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
7386 this.emit_relaxed_cmp(Size::S64, dst, cmp)?;
7387 this.assembler.emit_bcond_label(Condition::Ne, nosame)?;
7388 this.assembler.emit_stlxr(
7389 Size::S32,
7390 Location::GPR(tmp),
7391 org,
7392 Location::GPR(addr),
7393 )?;
7394 this.assembler
7395 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7396 this.assembler.emit_dmb()?;
7397
7398 this.emit_label(nosame)?;
7399 if dst != ret {
7400 this.move_location(Size::S64, ret, dst)?;
7401 }
7402 for r in temps {
7403 this.release_gpr(r);
7404 }
7405 this.release_gpr(tmp);
7406 Ok(())
7407 },
7408 )
7409 }
7410
7411 fn f32_load(
7412 &mut self,
7413 addr: Location,
7414 memarg: &MemArg,
7415 ret: Location,
7416 need_check: bool,
7417 imported_memories: bool,
7418 offset: i32,
7419 heap_access_oob: Label,
7420 unaligned_atomic: Label,
7421 ) -> Result<(), CompileError> {
7422 self.memory_op(
7423 addr,
7424 memarg,
7425 false,
7426 4,
7427 need_check,
7428 imported_memories,
7429 offset,
7430 heap_access_oob,
7431 unaligned_atomic,
7432 |this, addr| this.emit_relaxed_ldr32(Size::S32, ret, Location::Memory(addr, 0)),
7433 )
7434 }
7435
7436 fn f32_save(
7437 &mut self,
7438 target_value: Location,
7439 memarg: &MemArg,
7440 target_addr: Location,
7441 canonicalize: bool,
7442 need_check: bool,
7443 imported_memories: bool,
7444 offset: i32,
7445 heap_access_oob: Label,
7446 unaligned_atomic: Label,
7447 ) -> Result<(), CompileError> {
7448 self.memory_op(
7449 target_addr,
7450 memarg,
7451 false,
7452 4,
7453 need_check,
7454 imported_memories,
7455 offset,
7456 heap_access_oob,
7457 unaligned_atomic,
7458 |this, addr| {
7459 if !canonicalize {
7460 this.emit_relaxed_str32(target_value, Location::Memory(addr, 0))
7461 } else {
7462 this.canonicalize_nan(Size::S32, target_value, Location::Memory(addr, 0))
7463 }
7464 },
7465 )
7466 }
7467
7468 fn f64_load(
7469 &mut self,
7470 addr: Location,
7471 memarg: &MemArg,
7472 ret: Location,
7473 need_check: bool,
7474 imported_memories: bool,
7475 offset: i32,
7476 heap_access_oob: Label,
7477 unaligned_atomic: Label,
7478 ) -> Result<(), CompileError> {
7479 self.memory_op(
7480 addr,
7481 memarg,
7482 false,
7483 8,
7484 need_check,
7485 imported_memories,
7486 offset,
7487 heap_access_oob,
7488 unaligned_atomic,
7489 |this, addr| this.emit_relaxed_ldr64(Size::S64, ret, Location::Memory(addr, 0)),
7490 )
7491 }
7492
7493 fn f64_save(
7494 &mut self,
7495 target_value: Location,
7496 memarg: &MemArg,
7497 target_addr: Location,
7498 canonicalize: bool,
7499 need_check: bool,
7500 imported_memories: bool,
7501 offset: i32,
7502 heap_access_oob: Label,
7503 unaligned_atomic: Label,
7504 ) -> Result<(), CompileError> {
7505 self.memory_op(
7506 target_addr,
7507 memarg,
7508 false,
7509 8,
7510 need_check,
7511 imported_memories,
7512 offset,
7513 heap_access_oob,
7514 unaligned_atomic,
7515 |this, addr| {
7516 if !canonicalize {
7517 this.emit_relaxed_str64(target_value, Location::Memory(addr, 0))
7518 } else {
7519 this.canonicalize_nan(Size::S64, target_value, Location::Memory(addr, 0))
7520 }
7521 },
7522 )
7523 }
7524
7525 fn convert_f64_i64(
7526 &mut self,
7527 loc: Location,
7528 signed: bool,
7529 ret: Location,
7530 ) -> Result<(), CompileError> {
7531 let mut gprs = vec![];
7532 let mut neons = vec![];
7533 let src = self.location_to_reg(Size::S64, loc, &mut gprs, ImmType::NoneXzr, true, None)?;
7534 let dest = self.location_to_neon(Size::S64, ret, &mut neons, ImmType::None, false)?;
7535 if signed {
7536 self.assembler.emit_scvtf(Size::S64, src, Size::S64, dest)?;
7537 } else {
7538 self.assembler.emit_ucvtf(Size::S64, src, Size::S64, dest)?;
7539 }
7540 if ret != dest {
7541 self.move_location(Size::S64, dest, ret)?;
7542 }
7543 for r in gprs {
7544 self.release_gpr(r);
7545 }
7546 for r in neons {
7547 self.release_simd(r);
7548 }
7549 Ok(())
7550 }
7551
7552 fn convert_f64_i32(
7553 &mut self,
7554 loc: Location,
7555 signed: bool,
7556 ret: Location,
7557 ) -> Result<(), CompileError> {
7558 let mut gprs = vec![];
7559 let mut neons = vec![];
7560 let src = self.location_to_reg(Size::S32, loc, &mut gprs, ImmType::NoneXzr, true, None)?;
7561 let dest = self.location_to_neon(Size::S64, ret, &mut neons, ImmType::None, false)?;
7562 if signed {
7563 self.assembler.emit_scvtf(Size::S32, src, Size::S64, dest)?;
7564 } else {
7565 self.assembler.emit_ucvtf(Size::S32, src, Size::S64, dest)?;
7566 }
7567 if ret != dest {
7568 self.move_location(Size::S64, dest, ret)?;
7569 }
7570 for r in gprs {
7571 self.release_gpr(r);
7572 }
7573 for r in neons {
7574 self.release_simd(r);
7575 }
7576 Ok(())
7577 }
7578
7579 fn convert_f32_i64(
7580 &mut self,
7581 loc: Location,
7582 signed: bool,
7583 ret: Location,
7584 ) -> Result<(), CompileError> {
7585 let mut gprs = vec![];
7586 let mut neons = vec![];
7587 let src = self.location_to_reg(Size::S64, loc, &mut gprs, ImmType::NoneXzr, true, None)?;
7588 let dest = self.location_to_neon(Size::S32, ret, &mut neons, ImmType::None, false)?;
7589 if signed {
7590 self.assembler.emit_scvtf(Size::S64, src, Size::S32, dest)?;
7591 } else {
7592 self.assembler.emit_ucvtf(Size::S64, src, Size::S32, dest)?;
7593 }
7594 if ret != dest {
7595 self.move_location(Size::S32, dest, ret)?;
7596 }
7597 for r in gprs {
7598 self.release_gpr(r);
7599 }
7600 for r in neons {
7601 self.release_simd(r);
7602 }
7603 Ok(())
7604 }
7605
7606 fn convert_f32_i32(
7607 &mut self,
7608 loc: Location,
7609 signed: bool,
7610 ret: Location,
7611 ) -> Result<(), CompileError> {
7612 let mut gprs = vec![];
7613 let mut neons = vec![];
7614 let src = self.location_to_reg(Size::S32, loc, &mut gprs, ImmType::NoneXzr, true, None)?;
7615 let dest = self.location_to_neon(Size::S32, ret, &mut neons, ImmType::None, false)?;
7616 if signed {
7617 self.assembler.emit_scvtf(Size::S32, src, Size::S32, dest)?;
7618 } else {
7619 self.assembler.emit_ucvtf(Size::S32, src, Size::S32, dest)?;
7620 }
7621 if ret != dest {
7622 self.move_location(Size::S32, dest, ret)?;
7623 }
7624 for r in gprs {
7625 self.release_gpr(r);
7626 }
7627 for r in neons {
7628 self.release_simd(r);
7629 }
7630 Ok(())
7631 }
7632
7633 fn convert_i64_f64(
7634 &mut self,
7635 loc: Location,
7636 ret: Location,
7637 signed: bool,
7638 sat: bool,
7639 ) -> Result<(), CompileError> {
7640 let mut gprs = vec![];
7641 let mut neons = vec![];
7642 let src = self.location_to_neon(Size::S64, loc, &mut neons, ImmType::None, true)?;
7643 let dest = self.location_to_reg(Size::S64, ret, &mut gprs, ImmType::None, false, None)?;
7644 let old_fpcr = if !sat {
7645 self.reset_exception_fpsr()?;
7646 self.set_trap_enabled(&mut gprs)?
7647 } else {
7648 GPR::XzrSp
7649 };
7650 if signed {
7651 self.assembler
7652 .emit_fcvtzs(Size::S64, src, Size::S64, dest)?;
7653 } else {
7654 self.assembler
7655 .emit_fcvtzu(Size::S64, src, Size::S64, dest)?;
7656 }
7657 if !sat {
7658 self.trap_float_conversion_errors(old_fpcr, Size::S64, src, &mut gprs)?;
7659 }
7660 if ret != dest {
7661 self.move_location(Size::S64, dest, ret)?;
7662 }
7663 for r in gprs {
7664 self.release_gpr(r);
7665 }
7666 for r in neons {
7667 self.release_simd(r);
7668 }
7669 Ok(())
7670 }
7671
7672 fn convert_i32_f64(
7673 &mut self,
7674 loc: Location,
7675 ret: Location,
7676 signed: bool,
7677 sat: bool,
7678 ) -> Result<(), CompileError> {
7679 let mut gprs = vec![];
7680 let mut neons = vec![];
7681 let src = self.location_to_neon(Size::S64, loc, &mut neons, ImmType::None, true)?;
7682 let dest = self.location_to_reg(Size::S32, ret, &mut gprs, ImmType::None, false, None)?;
7683 let old_fpcr = if !sat {
7684 self.reset_exception_fpsr()?;
7685 self.set_trap_enabled(&mut gprs)?
7686 } else {
7687 GPR::XzrSp
7688 };
7689 if signed {
7690 self.assembler
7691 .emit_fcvtzs(Size::S64, src, Size::S32, dest)?;
7692 } else {
7693 self.assembler
7694 .emit_fcvtzu(Size::S64, src, Size::S32, dest)?;
7695 }
7696 if !sat {
7697 self.trap_float_conversion_errors(old_fpcr, Size::S64, src, &mut gprs)?;
7698 }
7699 if ret != dest {
7700 self.move_location(Size::S32, dest, ret)?;
7701 }
7702 for r in gprs {
7703 self.release_gpr(r);
7704 }
7705 for r in neons {
7706 self.release_simd(r);
7707 }
7708 Ok(())
7709 }
7710
7711 fn convert_i64_f32(
7712 &mut self,
7713 loc: Location,
7714 ret: Location,
7715 signed: bool,
7716 sat: bool,
7717 ) -> Result<(), CompileError> {
7718 let mut gprs = vec![];
7719 let mut neons = vec![];
7720 let src = self.location_to_neon(Size::S32, loc, &mut neons, ImmType::None, true)?;
7721 let dest = self.location_to_reg(Size::S64, ret, &mut gprs, ImmType::None, false, None)?;
7722 let old_fpcr = if !sat {
7723 self.reset_exception_fpsr()?;
7724 self.set_trap_enabled(&mut gprs)?
7725 } else {
7726 GPR::XzrSp
7727 };
7728 if signed {
7729 self.assembler
7730 .emit_fcvtzs(Size::S32, src, Size::S64, dest)?;
7731 } else {
7732 self.assembler
7733 .emit_fcvtzu(Size::S32, src, Size::S64, dest)?;
7734 }
7735 if !sat {
7736 self.trap_float_conversion_errors(old_fpcr, Size::S32, src, &mut gprs)?;
7737 }
7738 if ret != dest {
7739 self.move_location(Size::S64, dest, ret)?;
7740 }
7741 for r in gprs {
7742 self.release_gpr(r);
7743 }
7744 for r in neons {
7745 self.release_simd(r);
7746 }
7747 Ok(())
7748 }
7749
7750 fn convert_i32_f32(
7751 &mut self,
7752 loc: Location,
7753 ret: Location,
7754 signed: bool,
7755 sat: bool,
7756 ) -> Result<(), CompileError> {
7757 let mut gprs = vec![];
7758 let mut neons = vec![];
7759 let src = self.location_to_neon(Size::S32, loc, &mut neons, ImmType::None, true)?;
7760 let dest = self.location_to_reg(Size::S32, ret, &mut gprs, ImmType::None, false, None)?;
7761 let old_fpcr = if !sat {
7762 self.reset_exception_fpsr()?;
7763 self.set_trap_enabled(&mut gprs)?
7764 } else {
7765 GPR::XzrSp
7766 };
7767 if signed {
7768 self.assembler
7769 .emit_fcvtzs(Size::S32, src, Size::S32, dest)?;
7770 } else {
7771 self.assembler
7772 .emit_fcvtzu(Size::S32, src, Size::S32, dest)?;
7773 }
7774 if !sat {
7775 self.trap_float_conversion_errors(old_fpcr, Size::S32, src, &mut gprs)?;
7776 }
7777 if ret != dest {
7778 self.move_location(Size::S32, dest, ret)?;
7779 }
7780 for r in gprs {
7781 self.release_gpr(r);
7782 }
7783 for r in neons {
7784 self.release_simd(r);
7785 }
7786 Ok(())
7787 }
7788
7789 fn convert_f64_f32(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7790 self.emit_relaxed_binop_neon(Assembler::emit_fcvt, Size::S32, loc, ret, true)
7791 }
7792
7793 fn convert_f32_f64(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7794 self.emit_relaxed_binop_neon(Assembler::emit_fcvt, Size::S64, loc, ret, true)
7795 }
7796
7797 fn f64_neg(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7798 self.emit_relaxed_binop_neon(Assembler::emit_fneg, Size::S64, loc, ret, true)
7799 }
7800
7801 fn f64_abs(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7802 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
7803 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7804 })?;
7805
7806 self.move_location(Size::S64, loc, Location::GPR(tmp))?;
7807 self.assembler.emit_and(
7808 Size::S64,
7809 Location::GPR(tmp),
7810 Location::Imm64(0x7fffffffffffffffu64),
7811 Location::GPR(tmp),
7812 )?;
7813 self.move_location(Size::S64, Location::GPR(tmp), ret)?;
7814
7815 self.release_gpr(tmp);
7816 Ok(())
7817 }
7818
7819 fn emit_i64_copysign(&mut self, tmp1: GPR, tmp2: GPR) -> Result<(), CompileError> {
7820 self.assembler.emit_and(
7821 Size::S64,
7822 Location::GPR(tmp1),
7823 Location::Imm64(0x7fffffffffffffffu64),
7824 Location::GPR(tmp1),
7825 )?;
7826
7827 self.assembler.emit_and(
7828 Size::S64,
7829 Location::GPR(tmp2),
7830 Location::Imm64(0x8000000000000000u64),
7831 Location::GPR(tmp2),
7832 )?;
7833
7834 self.assembler.emit_or(
7835 Size::S64,
7836 Location::GPR(tmp1),
7837 Location::GPR(tmp2),
7838 Location::GPR(tmp1),
7839 )
7840 }
7841
7842 fn f64_sqrt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7843 self.emit_relaxed_binop_neon(Assembler::emit_fsqrt, Size::S64, loc, ret, true)
7844 }
7845
7846 fn f64_trunc(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7847 self.emit_relaxed_binop_neon(Assembler::emit_frintz, Size::S64, loc, ret, true)
7848 }
7849
7850 fn f64_ceil(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7851 self.emit_relaxed_binop_neon(Assembler::emit_frintp, Size::S64, loc, ret, true)
7852 }
7853
7854 fn f64_floor(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7855 self.emit_relaxed_binop_neon(Assembler::emit_frintm, Size::S64, loc, ret, true)
7856 }
7857
7858 fn f64_nearest(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7859 self.emit_relaxed_binop_neon(Assembler::emit_frintn, Size::S64, loc, ret, true)
7860 }
7861
7862 fn f64_cmp_ge(
7863 &mut self,
7864 loc_a: Location,
7865 loc_b: Location,
7866 ret: Location,
7867 ) -> Result<(), CompileError> {
7868 let mut temps = vec![];
7869 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7870 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S64, loc_b, loc_a, false)?;
7871 self.assembler.emit_cset(Size::S32, dest, Condition::Ls)?;
7872 if ret != dest {
7873 self.move_location(Size::S32, dest, ret)?;
7874 }
7875 for r in temps {
7876 self.release_gpr(r);
7877 }
7878 Ok(())
7879 }
7880
7881 fn f64_cmp_gt(
7882 &mut self,
7883 loc_a: Location,
7884 loc_b: Location,
7885 ret: Location,
7886 ) -> Result<(), CompileError> {
7887 let mut temps = vec![];
7888 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7889 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S64, loc_b, loc_a, false)?;
7890 self.assembler.emit_cset(Size::S32, dest, Condition::Cc)?;
7891 if ret != dest {
7892 self.move_location(Size::S32, dest, ret)?;
7893 }
7894 for r in temps {
7895 self.release_gpr(r);
7896 }
7897 Ok(())
7898 }
7899
7900 fn f64_cmp_le(
7901 &mut self,
7902 loc_a: Location,
7903 loc_b: Location,
7904 ret: Location,
7905 ) -> Result<(), CompileError> {
7906 let mut temps = vec![];
7907 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7908 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S64, loc_a, loc_b, false)?;
7909 self.assembler.emit_cset(Size::S32, dest, Condition::Ls)?;
7910 if ret != dest {
7911 self.move_location(Size::S32, dest, ret)?;
7912 }
7913 for r in temps {
7914 self.release_gpr(r);
7915 }
7916 Ok(())
7917 }
7918
7919 fn f64_cmp_lt(
7920 &mut self,
7921 loc_a: Location,
7922 loc_b: Location,
7923 ret: Location,
7924 ) -> Result<(), CompileError> {
7925 let mut temps = vec![];
7926 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7927 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S64, loc_a, loc_b, false)?;
7928 self.assembler.emit_cset(Size::S32, dest, Condition::Cc)?;
7929 if ret != dest {
7930 self.move_location(Size::S32, dest, ret)?;
7931 }
7932 for r in temps {
7933 self.release_gpr(r);
7934 }
7935 Ok(())
7936 }
7937
7938 fn f64_cmp_ne(
7939 &mut self,
7940 loc_a: Location,
7941 loc_b: Location,
7942 ret: Location,
7943 ) -> Result<(), CompileError> {
7944 let mut temps = vec![];
7945 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7946 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S64, loc_a, loc_b, false)?;
7947 self.assembler.emit_cset(Size::S32, dest, Condition::Ne)?;
7948 if ret != dest {
7949 self.move_location(Size::S32, dest, ret)?;
7950 }
7951 for r in temps {
7952 self.release_gpr(r);
7953 }
7954 Ok(())
7955 }
7956
7957 fn f64_cmp_eq(
7958 &mut self,
7959 loc_a: Location,
7960 loc_b: Location,
7961 ret: Location,
7962 ) -> Result<(), CompileError> {
7963 let mut temps = vec![];
7964 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7965 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S64, loc_a, loc_b, false)?;
7966 self.assembler.emit_cset(Size::S32, dest, Condition::Eq)?;
7967 if ret != dest {
7968 self.move_location(Size::S32, dest, ret)?;
7969 }
7970 for r in temps {
7971 self.release_gpr(r);
7972 }
7973 Ok(())
7974 }
7975
7976 fn f64_min(
7977 &mut self,
7978 loc_a: Location,
7979 loc_b: Location,
7980 ret: Location,
7981 ) -> Result<(), CompileError> {
7982 let mut temps = vec![];
7983 let old_fpcr = self.set_default_nan(&mut temps)?;
7984 self.emit_relaxed_binop3_neon(
7985 Assembler::emit_fmin,
7986 Size::S64,
7987 loc_a,
7988 loc_b,
7989 ret,
7990 ImmType::None,
7991 )?;
7992 self.restore_fpcr(old_fpcr)?;
7993 for r in temps {
7994 self.release_gpr(r);
7995 }
7996 Ok(())
7997 }
7998
7999 fn f64_max(
8000 &mut self,
8001 loc_a: Location,
8002 loc_b: Location,
8003 ret: Location,
8004 ) -> Result<(), CompileError> {
8005 let mut temps = vec![];
8006 let old_fpcr = self.set_default_nan(&mut temps)?;
8007 self.emit_relaxed_binop3_neon(
8008 Assembler::emit_fmax,
8009 Size::S64,
8010 loc_a,
8011 loc_b,
8012 ret,
8013 ImmType::None,
8014 )?;
8015 self.restore_fpcr(old_fpcr)?;
8016 for r in temps {
8017 self.release_gpr(r);
8018 }
8019 Ok(())
8020 }
8021
8022 fn f64_add(
8023 &mut self,
8024 loc_a: Location,
8025 loc_b: Location,
8026 ret: Location,
8027 ) -> Result<(), CompileError> {
8028 self.emit_relaxed_binop3_neon(
8029 Assembler::emit_fadd,
8030 Size::S64,
8031 loc_a,
8032 loc_b,
8033 ret,
8034 ImmType::None,
8035 )
8036 }
8037
8038 fn f64_sub(
8039 &mut self,
8040 loc_a: Location,
8041 loc_b: Location,
8042 ret: Location,
8043 ) -> Result<(), CompileError> {
8044 self.emit_relaxed_binop3_neon(
8045 Assembler::emit_fsub,
8046 Size::S64,
8047 loc_a,
8048 loc_b,
8049 ret,
8050 ImmType::None,
8051 )
8052 }
8053
8054 fn f64_mul(
8055 &mut self,
8056 loc_a: Location,
8057 loc_b: Location,
8058 ret: Location,
8059 ) -> Result<(), CompileError> {
8060 self.emit_relaxed_binop3_neon(
8061 Assembler::emit_fmul,
8062 Size::S64,
8063 loc_a,
8064 loc_b,
8065 ret,
8066 ImmType::None,
8067 )
8068 }
8069
8070 fn f64_div(
8071 &mut self,
8072 loc_a: Location,
8073 loc_b: Location,
8074 ret: Location,
8075 ) -> Result<(), CompileError> {
8076 self.emit_relaxed_binop3_neon(
8077 Assembler::emit_fdiv,
8078 Size::S64,
8079 loc_a,
8080 loc_b,
8081 ret,
8082 ImmType::None,
8083 )
8084 }
8085
8086 fn f32_neg(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
8087 self.emit_relaxed_binop_neon(Assembler::emit_fneg, Size::S32, loc, ret, true)
8088 }
8089
8090 fn f32_abs(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
8091 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
8092 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
8093 })?;
8094 self.move_location(Size::S32, loc, Location::GPR(tmp))?;
8095 self.assembler.emit_and(
8096 Size::S32,
8097 Location::GPR(tmp),
8098 Location::Imm32(0x7fffffffu32),
8099 Location::GPR(tmp),
8100 )?;
8101 self.move_location(Size::S32, Location::GPR(tmp), ret)?;
8102 self.release_gpr(tmp);
8103 Ok(())
8104 }
8105
8106 fn emit_i32_copysign(&mut self, tmp1: GPR, tmp2: GPR) -> Result<(), CompileError> {
8107 self.assembler.emit_and(
8108 Size::S32,
8109 Location::GPR(tmp1),
8110 Location::Imm32(0x7fffffffu32),
8111 Location::GPR(tmp1),
8112 )?;
8113 self.assembler.emit_and(
8114 Size::S32,
8115 Location::GPR(tmp2),
8116 Location::Imm32(0x80000000u32),
8117 Location::GPR(tmp2),
8118 )?;
8119 self.assembler.emit_or(
8120 Size::S32,
8121 Location::GPR(tmp1),
8122 Location::GPR(tmp2),
8123 Location::GPR(tmp1),
8124 )
8125 }
8126
8127 fn f32_sqrt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
8128 self.emit_relaxed_binop_neon(Assembler::emit_fsqrt, Size::S32, loc, ret, true)
8129 }
8130
8131 fn f32_trunc(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
8132 self.emit_relaxed_binop_neon(Assembler::emit_frintz, Size::S32, loc, ret, true)
8133 }
8134
8135 fn f32_ceil(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
8136 self.emit_relaxed_binop_neon(Assembler::emit_frintp, Size::S32, loc, ret, true)
8137 }
8138
8139 fn f32_floor(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
8140 self.emit_relaxed_binop_neon(Assembler::emit_frintm, Size::S32, loc, ret, true)
8141 }
8142
8143 fn f32_nearest(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
8144 self.emit_relaxed_binop_neon(Assembler::emit_frintn, Size::S32, loc, ret, true)
8145 }
8146
8147 fn f32_cmp_ge(
8148 &mut self,
8149 loc_a: Location,
8150 loc_b: Location,
8151 ret: Location,
8152 ) -> Result<(), CompileError> {
8153 let mut temps = vec![];
8154 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
8155 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S32, loc_b, loc_a, false)?;
8156 self.assembler.emit_cset(Size::S32, dest, Condition::Ls)?;
8157 if ret != dest {
8158 self.move_location(Size::S32, dest, ret)?;
8159 }
8160 for r in temps {
8161 self.release_gpr(r);
8162 }
8163 Ok(())
8164 }
8165
8166 fn f32_cmp_gt(
8167 &mut self,
8168 loc_a: Location,
8169 loc_b: Location,
8170 ret: Location,
8171 ) -> Result<(), CompileError> {
8172 let mut temps = vec![];
8173 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
8174 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S32, loc_b, loc_a, false)?;
8175 self.assembler.emit_cset(Size::S32, dest, Condition::Cc)?;
8176 if ret != dest {
8177 self.move_location(Size::S32, dest, ret)?;
8178 }
8179 for r in temps {
8180 self.release_gpr(r);
8181 }
8182 Ok(())
8183 }
8184
8185 fn f32_cmp_le(
8186 &mut self,
8187 loc_a: Location,
8188 loc_b: Location,
8189 ret: Location,
8190 ) -> Result<(), CompileError> {
8191 let mut temps = vec![];
8192 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
8193 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S32, loc_a, loc_b, false)?;
8194 self.assembler.emit_cset(Size::S32, dest, Condition::Ls)?;
8195 if ret != dest {
8196 self.move_location(Size::S32, dest, ret)?;
8197 }
8198 for r in temps {
8199 self.release_gpr(r);
8200 }
8201 Ok(())
8202 }
8203
8204 fn f32_cmp_lt(
8205 &mut self,
8206 loc_a: Location,
8207 loc_b: Location,
8208 ret: Location,
8209 ) -> Result<(), CompileError> {
8210 let mut temps = vec![];
8211 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
8212 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S32, loc_a, loc_b, false)?;
8213 self.assembler.emit_cset(Size::S32, dest, Condition::Cc)?;
8214 if ret != dest {
8215 self.move_location(Size::S32, dest, ret)?;
8216 }
8217 for r in temps {
8218 self.release_gpr(r);
8219 }
8220 Ok(())
8221 }
8222
8223 fn f32_cmp_ne(
8224 &mut self,
8225 loc_a: Location,
8226 loc_b: Location,
8227 ret: Location,
8228 ) -> Result<(), CompileError> {
8229 let mut temps = vec![];
8230 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
8231 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S32, loc_a, loc_b, false)?;
8232 self.assembler.emit_cset(Size::S32, dest, Condition::Ne)?;
8233 if ret != dest {
8234 self.move_location(Size::S32, dest, ret)?;
8235 }
8236 for r in temps {
8237 self.release_gpr(r);
8238 }
8239 Ok(())
8240 }
8241
8242 fn f32_cmp_eq(
8243 &mut self,
8244 loc_a: Location,
8245 loc_b: Location,
8246 ret: Location,
8247 ) -> Result<(), CompileError> {
8248 let mut temps = vec![];
8249 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
8250 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S32, loc_a, loc_b, false)?;
8251 self.assembler.emit_cset(Size::S32, dest, Condition::Eq)?;
8252 if ret != dest {
8253 self.move_location(Size::S32, dest, ret)?;
8254 }
8255 for r in temps {
8256 self.release_gpr(r);
8257 }
8258 Ok(())
8259 }
8260
8261 fn f32_min(
8262 &mut self,
8263 loc_a: Location,
8264 loc_b: Location,
8265 ret: Location,
8266 ) -> Result<(), CompileError> {
8267 let mut temps = vec![];
8268 let old_fpcr = self.set_default_nan(&mut temps)?;
8269 self.emit_relaxed_binop3_neon(
8270 Assembler::emit_fmin,
8271 Size::S32,
8272 loc_a,
8273 loc_b,
8274 ret,
8275 ImmType::None,
8276 )?;
8277 self.restore_fpcr(old_fpcr)?;
8278 for r in temps {
8279 self.release_gpr(r);
8280 }
8281 Ok(())
8282 }
8283
8284 fn f32_max(
8285 &mut self,
8286 loc_a: Location,
8287 loc_b: Location,
8288 ret: Location,
8289 ) -> Result<(), CompileError> {
8290 let mut temps = vec![];
8291 let old_fpcr = self.set_default_nan(&mut temps)?;
8292 self.emit_relaxed_binop3_neon(
8293 Assembler::emit_fmax,
8294 Size::S32,
8295 loc_a,
8296 loc_b,
8297 ret,
8298 ImmType::None,
8299 )?;
8300 self.restore_fpcr(old_fpcr)?;
8301 for r in temps {
8302 self.release_gpr(r);
8303 }
8304 Ok(())
8305 }
8306
8307 fn f32_add(
8308 &mut self,
8309 loc_a: Location,
8310 loc_b: Location,
8311 ret: Location,
8312 ) -> Result<(), CompileError> {
8313 self.emit_relaxed_binop3_neon(
8314 Assembler::emit_fadd,
8315 Size::S32,
8316 loc_a,
8317 loc_b,
8318 ret,
8319 ImmType::None,
8320 )
8321 }
8322
8323 fn f32_sub(
8324 &mut self,
8325 loc_a: Location,
8326 loc_b: Location,
8327 ret: Location,
8328 ) -> Result<(), CompileError> {
8329 self.emit_relaxed_binop3_neon(
8330 Assembler::emit_fsub,
8331 Size::S32,
8332 loc_a,
8333 loc_b,
8334 ret,
8335 ImmType::None,
8336 )
8337 }
8338
8339 fn f32_mul(
8340 &mut self,
8341 loc_a: Location,
8342 loc_b: Location,
8343 ret: Location,
8344 ) -> Result<(), CompileError> {
8345 self.emit_relaxed_binop3_neon(
8346 Assembler::emit_fmul,
8347 Size::S32,
8348 loc_a,
8349 loc_b,
8350 ret,
8351 ImmType::None,
8352 )
8353 }
8354
8355 fn f32_div(
8356 &mut self,
8357 loc_a: Location,
8358 loc_b: Location,
8359 ret: Location,
8360 ) -> Result<(), CompileError> {
8361 self.emit_relaxed_binop3_neon(
8362 Assembler::emit_fdiv,
8363 Size::S32,
8364 loc_a,
8365 loc_b,
8366 ret,
8367 ImmType::None,
8368 )
8369 }
8370
8371 fn gen_std_trampoline(
8372 &self,
8373 sig: &FunctionType,
8374 calling_convention: CallingConvention,
8375 progress_callback: Option<&CompilationProgressCallback>,
8376 ) -> Result<FunctionBody, CompileError> {
8377 gen_std_trampoline_arm64(sig, calling_convention, progress_callback)
8378 }
8379 fn gen_std_dynamic_import_trampoline(
8382 &self,
8383 vmoffsets: &VMOffsets,
8384 sig: &FunctionType,
8385 calling_convention: CallingConvention,
8386 progress_callback: Option<&CompilationProgressCallback>,
8387 ) -> Result<FunctionBody, CompileError> {
8388 gen_std_dynamic_import_trampoline_arm64(
8389 vmoffsets,
8390 sig,
8391 calling_convention,
8392 progress_callback,
8393 )
8394 }
8395 fn gen_import_call_trampoline(
8398 &self,
8399 vmoffsets: &VMOffsets,
8400 index: FunctionIndex,
8401 sig: &FunctionType,
8402 calling_convention: CallingConvention,
8403 progress_callback: Option<&CompilationProgressCallback>,
8404 ) -> Result<CustomSection, CompileError> {
8405 gen_import_call_trampoline_arm64(
8406 vmoffsets,
8407 index,
8408 sig,
8409 calling_convention,
8410 progress_callback,
8411 )
8412 }
8413
8414 #[cfg(feature = "unwind")]
8415 fn gen_dwarf_unwind_info(&mut self, code_len: usize) -> Option<UnwindInstructions> {
8416 let mut instructions = vec![];
8417 for &(instruction_offset, ref inst) in &self.unwind_ops {
8418 let instruction_offset = instruction_offset as u32;
8419 match *inst {
8420 UnwindOps::PushFP { up_to_sp } => {
8421 instructions.push((
8422 instruction_offset,
8423 CallFrameInstruction::CfaOffset(up_to_sp as i32),
8424 ));
8425 instructions.push((
8426 instruction_offset,
8427 CallFrameInstruction::Offset(AArch64::X29, -(up_to_sp as i32)),
8428 ));
8429 }
8430 UnwindOps::Push2Regs {
8431 reg1,
8432 reg2,
8433 up_to_sp,
8434 } => {
8435 instructions.push((
8436 instruction_offset,
8437 CallFrameInstruction::CfaOffset(up_to_sp as i32),
8438 ));
8439 instructions.push((
8440 instruction_offset,
8441 CallFrameInstruction::Offset(reg2.dwarf_index(), -(up_to_sp as i32) + 8),
8442 ));
8443 instructions.push((
8444 instruction_offset,
8445 CallFrameInstruction::Offset(reg1.dwarf_index(), -(up_to_sp as i32)),
8446 ));
8447 }
8448 UnwindOps::DefineNewFrame => {
8449 instructions.push((
8450 instruction_offset,
8451 CallFrameInstruction::CfaRegister(AArch64::X29),
8452 ));
8453 }
8454 UnwindOps::SaveRegister { reg, bp_neg_offset } => instructions.push((
8455 instruction_offset,
8456 CallFrameInstruction::Offset(reg.dwarf_index(), -bp_neg_offset),
8457 )),
8458 UnwindOps::SubtractFP { .. } => unimplemented!(),
8459 }
8460 }
8461 Some(UnwindInstructions {
8462 instructions,
8463 len: code_len as u32,
8464 })
8465 }
8466 #[cfg(not(feature = "unwind"))]
8467
8468 fn gen_dwarf_unwind_info(&mut self, _code_len: usize) -> Option<UnwindInstructions> {
8469 None
8470 }
8471
8472 fn gen_windows_unwind_info(&mut self, _code_len: usize) -> Option<Vec<u8>> {
8473 None
8474 }
8475}
8476
8477#[cfg(test)]
8478mod test {
8479 use super::*;
8480
8481 fn test_move_location(machine: &mut MachineARM64, size: Size) -> Result<(), CompileError> {
8482 machine.move_location(size, Location::GPR(GPR::X1), Location::GPR(GPR::X2))?;
8483 machine.move_location(size, Location::GPR(GPR::X1), Location::Memory(GPR::X2, 10))?;
8484 machine.move_location(size, Location::GPR(GPR::X1), Location::Memory(GPR::X2, -10))?;
8485 machine.move_location(
8486 size,
8487 Location::GPR(GPR::X1),
8488 Location::Memory(GPR::X2, 1024),
8489 )?;
8490 machine.move_location(
8491 size,
8492 Location::GPR(GPR::X1),
8493 Location::Memory(GPR::X2, -1024),
8494 )?;
8495 machine.move_location(size, Location::Memory(GPR::X2, 10), Location::GPR(GPR::X1))?;
8496 machine.move_location(size, Location::Memory(GPR::X2, -10), Location::GPR(GPR::X1))?;
8497 machine.move_location(
8498 size,
8499 Location::Memory(GPR::X2, 1024),
8500 Location::GPR(GPR::X1),
8501 )?;
8502 machine.move_location(
8503 size,
8504 Location::Memory(GPR::X2, -1024),
8505 Location::GPR(GPR::X1),
8506 )?;
8507 machine.move_location(size, Location::GPR(GPR::X1), Location::SIMD(NEON::V0))?;
8508 machine.move_location(size, Location::SIMD(NEON::V0), Location::GPR(GPR::X1))?;
8509 machine.move_location(
8510 size,
8511 Location::SIMD(NEON::V0),
8512 Location::Memory(GPR::X2, 10),
8513 )?;
8514 machine.move_location(
8515 size,
8516 Location::SIMD(NEON::V0),
8517 Location::Memory(GPR::X2, -10),
8518 )?;
8519 machine.move_location(
8520 size,
8521 Location::SIMD(NEON::V0),
8522 Location::Memory(GPR::X2, 1024),
8523 )?;
8524 machine.move_location(
8525 size,
8526 Location::SIMD(NEON::V0),
8527 Location::Memory(GPR::X2, -1024),
8528 )?;
8529 machine.move_location(
8530 size,
8531 Location::Memory(GPR::X2, 10),
8532 Location::SIMD(NEON::V0),
8533 )?;
8534 machine.move_location(
8535 size,
8536 Location::Memory(GPR::X2, -10),
8537 Location::SIMD(NEON::V0),
8538 )?;
8539 machine.move_location(
8540 size,
8541 Location::Memory(GPR::X2, 1024),
8542 Location::SIMD(NEON::V0),
8543 )?;
8544 machine.move_location(
8545 size,
8546 Location::Memory(GPR::X2, -1024),
8547 Location::SIMD(NEON::V0),
8548 )?;
8549
8550 Ok(())
8551 }
8552
8553 fn test_move_location_extended(
8554 machine: &mut MachineARM64,
8555 signed: bool,
8556 sized: Size,
8557 ) -> Result<(), CompileError> {
8558 machine.move_location_extend(
8559 sized,
8560 signed,
8561 Location::GPR(GPR::X0),
8562 Size::S64,
8563 Location::GPR(GPR::X1),
8564 )?;
8565 machine.move_location_extend(
8566 sized,
8567 signed,
8568 Location::GPR(GPR::X0),
8569 Size::S64,
8570 Location::Memory(GPR::X1, 10),
8571 )?;
8572 machine.move_location_extend(
8573 sized,
8574 signed,
8575 Location::GPR(GPR::X0),
8576 Size::S64,
8577 Location::Memory(GPR::X1, 16),
8578 )?;
8579 machine.move_location_extend(
8580 sized,
8581 signed,
8582 Location::GPR(GPR::X0),
8583 Size::S64,
8584 Location::Memory(GPR::X1, -16),
8585 )?;
8586 machine.move_location_extend(
8587 sized,
8588 signed,
8589 Location::GPR(GPR::X0),
8590 Size::S64,
8591 Location::Memory(GPR::X1, 1024),
8592 )?;
8593 machine.move_location_extend(
8594 sized,
8595 signed,
8596 Location::GPR(GPR::X0),
8597 Size::S64,
8598 Location::Memory(GPR::X1, -1024),
8599 )?;
8600 machine.move_location_extend(
8601 sized,
8602 signed,
8603 Location::Memory(GPR::X0, 10),
8604 Size::S64,
8605 Location::GPR(GPR::X1),
8606 )?;
8607
8608 Ok(())
8609 }
8610
8611 fn test_binop_op(
8612 machine: &mut MachineARM64,
8613 op: fn(&mut MachineARM64, Location, Location, Location) -> Result<(), CompileError>,
8614 ) -> Result<(), CompileError> {
8615 op(
8616 machine,
8617 Location::GPR(GPR::X2),
8618 Location::GPR(GPR::X2),
8619 Location::GPR(GPR::X0),
8620 )?;
8621 op(
8622 machine,
8623 Location::GPR(GPR::X2),
8624 Location::Imm32(10),
8625 Location::GPR(GPR::X0),
8626 )?;
8627 op(
8628 machine,
8629 Location::GPR(GPR::X0),
8630 Location::GPR(GPR::X0),
8631 Location::GPR(GPR::X0),
8632 )?;
8633 op(
8634 machine,
8635 Location::Imm32(10),
8636 Location::GPR(GPR::X2),
8637 Location::GPR(GPR::X0),
8638 )?;
8639 op(
8640 machine,
8641 Location::GPR(GPR::X0),
8642 Location::GPR(GPR::X2),
8643 Location::Memory(GPR::X0, 10),
8644 )?;
8645 op(
8646 machine,
8647 Location::GPR(GPR::X0),
8648 Location::Memory(GPR::X2, 16),
8649 Location::Memory(GPR::X0, 10),
8650 )?;
8651 op(
8652 machine,
8653 Location::Memory(GPR::X0, 0),
8654 Location::Memory(GPR::X2, 16),
8655 Location::Memory(GPR::X0, 10),
8656 )?;
8657
8658 Ok(())
8659 }
8660
8661 fn test_float_binop_op(
8662 machine: &mut MachineARM64,
8663 op: fn(&mut MachineARM64, Location, Location, Location) -> Result<(), CompileError>,
8664 ) -> Result<(), CompileError> {
8665 op(
8666 machine,
8667 Location::SIMD(NEON::V3),
8668 Location::SIMD(NEON::V2),
8669 Location::SIMD(NEON::V0),
8670 )?;
8671 op(
8672 machine,
8673 Location::SIMD(NEON::V0),
8674 Location::SIMD(NEON::V2),
8675 Location::SIMD(NEON::V0),
8676 )?;
8677 op(
8678 machine,
8679 Location::SIMD(NEON::V0),
8680 Location::SIMD(NEON::V0),
8681 Location::SIMD(NEON::V0),
8682 )?;
8683 op(
8684 machine,
8685 Location::Memory(GPR::X0, 0),
8686 Location::SIMD(NEON::V2),
8687 Location::SIMD(NEON::V0),
8688 )?;
8689 op(
8690 machine,
8691 Location::Memory(GPR::X0, 0),
8692 Location::Memory(GPR::X1, 10),
8693 Location::SIMD(NEON::V0),
8694 )?;
8695 op(
8696 machine,
8697 Location::Memory(GPR::X0, 0),
8698 Location::Memory(GPR::X1, 16),
8699 Location::Memory(GPR::X2, 32),
8700 )?;
8701 op(
8702 machine,
8703 Location::SIMD(NEON::V0),
8704 Location::Memory(GPR::X1, 16),
8705 Location::Memory(GPR::X2, 32),
8706 )?;
8707 op(
8708 machine,
8709 Location::SIMD(NEON::V0),
8710 Location::SIMD(NEON::V1),
8711 Location::Memory(GPR::X2, 32),
8712 )?;
8713
8714 Ok(())
8715 }
8716
8717 fn test_float_cmp_op(
8718 machine: &mut MachineARM64,
8719 op: fn(&mut MachineARM64, Location, Location, Location) -> Result<(), CompileError>,
8720 ) -> Result<(), CompileError> {
8721 op(
8722 machine,
8723 Location::SIMD(NEON::V3),
8724 Location::SIMD(NEON::V2),
8725 Location::GPR(GPR::X0),
8726 )?;
8727 op(
8728 machine,
8729 Location::SIMD(NEON::V0),
8730 Location::SIMD(NEON::V0),
8731 Location::GPR(GPR::X0),
8732 )?;
8733 op(
8734 machine,
8735 Location::Memory(GPR::X1, 0),
8736 Location::SIMD(NEON::V2),
8737 Location::GPR(GPR::X0),
8738 )?;
8739 op(
8740 machine,
8741 Location::Memory(GPR::X1, 0),
8742 Location::Memory(GPR::X2, 10),
8743 Location::GPR(GPR::X0),
8744 )?;
8745 op(
8746 machine,
8747 Location::Memory(GPR::X1, 0),
8748 Location::Memory(GPR::X2, 16),
8749 Location::Memory(GPR::X0, 32),
8750 )?;
8751 op(
8752 machine,
8753 Location::SIMD(NEON::V0),
8754 Location::Memory(GPR::X2, 16),
8755 Location::Memory(GPR::X0, 32),
8756 )?;
8757 op(
8758 machine,
8759 Location::SIMD(NEON::V0),
8760 Location::SIMD(NEON::V1),
8761 Location::Memory(GPR::X0, 32),
8762 )?;
8763
8764 Ok(())
8765 }
8766
8767 #[test]
8768 fn tests_arm64() -> Result<(), CompileError> {
8769 let mut machine = MachineARM64::new(None);
8770
8771 test_move_location(&mut machine, Size::S32)?;
8772 test_move_location(&mut machine, Size::S64)?;
8773 test_move_location_extended(&mut machine, false, Size::S8)?;
8774 test_move_location_extended(&mut machine, false, Size::S16)?;
8775 test_move_location_extended(&mut machine, false, Size::S32)?;
8776 test_move_location_extended(&mut machine, true, Size::S8)?;
8777 test_move_location_extended(&mut machine, true, Size::S16)?;
8778 test_move_location_extended(&mut machine, true, Size::S32)?;
8779 test_binop_op(&mut machine, MachineARM64::emit_binop_add32)?;
8780 test_binop_op(&mut machine, MachineARM64::emit_binop_add64)?;
8781 test_binop_op(&mut machine, MachineARM64::emit_binop_sub32)?;
8782 test_binop_op(&mut machine, MachineARM64::emit_binop_sub64)?;
8783 test_binop_op(&mut machine, MachineARM64::emit_binop_and32)?;
8784 test_binop_op(&mut machine, MachineARM64::emit_binop_and64)?;
8785 test_binop_op(&mut machine, MachineARM64::emit_binop_xor32)?;
8786 test_binop_op(&mut machine, MachineARM64::emit_binop_xor64)?;
8787 test_binop_op(&mut machine, MachineARM64::emit_binop_or32)?;
8788 test_binop_op(&mut machine, MachineARM64::emit_binop_or64)?;
8789 test_binop_op(&mut machine, MachineARM64::emit_binop_mul32)?;
8790 test_binop_op(&mut machine, MachineARM64::emit_binop_mul64)?;
8791 test_float_binop_op(&mut machine, MachineARM64::f32_add)?;
8792 test_float_binop_op(&mut machine, MachineARM64::f32_sub)?;
8793 test_float_binop_op(&mut machine, MachineARM64::f32_mul)?;
8794 test_float_binop_op(&mut machine, MachineARM64::f32_div)?;
8795 test_float_cmp_op(&mut machine, MachineARM64::f32_cmp_eq)?;
8796 test_float_cmp_op(&mut machine, MachineARM64::f32_cmp_lt)?;
8797 test_float_cmp_op(&mut machine, MachineARM64::f32_cmp_le)?;
8798
8799 Ok(())
8800 }
8801}