1use dynasmrt::{VecAssembler, aarch64::Aarch64Relocation};
2#[cfg(feature = "unwind")]
3use gimli::{AArch64, write::CallFrameInstruction};
4
5use wasmer_compiler::{
6 types::{
7 address_map::InstructionAddressMap,
8 function::FunctionBody,
9 relocation::{Relocation, RelocationKind, RelocationTarget},
10 section::CustomSection,
11 },
12 wasmparser::{MemArg, ValType as WpType},
13};
14use wasmer_types::{
15 CompileError, FunctionIndex, FunctionType, SourceLoc, TrapCode, TrapInformation, VMOffsets,
16 target::{CallingConvention, CpuFeature, Target},
17};
18
19use crate::{
20 arm64_decl::{GPR, NEON, new_machine_state},
21 codegen_error,
22 common_decl::*,
23 emitter_arm64::*,
24 location::{Location as AbstractLocation, Reg},
25 machine::*,
26 unwind::{UnwindInstructions, UnwindOps, UnwindRegister},
27};
28
29type Assembler = VecAssembler<Aarch64Relocation>;
30type Location = AbstractLocation<GPR, NEON>;
31
32pub struct MachineARM64 {
33 assembler: Assembler,
34 used_gprs: u32,
35 used_simd: u32,
36 trap_table: TrapTable,
37 instructions_address_map: Vec<InstructionAddressMap>,
40 src_loc: u32,
42 pushed: bool,
44 unwind_ops: Vec<(usize, UnwindOps<GPR, NEON>)>,
46 has_neon: bool,
48}
49
50#[allow(dead_code)]
51#[derive(PartialEq)]
52enum ImmType {
53 None,
54 NoneXzr,
55 Bits8,
56 Bits12,
57 Shift32,
58 Shift32No0,
59 Shift64,
60 Shift64No0,
61 Logical32,
62 Logical64,
63 UnscaledOffset,
64 OffsetByte,
65 OffsetHWord,
66 OffsetWord,
67 OffsetDWord,
68}
69
70#[allow(dead_code)]
71impl MachineARM64 {
72 pub fn new(target: Option<Target>) -> Self {
73 let has_neon = match target {
77 Some(ref target) => target.cpu_features().contains(CpuFeature::NEON),
78 None => false,
79 };
80
81 MachineARM64 {
82 assembler: Assembler::new(0),
83 used_gprs: 0,
84 used_simd: 0,
85 trap_table: TrapTable::default(),
86 instructions_address_map: vec![],
87 src_loc: 0,
88 pushed: false,
89 unwind_ops: vec![],
90 has_neon,
91 }
92 }
93 fn compatible_imm(&self, imm: i64, ty: ImmType) -> bool {
94 match ty {
95 ImmType::None => false,
96 ImmType::NoneXzr => false,
97 ImmType::Bits8 => (0..256).contains(&imm),
98 ImmType::Bits12 => (0..0x1000).contains(&imm),
99 ImmType::Shift32 => (0..32).contains(&imm),
100 ImmType::Shift32No0 => (1..32).contains(&imm),
101 ImmType::Shift64 => (0..64).contains(&imm),
102 ImmType::Shift64No0 => (1..64).contains(&imm),
103 ImmType::Logical32 => encode_logical_immediate_32bit(imm as u32).is_some(),
104 ImmType::Logical64 => encode_logical_immediate_64bit(imm as u64).is_some(),
105 ImmType::UnscaledOffset => (imm > -256) && (imm < 256),
106 ImmType::OffsetByte => (0..0x1000).contains(&imm),
107 ImmType::OffsetHWord => (imm & 1 == 0) && (0..0x2000).contains(&imm),
108 ImmType::OffsetWord => (imm & 3 == 0) && (0..0x4000).contains(&imm),
109 ImmType::OffsetDWord => (imm & 7 == 0) && (0..0x8000).contains(&imm),
110 }
111 }
112
113 fn location_to_reg(
114 &mut self,
115 sz: Size,
116 src: Location,
117 temps: &mut Vec<GPR>,
118 allow_imm: ImmType,
119 read_val: bool,
120 wanted: Option<GPR>,
121 ) -> Result<Location, CompileError> {
122 match src {
123 Location::GPR(_) | Location::SIMD(_) => Ok(src),
124 Location::Imm8(val) => {
125 if allow_imm == ImmType::NoneXzr && val == 0 {
126 Ok(Location::GPR(GPR::XzrSp))
127 } else if self.compatible_imm(val as i64, allow_imm) {
128 Ok(src)
129 } else {
130 let tmp = if let Some(wanted) = wanted {
131 wanted
132 } else {
133 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
134 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
135 })?;
136 temps.push(tmp);
137 tmp
138 };
139 self.assembler
140 .emit_mov_imm(Location::GPR(tmp), val as u64)?;
141 Ok(Location::GPR(tmp))
142 }
143 }
144 Location::Imm32(val) => {
145 if allow_imm == ImmType::NoneXzr && val == 0 {
146 Ok(Location::GPR(GPR::XzrSp))
147 } else if self.compatible_imm(val as i64, allow_imm) {
148 Ok(src)
149 } else {
150 let tmp = if let Some(wanted) = wanted {
151 wanted
152 } else {
153 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
154 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
155 })?;
156 temps.push(tmp);
157 tmp
158 };
159 self.assembler
160 .emit_mov_imm(Location::GPR(tmp), (val as i64) as u64)?;
161 Ok(Location::GPR(tmp))
162 }
163 }
164 Location::Imm64(val) => {
165 if allow_imm == ImmType::NoneXzr && val == 0 {
166 Ok(Location::GPR(GPR::XzrSp))
167 } else if self.compatible_imm(val as i64, allow_imm) {
168 Ok(src)
169 } else {
170 let tmp = if let Some(wanted) = wanted {
171 wanted
172 } else {
173 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
174 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
175 })?;
176 temps.push(tmp);
177 tmp
178 };
179 self.assembler
180 .emit_mov_imm(Location::GPR(tmp), val as u64)?;
181 Ok(Location::GPR(tmp))
182 }
183 }
184 Location::Memory(reg, val) => {
185 let tmp = if let Some(wanted) = wanted {
186 wanted
187 } else {
188 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
189 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
190 })?;
191 temps.push(tmp);
192 tmp
193 };
194 if read_val {
195 let offsize = match sz {
196 Size::S8 => ImmType::OffsetByte,
197 Size::S16 => ImmType::OffsetHWord,
198 Size::S32 => ImmType::OffsetWord,
199 Size::S64 => ImmType::OffsetDWord,
200 };
201 if sz == Size::S8 {
202 if self.compatible_imm(val as i64, offsize) {
203 self.assembler.emit_ldrb(
204 sz,
205 Location::GPR(tmp),
206 Location::Memory(reg, val as _),
207 )?;
208 } else {
209 if reg == tmp {
210 codegen_error!("singlepass reg==tmp unreachable");
211 }
212 self.assembler
213 .emit_mov_imm(Location::GPR(tmp), (val as i64) as u64)?;
214 self.assembler.emit_ldrb(
215 sz,
216 Location::GPR(tmp),
217 Location::Memory2(reg, tmp, Multiplier::One, 0),
218 )?;
219 }
220 } else if sz == Size::S16 {
221 if self.compatible_imm(val as i64, offsize) {
222 self.assembler.emit_ldrh(
223 sz,
224 Location::GPR(tmp),
225 Location::Memory(reg, val as _),
226 )?;
227 } else {
228 if reg == tmp {
229 codegen_error!("singlepass reg==tmp unreachable");
230 }
231 self.assembler
232 .emit_mov_imm(Location::GPR(tmp), (val as i64) as u64)?;
233 self.assembler.emit_ldrh(
234 sz,
235 Location::GPR(tmp),
236 Location::Memory2(reg, tmp, Multiplier::One, 0),
237 )?;
238 }
239 } else if self.compatible_imm(val as i64, offsize) {
240 self.assembler.emit_ldr(
241 sz,
242 Location::GPR(tmp),
243 Location::Memory(reg, val as _),
244 )?;
245 } else if self.compatible_imm(val as i64, ImmType::UnscaledOffset) {
246 self.assembler.emit_ldur(sz, Location::GPR(tmp), reg, val)?;
247 } else {
248 if reg == tmp {
249 codegen_error!("singlepass reg == tmp unreachable");
250 }
251 self.assembler
252 .emit_mov_imm(Location::GPR(tmp), (val as i64) as u64)?;
253 self.assembler.emit_ldr(
254 sz,
255 Location::GPR(tmp),
256 Location::Memory2(reg, tmp, Multiplier::One, 0),
257 )?;
258 }
259 }
260 Ok(Location::GPR(tmp))
261 }
262 _ => codegen_error!("singlepass can't emit location_to_reg {:?} {:?}", sz, src),
263 }
264 }
265 fn location_to_neon(
266 &mut self,
267 sz: Size,
268 src: Location,
269 temps: &mut Vec<NEON>,
270 allow_imm: ImmType,
271 read_val: bool,
272 ) -> Result<Location, CompileError> {
273 match src {
274 Location::SIMD(_) => Ok(src),
275 Location::GPR(_) => {
276 let tmp = self.acquire_temp_simd().ok_or_else(|| {
277 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
278 })?;
279 temps.push(tmp);
280 if read_val {
281 self.assembler.emit_mov(sz, src, Location::SIMD(tmp))?;
282 }
283 Ok(Location::SIMD(tmp))
284 }
285 Location::Imm8(val) => {
286 if self.compatible_imm(val as i64, allow_imm) {
287 Ok(src)
288 } else {
289 let gpr = self.acquire_temp_gpr().ok_or_else(|| {
290 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
291 })?;
292 let tmp = self.acquire_temp_simd().ok_or_else(|| {
293 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
294 })?;
295 temps.push(tmp);
296 self.assembler
297 .emit_mov_imm(Location::GPR(gpr), val as u64)?;
298 self.assembler
299 .emit_mov(sz, Location::GPR(gpr), Location::SIMD(tmp))?;
300 self.release_gpr(gpr);
301 Ok(Location::SIMD(tmp))
302 }
303 }
304 Location::Imm32(val) => {
305 if self.compatible_imm(val as i64, allow_imm) {
306 Ok(src)
307 } else {
308 let gpr = self.acquire_temp_gpr().ok_or_else(|| {
309 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
310 })?;
311 let tmp = self.acquire_temp_simd().ok_or_else(|| {
312 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
313 })?;
314 temps.push(tmp);
315 self.assembler
316 .emit_mov_imm(Location::GPR(gpr), (val as i64) as u64)?;
317 self.assembler
318 .emit_mov(sz, Location::GPR(gpr), Location::SIMD(tmp))?;
319 self.release_gpr(gpr);
320 Ok(Location::SIMD(tmp))
321 }
322 }
323 Location::Imm64(val) => {
324 if self.compatible_imm(val as i64, allow_imm) {
325 Ok(src)
326 } else {
327 let gpr = self.acquire_temp_gpr().ok_or_else(|| {
328 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
329 })?;
330 let tmp = self.acquire_temp_simd().ok_or_else(|| {
331 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
332 })?;
333 temps.push(tmp);
334 self.assembler
335 .emit_mov_imm(Location::GPR(gpr), val as u64)?;
336 self.assembler
337 .emit_mov(sz, Location::GPR(gpr), Location::SIMD(tmp))?;
338 self.release_gpr(gpr);
339 Ok(Location::SIMD(tmp))
340 }
341 }
342 Location::Memory(reg, val) => {
343 let tmp = self.acquire_temp_simd().ok_or_else(|| {
344 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
345 })?;
346 temps.push(tmp);
347 if read_val {
348 let offsize = if sz == Size::S32 {
349 ImmType::OffsetWord
350 } else {
351 ImmType::OffsetDWord
352 };
353 if self.compatible_imm(val as i64, offsize) {
354 self.assembler.emit_ldr(
355 sz,
356 Location::SIMD(tmp),
357 Location::Memory(reg, val as _),
358 )?;
359 } else if self.compatible_imm(val as i64, ImmType::UnscaledOffset) {
360 self.assembler
361 .emit_ldur(sz, Location::SIMD(tmp), reg, val)?;
362 } else {
363 let gpr = self.acquire_temp_gpr().ok_or_else(|| {
364 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
365 })?;
366 self.assembler
367 .emit_mov_imm(Location::GPR(gpr), (val as i64) as u64)?;
368 self.assembler.emit_ldr(
369 sz,
370 Location::SIMD(tmp),
371 Location::Memory2(reg, gpr, Multiplier::One, 0),
372 )?;
373 self.release_gpr(gpr);
374 }
375 }
376 Ok(Location::SIMD(tmp))
377 }
378 _ => codegen_error!("singlepass can't emit location_to_neon {:?} {:?}", sz, src),
379 }
380 }
381
382 fn emit_relaxed_binop(
383 &mut self,
384 op: fn(&mut Assembler, Size, Location, Location) -> Result<(), CompileError>,
385 sz: Size,
386 src: Location,
387 dst: Location,
388 putback: bool,
389 ) -> Result<(), CompileError> {
390 let mut temps = vec![];
391 let src_imm = if putback {
392 ImmType::None
393 } else {
394 ImmType::Bits12
395 };
396 let src = self.location_to_reg(sz, src, &mut temps, src_imm, true, None)?;
397 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, !putback, None)?;
398 op(&mut self.assembler, sz, src, dest)?;
399 if dst != dest && putback {
400 self.move_location(sz, dest, dst)?;
401 }
402 for r in temps {
403 self.release_gpr(r);
404 }
405 Ok(())
406 }
407 fn emit_relaxed_binop_neon(
408 &mut self,
409 op: fn(&mut Assembler, Size, Location, Location) -> Result<(), CompileError>,
410 sz: Size,
411 src: Location,
412 dst: Location,
413 putback: bool,
414 ) -> Result<(), CompileError> {
415 let mut temps = vec![];
416 let src = self.location_to_neon(sz, src, &mut temps, ImmType::None, true)?;
417 let dest = self.location_to_neon(sz, dst, &mut temps, ImmType::None, !putback)?;
418 op(&mut self.assembler, sz, src, dest)?;
419 if dst != dest && putback {
420 self.move_location(sz, dest, dst)?;
421 }
422 for r in temps {
423 self.release_simd(r);
424 }
425 Ok(())
426 }
427 fn emit_relaxed_binop3(
428 &mut self,
429 op: fn(&mut Assembler, Size, Location, Location, Location) -> Result<(), CompileError>,
430 sz: Size,
431 src1: Location,
432 src2: Location,
433 dst: Location,
434 allow_imm: ImmType,
435 ) -> Result<(), CompileError> {
436 let mut temps = vec![];
437 let src1 = self.location_to_reg(sz, src1, &mut temps, ImmType::None, true, None)?;
438 let src2 = self.location_to_reg(sz, src2, &mut temps, allow_imm, true, None)?;
439 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
440 op(&mut self.assembler, sz, src1, src2, dest)?;
441 if dst != dest {
442 self.move_location(sz, dest, dst)?;
443 }
444 for r in temps {
445 self.release_gpr(r);
446 }
447 Ok(())
448 }
449 fn emit_relaxed_binop3_neon(
450 &mut self,
451 op: fn(&mut Assembler, Size, Location, Location, Location) -> Result<(), CompileError>,
452 sz: Size,
453 src1: Location,
454 src2: Location,
455 dst: Location,
456 allow_imm: ImmType,
457 ) -> Result<(), CompileError> {
458 let mut temps = vec![];
459 let src1 = self.location_to_neon(sz, src1, &mut temps, ImmType::None, true)?;
460 let src2 = self.location_to_neon(sz, src2, &mut temps, allow_imm, true)?;
461 let dest = self.location_to_neon(sz, dst, &mut temps, ImmType::None, false)?;
462 op(&mut self.assembler, sz, src1, src2, dest)?;
463 if dst != dest {
464 self.move_location(sz, dest, dst)?;
465 }
466 for r in temps {
467 self.release_simd(r);
468 }
469 Ok(())
470 }
471 fn emit_relaxed_ldr64(
472 &mut self,
473 sz: Size,
474 dst: Location,
475 src: Location,
476 ) -> Result<(), CompileError> {
477 let mut temps = vec![];
478 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
479 match src {
480 Location::Memory(addr, offset) => {
481 if self.compatible_imm(offset as i64, ImmType::OffsetDWord) {
482 self.assembler.emit_ldr(Size::S64, dest, src)?;
483 } else if self.compatible_imm(offset as i64, ImmType::UnscaledOffset) {
484 self.assembler.emit_ldur(Size::S64, dest, addr, offset)?;
485 } else {
486 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
487 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
488 })?;
489 self.assembler
490 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
491 self.assembler.emit_ldr(
492 Size::S64,
493 dest,
494 Location::Memory2(addr, tmp, Multiplier::One, 0),
495 )?;
496 temps.push(tmp);
497 }
498 }
499 _ => codegen_error!("singplass emit_relaxed_ldr64 unreachable"),
500 }
501 if dst != dest {
502 self.move_location(sz, dest, dst)?;
503 }
504 for r in temps {
505 self.release_gpr(r);
506 }
507 Ok(())
508 }
509 fn emit_relaxed_ldr32(
510 &mut self,
511 sz: Size,
512 dst: Location,
513 src: Location,
514 ) -> Result<(), CompileError> {
515 let mut temps = vec![];
516 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
517 match src {
518 Location::Memory(addr, offset) => {
519 if self.compatible_imm(offset as i64, ImmType::OffsetWord) {
520 self.assembler.emit_ldr(Size::S32, dest, src)?;
521 } else if self.compatible_imm(offset as i64, ImmType::UnscaledOffset) {
522 self.assembler.emit_ldur(Size::S32, dest, addr, offset)?;
523 } else {
524 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
525 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
526 })?;
527 self.assembler
528 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
529 self.assembler.emit_ldr(
530 Size::S32,
531 dest,
532 Location::Memory2(addr, tmp, Multiplier::One, 0),
533 )?;
534 temps.push(tmp);
535 }
536 }
537 _ => codegen_error!("singlepass emit_relaxed_ldr32 unreachable"),
538 }
539 if dst != dest {
540 self.move_location(sz, dest, dst)?;
541 }
542 for r in temps {
543 self.release_gpr(r);
544 }
545 Ok(())
546 }
547 fn emit_relaxed_ldr32s(
548 &mut self,
549 sz: Size,
550 dst: Location,
551 src: Location,
552 ) -> Result<(), CompileError> {
553 let mut temps = vec![];
554 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
555 match src {
556 Location::Memory(addr, offset) => {
557 if self.compatible_imm(offset as i64, ImmType::OffsetWord) {
558 self.assembler.emit_ldrsw(Size::S64, dest, src)?;
559 } else {
560 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
561 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
562 })?;
563 self.assembler
564 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
565 self.assembler.emit_ldrsw(
566 Size::S64,
567 dest,
568 Location::Memory2(addr, tmp, Multiplier::One, 0),
569 )?;
570 temps.push(tmp);
571 }
572 }
573 _ => codegen_error!("singplepass emit_relaxed_ldr32s unreachable"),
574 }
575 if dst != dest {
576 self.move_location(sz, dest, dst)?;
577 }
578 for r in temps {
579 self.release_gpr(r);
580 }
581 Ok(())
582 }
583 fn emit_relaxed_ldr16(
584 &mut self,
585 sz: Size,
586 dst: Location,
587 src: Location,
588 ) -> Result<(), CompileError> {
589 let mut temps = vec![];
590 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
591 match src {
592 Location::Memory(addr, offset) => {
593 if self.compatible_imm(offset as i64, ImmType::OffsetHWord) {
594 self.assembler.emit_ldrh(Size::S32, dest, src)?;
595 } else {
596 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
597 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
598 })?;
599 self.assembler
600 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
601 self.assembler.emit_ldrh(
602 Size::S32,
603 dest,
604 Location::Memory2(addr, tmp, Multiplier::One, 0),
605 )?;
606 temps.push(tmp);
607 }
608 }
609 _ => codegen_error!("singlpass emit_relaxed_ldr16 unreachable"),
610 }
611 if dst != dest {
612 self.move_location(sz, dest, dst)?;
613 }
614 for r in temps {
615 self.release_gpr(r);
616 }
617 Ok(())
618 }
619 fn emit_relaxed_ldr16s(
620 &mut self,
621 sz: Size,
622 dst: Location,
623 src: Location,
624 ) -> Result<(), CompileError> {
625 let mut temps = vec![];
626 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
627 match src {
628 Location::Memory(addr, offset) => {
629 if self.compatible_imm(offset as i64, ImmType::OffsetHWord) {
630 self.assembler.emit_ldrsh(sz, dest, src)?;
631 } else {
632 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
633 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
634 })?;
635 self.assembler
636 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
637 self.assembler.emit_ldrsh(
638 sz,
639 dest,
640 Location::Memory2(addr, tmp, Multiplier::One, 0),
641 )?;
642 temps.push(tmp);
643 }
644 }
645 _ => codegen_error!("singlepass emit_relaxed_ldr16s unreachable"),
646 }
647 if dst != dest {
648 self.move_location(sz, dest, dst)?;
649 }
650 for r in temps {
651 self.release_gpr(r);
652 }
653 Ok(())
654 }
655 fn emit_relaxed_ldr8(
656 &mut self,
657 sz: Size,
658 dst: Location,
659 src: Location,
660 ) -> Result<(), CompileError> {
661 let mut temps = vec![];
662 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
663 match src {
664 Location::Memory(addr, offset) => {
665 if self.compatible_imm(offset as i64, ImmType::OffsetByte) {
666 self.assembler.emit_ldrb(Size::S32, dest, src)?;
667 } else {
668 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
669 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
670 })?;
671 self.assembler
672 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
673 self.assembler.emit_ldrb(
674 Size::S32,
675 dest,
676 Location::Memory2(addr, tmp, Multiplier::One, 0),
677 )?;
678 temps.push(tmp);
679 }
680 }
681 _ => codegen_error!("singplepass emit_relaxed_ldr8 unreachable"),
682 }
683 if dst != dest {
684 self.move_location(sz, dest, dst)?;
685 }
686 for r in temps {
687 self.release_gpr(r);
688 }
689 Ok(())
690 }
691 fn emit_relaxed_ldr8s(
692 &mut self,
693 sz: Size,
694 dst: Location,
695 src: Location,
696 ) -> Result<(), CompileError> {
697 let mut temps = vec![];
698 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
699 match src {
700 Location::Memory(addr, offset) => {
701 if self.compatible_imm(offset as i64, ImmType::OffsetByte) {
702 self.assembler.emit_ldrsb(sz, dest, src)?;
703 } else {
704 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
705 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
706 })?;
707 self.assembler
708 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
709 self.assembler.emit_ldrsb(
710 sz,
711 dest,
712 Location::Memory2(addr, tmp, Multiplier::One, 0),
713 )?;
714 temps.push(tmp);
715 }
716 }
717 _ => codegen_error!("singlepass emit_relaxed_ldr8s unreachable"),
718 }
719 if dst != dest {
720 self.move_location(sz, dest, dst)?;
721 }
722 for r in temps {
723 self.release_gpr(r);
724 }
725 Ok(())
726 }
727 fn emit_relaxed_str64(&mut self, dst: Location, src: Location) -> Result<(), CompileError> {
728 let mut temps = vec![];
729 let dst = self.location_to_reg(Size::S64, dst, &mut temps, ImmType::NoneXzr, true, None)?;
730 match src {
731 Location::Memory(addr, offset) => {
732 if self.compatible_imm(offset as i64, ImmType::OffsetDWord) {
733 self.assembler.emit_str(Size::S64, dst, src)?;
734 } else if self.compatible_imm(offset as i64, ImmType::UnscaledOffset) {
735 self.assembler.emit_stur(Size::S64, dst, addr, offset)?;
736 } else {
737 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
738 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
739 })?;
740 self.assembler
741 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
742 self.assembler.emit_str(
743 Size::S64,
744 dst,
745 Location::Memory2(addr, tmp, Multiplier::One, 0),
746 )?;
747 temps.push(tmp);
748 }
749 }
750 _ => codegen_error!("singlepass can't emit str64 {:?} {:?}", dst, src),
751 }
752 for r in temps {
753 self.release_gpr(r);
754 }
755 Ok(())
756 }
757 fn emit_relaxed_str32(&mut self, dst: Location, src: Location) -> Result<(), CompileError> {
758 let mut temps = vec![];
759 let dst = self.location_to_reg(Size::S64, dst, &mut temps, ImmType::NoneXzr, true, None)?;
760 match src {
761 Location::Memory(addr, offset) => {
762 if self.compatible_imm(offset as i64, ImmType::OffsetWord) {
763 self.assembler.emit_str(Size::S32, dst, src)?;
764 } else if self.compatible_imm(offset as i64, ImmType::UnscaledOffset) {
765 self.assembler.emit_stur(Size::S32, dst, addr, offset)?;
766 } else {
767 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
768 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
769 })?;
770 self.assembler
771 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
772 self.assembler.emit_str(
773 Size::S32,
774 dst,
775 Location::Memory2(addr, tmp, Multiplier::One, 0),
776 )?;
777 temps.push(tmp);
778 }
779 }
780 _ => codegen_error!("singplepass emit_relaxed_str32 unreachable"),
781 }
782 for r in temps {
783 self.release_gpr(r);
784 }
785 Ok(())
786 }
787 fn emit_relaxed_str16(&mut self, dst: Location, src: Location) -> Result<(), CompileError> {
788 let mut temps = vec![];
789 let dst = self.location_to_reg(Size::S64, dst, &mut temps, ImmType::NoneXzr, true, None)?;
790 match src {
791 Location::Memory(addr, offset) => {
792 if self.compatible_imm(offset as i64, ImmType::OffsetHWord) {
793 self.assembler.emit_strh(Size::S32, dst, src)?;
794 } else {
795 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
796 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
797 })?;
798 self.assembler
799 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
800 self.assembler.emit_strh(
801 Size::S32,
802 dst,
803 Location::Memory2(addr, tmp, Multiplier::One, 0),
804 )?;
805 temps.push(tmp);
806 }
807 }
808 _ => codegen_error!("singlepass emit_relaxed_str16 unreachable"),
809 }
810 for r in temps {
811 self.release_gpr(r);
812 }
813 Ok(())
814 }
815 fn emit_relaxed_str8(&mut self, dst: Location, src: Location) -> Result<(), CompileError> {
816 let mut temps = vec![];
817 let dst = self.location_to_reg(Size::S64, dst, &mut temps, ImmType::NoneXzr, true, None)?;
818 match src {
819 Location::Memory(addr, offset) => {
820 if self.compatible_imm(offset as i64, ImmType::OffsetByte) {
821 self.assembler
822 .emit_strb(Size::S32, dst, Location::Memory(addr, offset))?;
823 } else {
824 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
825 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
826 })?;
827 self.assembler
828 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
829 self.assembler.emit_strb(
830 Size::S32,
831 dst,
832 Location::Memory2(addr, tmp, Multiplier::One, 0),
833 )?;
834 temps.push(tmp);
835 }
836 }
837 _ => codegen_error!("singlepass emit_relaxed_str8 unreachable"),
838 }
839 for r in temps {
840 self.release_gpr(r);
841 }
842 Ok(())
843 }
844 fn emit_cmpop_i64_dynamic_b(
846 &mut self,
847 c: Condition,
848 loc_a: Location,
849 loc_b: Location,
850 ret: Location,
851 ) -> Result<(), CompileError> {
852 match ret {
853 Location::GPR(_) => {
854 self.emit_relaxed_cmp(Size::S64, loc_b, loc_a)?;
855 self.assembler.emit_cset(Size::S32, ret, c)?;
856 }
857 Location::Memory(_, _) => {
858 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
859 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
860 })?;
861 self.emit_relaxed_cmp(Size::S64, loc_b, loc_a)?;
862 self.assembler.emit_cset(Size::S32, Location::GPR(tmp), c)?;
863 self.move_location(Size::S32, Location::GPR(tmp), ret)?;
864 self.release_gpr(tmp);
865 }
866 _ => {
867 codegen_error!("singlepass emit_compop_i64_dynamic_b unreachable");
868 }
869 }
870 Ok(())
871 }
872 fn emit_cmpop_i32_dynamic_b(
874 &mut self,
875 c: Condition,
876 loc_a: Location,
877 loc_b: Location,
878 ret: Location,
879 ) -> Result<(), CompileError> {
880 match ret {
881 Location::GPR(_) => {
882 self.emit_relaxed_cmp(Size::S32, loc_b, loc_a)?;
883 self.assembler.emit_cset(Size::S32, ret, c)?;
884 }
885 Location::Memory(_, _) => {
886 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
887 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
888 })?;
889 self.emit_relaxed_cmp(Size::S32, loc_b, loc_a)?;
890 self.assembler.emit_cset(Size::S32, Location::GPR(tmp), c)?;
891 self.move_location(Size::S32, Location::GPR(tmp), ret)?;
892 self.release_gpr(tmp);
893 }
894 _ => {
895 codegen_error!("singlepass emit_cmpop_i32_dynamic_b unreachable");
896 }
897 }
898 Ok(())
899 }
900
901 #[allow(clippy::too_many_arguments)]
902 fn memory_op<F: FnOnce(&mut Self, GPR) -> Result<(), CompileError>>(
903 &mut self,
904 addr: Location,
905 memarg: &MemArg,
906 check_alignment: bool,
907 value_size: usize,
908 need_check: bool,
909 imported_memories: bool,
910 offset: i32,
911 heap_access_oob: Label,
912 unaligned_atomic: Label,
913 cb: F,
914 ) -> Result<(), CompileError> {
915 let tmp_addr = self.acquire_temp_gpr().ok_or_else(|| {
916 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
917 })?;
918
919 let (base_loc, bound_loc) = if imported_memories {
921 self.emit_relaxed_binop(
923 Assembler::emit_mov,
924 Size::S64,
925 Location::Memory(self.get_vmctx_reg(), offset),
926 Location::GPR(tmp_addr),
927 true,
928 )?;
929 (Location::Memory(tmp_addr, 0), Location::Memory(tmp_addr, 8))
930 } else {
931 (
932 Location::Memory(self.get_vmctx_reg(), offset),
933 Location::Memory(self.get_vmctx_reg(), offset + 8),
934 )
935 };
936
937 let tmp_base = self.acquire_temp_gpr().ok_or_else(|| {
938 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
939 })?;
940 let tmp_bound = self.acquire_temp_gpr().ok_or_else(|| {
941 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
942 })?;
943
944 self.emit_relaxed_ldr64(Size::S64, Location::GPR(tmp_base), base_loc)?;
946
947 if need_check {
949 self.emit_relaxed_ldr64(Size::S64, Location::GPR(tmp_bound), bound_loc)?;
950
951 self.assembler.emit_add(
955 Size::S64,
956 Location::GPR(tmp_bound),
957 Location::GPR(tmp_base),
958 Location::GPR(tmp_bound),
959 )?;
960 if self.compatible_imm(value_size as _, ImmType::Bits12) {
961 self.assembler.emit_sub(
962 Size::S64,
963 Location::GPR(tmp_bound),
964 Location::Imm32(value_size as _),
965 Location::GPR(tmp_bound),
966 )?;
967 } else {
968 let tmp2 = self.acquire_temp_gpr().ok_or_else(|| {
969 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
970 })?;
971 self.assembler
972 .emit_mov_imm(Location::GPR(tmp2), value_size as u64)?;
973 self.assembler.emit_sub(
974 Size::S64,
975 Location::GPR(tmp_bound),
976 Location::GPR(tmp2),
977 Location::GPR(tmp_bound),
978 )?;
979 self.release_gpr(tmp2);
980 }
981 }
982
983 self.move_location(Size::S32, addr, Location::GPR(tmp_addr))?;
987
988 if memarg.offset != 0 {
990 if self.compatible_imm(memarg.offset as _, ImmType::Bits12) {
991 self.assembler.emit_adds(
992 Size::S32,
993 Location::Imm32(memarg.offset as u32),
994 Location::GPR(tmp_addr),
995 Location::GPR(tmp_addr),
996 )?;
997 } else {
998 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
999 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1000 })?;
1001 self.assembler
1002 .emit_mov_imm(Location::GPR(tmp), memarg.offset as _)?;
1003 self.assembler.emit_adds(
1004 Size::S32,
1005 Location::GPR(tmp_addr),
1006 Location::GPR(tmp),
1007 Location::GPR(tmp_addr),
1008 )?;
1009 self.release_gpr(tmp);
1010 }
1011
1012 self.assembler
1014 .emit_bcond_label_far(Condition::Cs, heap_access_oob)?;
1015 }
1016
1017 self.assembler.emit_add(
1019 Size::S64,
1020 Location::GPR(tmp_base),
1021 Location::GPR(tmp_addr),
1022 Location::GPR(tmp_addr),
1023 )?;
1024
1025 if need_check {
1026 self.assembler.emit_cmp(
1028 Size::S64,
1029 Location::GPR(tmp_bound),
1030 Location::GPR(tmp_addr),
1031 )?;
1032
1033 self.assembler
1035 .emit_bcond_label_far(Condition::Hi, heap_access_oob)?;
1036 }
1037
1038 self.release_gpr(tmp_bound);
1039 self.release_gpr(tmp_base);
1040
1041 let align = value_size as u32;
1042 if check_alignment && align != 1 {
1043 self.assembler.emit_tst(
1044 Size::S64,
1045 Location::Imm32(align - 1),
1046 Location::GPR(tmp_addr),
1047 )?;
1048 self.assembler
1049 .emit_bcond_label_far(Condition::Ne, unaligned_atomic)?;
1050 }
1051 let begin = self.assembler.get_offset().0;
1052 cb(self, tmp_addr)?;
1053 let end = self.assembler.get_offset().0;
1054 self.mark_address_range_with_trap_code(TrapCode::HeapAccessOutOfBounds, begin, end);
1055
1056 self.release_gpr(tmp_addr);
1057 Ok(())
1058 }
1059
1060 fn offset_is_ok(&self, size: Size, offset: i32) -> bool {
1080 if offset < 0 {
1081 return false;
1082 }
1083 let shift = match size {
1084 Size::S8 => 0,
1085 Size::S16 => 1,
1086 Size::S32 => 2,
1087 Size::S64 => 3,
1088 };
1089 if offset >= 0x1000 << shift {
1090 return false;
1091 }
1092 if (offset & ((1 << shift) - 1)) != 0 {
1093 return false;
1094 }
1095 true
1096 }
1097
1098 fn emit_push(&mut self, sz: Size, src: Location) -> Result<(), CompileError> {
1099 match (sz, src) {
1100 (Size::S64, Location::GPR(_)) | (Size::S64, Location::SIMD(_)) => {
1101 let offset = if self.pushed {
1102 0
1103 } else {
1104 self.assembler.emit_sub(
1105 Size::S64,
1106 Location::GPR(GPR::XzrSp),
1107 Location::Imm8(16),
1108 Location::GPR(GPR::XzrSp),
1109 )?;
1110 8
1111 };
1112 self.assembler
1113 .emit_stur(Size::S64, src, GPR::XzrSp, offset)?;
1114 self.pushed = !self.pushed;
1115 }
1116 (Size::S64, _) => {
1117 let mut temps = vec![];
1118 let src = self.location_to_reg(sz, src, &mut temps, ImmType::None, true, None)?;
1119 let offset = if self.pushed {
1120 0
1121 } else {
1122 self.assembler.emit_sub(
1123 Size::S64,
1124 Location::GPR(GPR::XzrSp),
1125 Location::Imm8(16),
1126 Location::GPR(GPR::XzrSp),
1127 )?;
1128 8
1129 };
1130 self.assembler
1131 .emit_stur(Size::S64, src, GPR::XzrSp, offset)?;
1132 self.pushed = !self.pushed;
1133 for r in temps {
1134 self.release_gpr(r);
1135 }
1136 }
1137 _ => codegen_error!("singlepass can't emit PUSH {:?} {:?}", sz, src),
1138 }
1139 Ok(())
1140 }
1141 fn emit_double_push(
1142 &mut self,
1143 sz: Size,
1144 src1: Location,
1145 src2: Location,
1146 ) -> Result<(), CompileError> {
1147 if !self.pushed {
1148 match (sz, src1, src2) {
1149 (Size::S64, Location::GPR(_), Location::GPR(_)) => {
1150 self.assembler
1151 .emit_stpdb(Size::S64, src1, src2, GPR::XzrSp, 16)?;
1152 }
1153 _ => {
1154 self.emit_push(sz, src1)?;
1155 self.emit_push(sz, src2)?;
1156 }
1157 }
1158 } else {
1159 self.emit_push(sz, src1)?;
1160 self.emit_push(sz, src2)?;
1161 }
1162 Ok(())
1163 }
1164 fn emit_pop(&mut self, sz: Size, dst: Location) -> Result<(), CompileError> {
1165 match (sz, dst) {
1166 (Size::S64, Location::GPR(_)) | (Size::S64, Location::SIMD(_)) => {
1167 let offset = if self.pushed { 8 } else { 0 };
1168 self.assembler
1169 .emit_ldur(Size::S64, dst, GPR::XzrSp, offset)?;
1170 if self.pushed {
1171 self.assembler.emit_add(
1172 Size::S64,
1173 Location::GPR(GPR::XzrSp),
1174 Location::Imm8(16),
1175 Location::GPR(GPR::XzrSp),
1176 )?;
1177 }
1178 self.pushed = !self.pushed;
1179 }
1180 _ => codegen_error!("singlepass can't emit POP {:?} {:?}", sz, dst),
1181 }
1182 Ok(())
1183 }
1184 fn emit_double_pop(
1185 &mut self,
1186 sz: Size,
1187 dst1: Location,
1188 dst2: Location,
1189 ) -> Result<(), CompileError> {
1190 if !self.pushed {
1191 match (sz, dst1, dst2) {
1192 (Size::S64, Location::GPR(_), Location::GPR(_)) => {
1193 self.assembler
1194 .emit_ldpia(Size::S64, dst1, dst2, GPR::XzrSp, 16)?;
1195 }
1196 _ => {
1197 self.emit_pop(sz, dst2)?;
1198 self.emit_pop(sz, dst1)?;
1199 }
1200 }
1201 } else {
1202 self.emit_pop(sz, dst2)?;
1203 self.emit_pop(sz, dst1)?;
1204 }
1205 Ok(())
1206 }
1207
1208 fn set_default_nan(&mut self, temps: &mut Vec<GPR>) -> Result<GPR, CompileError> {
1209 let old_fpcr = self.acquire_temp_gpr().ok_or_else(|| {
1211 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1212 })?;
1213 temps.push(old_fpcr);
1214 self.assembler.emit_read_fpcr(old_fpcr)?;
1215 let new_fpcr = self.acquire_temp_gpr().ok_or_else(|| {
1216 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1217 })?;
1218 temps.push(new_fpcr);
1219 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
1220 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1221 })?;
1222 temps.push(tmp);
1223 self.assembler
1224 .emit_mov(Size::S32, Location::Imm32(1), Location::GPR(tmp))?;
1225 self.assembler
1226 .emit_mov(Size::S64, Location::GPR(old_fpcr), Location::GPR(new_fpcr))?;
1227 self.assembler.emit_bfi(
1229 Size::S64,
1230 Location::GPR(tmp),
1231 25,
1232 1,
1233 Location::GPR(new_fpcr),
1234 )?;
1235 self.assembler.emit_write_fpcr(new_fpcr)?;
1236 Ok(old_fpcr)
1237 }
1238 fn set_trap_enabled(&mut self, temps: &mut Vec<GPR>) -> Result<GPR, CompileError> {
1239 let old_fpcr = self.acquire_temp_gpr().ok_or_else(|| {
1241 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1242 })?;
1243 temps.push(old_fpcr);
1244 self.assembler.emit_read_fpcr(old_fpcr)?;
1245 let new_fpcr = self.acquire_temp_gpr().ok_or_else(|| {
1246 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1247 })?;
1248 temps.push(new_fpcr);
1249 self.assembler
1250 .emit_mov(Size::S64, Location::GPR(old_fpcr), Location::GPR(new_fpcr))?;
1251 self.assembler
1253 .emit_bfc(Size::S64, 8, 1, Location::GPR(new_fpcr))?;
1254 self.assembler.emit_write_fpcr(new_fpcr)?;
1255 Ok(old_fpcr)
1256 }
1257 fn restore_fpcr(&mut self, old_fpcr: GPR) -> Result<(), CompileError> {
1258 self.assembler.emit_write_fpcr(old_fpcr)
1259 }
1260
1261 fn reset_exception_fpsr(&mut self) -> Result<(), CompileError> {
1262 let fpsr = self.acquire_temp_gpr().ok_or_else(|| {
1264 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1265 })?;
1266 self.assembler.emit_read_fpsr(fpsr)?;
1267 self.assembler
1269 .emit_bfc(Size::S64, 0, 1, Location::GPR(fpsr))?;
1270 self.assembler.emit_write_fpsr(fpsr)?;
1271 self.release_gpr(fpsr);
1272 Ok(())
1273 }
1274 fn read_fpsr(&mut self) -> Result<GPR, CompileError> {
1275 let fpsr = self.acquire_temp_gpr().ok_or_else(|| {
1276 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1277 })?;
1278 self.assembler.emit_read_fpsr(fpsr)?;
1279 Ok(fpsr)
1280 }
1281
1282 fn trap_float_convertion_errors(
1283 &mut self,
1284 old_fpcr: GPR,
1285 sz: Size,
1286 f: Location,
1287 temps: &mut Vec<GPR>,
1288 ) -> Result<(), CompileError> {
1289 let trap_badconv = self.assembler.get_label();
1290 let end = self.assembler.get_label();
1291
1292 let fpsr = self.read_fpsr()?;
1293 temps.push(fpsr);
1294 self.assembler
1296 .emit_tbz_label(Size::S32, Location::GPR(fpsr), 0, end)?;
1297 self.assembler
1299 .emit_bfc(Size::S64, 0, 4, Location::GPR(fpsr))?;
1300 self.restore_fpcr(old_fpcr)?;
1301 self.assembler.emit_fcmp(sz, f, f)?;
1302 self.assembler
1303 .emit_bcond_label(Condition::Vs, trap_badconv)?;
1304 self.emit_illegal_op_internal(TrapCode::IntegerOverflow)?;
1306
1307 self.emit_label(trap_badconv)?;
1308 self.emit_illegal_op_internal(TrapCode::BadConversionToInteger)?;
1309
1310 self.emit_label(end)?;
1311 self.restore_fpcr(old_fpcr)
1312 }
1313
1314 fn used_gprs_contains(&self, r: &GPR) -> bool {
1315 self.used_gprs & (1 << r.into_index()) != 0
1316 }
1317 fn used_simd_contains(&self, r: &NEON) -> bool {
1318 self.used_simd & (1 << r.into_index()) != 0
1319 }
1320 fn used_gprs_insert(&mut self, r: GPR) {
1321 self.used_gprs |= 1 << r.into_index();
1322 }
1323 fn used_simd_insert(&mut self, r: NEON) {
1324 self.used_simd |= 1 << r.into_index();
1325 }
1326 fn used_gprs_remove(&mut self, r: &GPR) -> bool {
1327 let ret = self.used_gprs_contains(r);
1328 self.used_gprs &= !(1 << r.into_index());
1329 ret
1330 }
1331 fn used_simd_remove(&mut self, r: &NEON) -> bool {
1332 let ret = self.used_simd_contains(r);
1333 self.used_simd &= !(1 << r.into_index());
1334 ret
1335 }
1336 fn emit_unwind_op(&mut self, op: UnwindOps<GPR, NEON>) {
1337 self.unwind_ops.push((self.get_offset().0, op));
1338 }
1339 fn emit_illegal_op_internal(&mut self, trap: TrapCode) -> Result<(), CompileError> {
1340 self.assembler.emit_udf(0xc0 | (trap as u8) as u16)
1341 }
1342}
1343
1344impl Machine for MachineARM64 {
1345 type GPR = GPR;
1346 type SIMD = NEON;
1347 fn assembler_get_offset(&self) -> Offset {
1348 self.assembler.get_offset()
1349 }
1350 fn index_from_gpr(&self, x: GPR) -> RegisterIndex {
1351 RegisterIndex(x as usize)
1352 }
1353 fn index_from_simd(&self, x: NEON) -> RegisterIndex {
1354 RegisterIndex(x as usize + 32)
1355 }
1356
1357 fn get_vmctx_reg(&self) -> GPR {
1358 GPR::X28
1359 }
1360
1361 fn get_used_gprs(&self) -> Vec<GPR> {
1362 GPR::iterator()
1363 .filter(|x| self.used_gprs & (1 << x.into_index()) != 0)
1364 .cloned()
1365 .collect()
1366 }
1367
1368 fn get_used_simd(&self) -> Vec<NEON> {
1369 NEON::iterator()
1370 .filter(|x| self.used_simd & (1 << x.into_index()) != 0)
1371 .cloned()
1372 .collect()
1373 }
1374
1375 fn pick_gpr(&self) -> Option<GPR> {
1376 use GPR::*;
1377 static REGS: &[GPR] = &[X9, X10, X11, X12, X13, X14, X15];
1378 for r in REGS {
1379 if !self.used_gprs_contains(r) {
1380 return Some(*r);
1381 }
1382 }
1383 None
1384 }
1385
1386 fn pick_temp_gpr(&self) -> Option<GPR> {
1388 use GPR::*;
1389 static REGS: &[GPR] = &[X8, X7, X6, X5, X4, X3, X2, X1];
1390 for r in REGS {
1391 if !self.used_gprs_contains(r) {
1392 return Some(*r);
1393 }
1394 }
1395 None
1396 }
1397
1398 fn acquire_temp_gpr(&mut self) -> Option<GPR> {
1399 let gpr = self.pick_temp_gpr();
1400 if let Some(x) = gpr {
1401 self.used_gprs_insert(x);
1402 }
1403 gpr
1404 }
1405
1406 fn release_gpr(&mut self, gpr: GPR) {
1407 assert!(self.used_gprs_remove(&gpr));
1408 }
1409
1410 fn reserve_unused_temp_gpr(&mut self, gpr: GPR) -> GPR {
1411 assert!(!self.used_gprs_contains(&gpr));
1412 self.used_gprs_insert(gpr);
1413 gpr
1414 }
1415
1416 fn reserve_gpr(&mut self, gpr: GPR) {
1417 self.used_gprs_insert(gpr);
1418 }
1419
1420 fn push_used_gpr(&mut self, used_gprs: &[GPR]) -> Result<usize, CompileError> {
1421 if used_gprs.len() % 2 == 1 {
1422 self.emit_push(Size::S64, Location::GPR(GPR::XzrSp))?;
1423 }
1424 for r in used_gprs.iter() {
1425 self.emit_push(Size::S64, Location::GPR(*r))?;
1426 }
1427 Ok(used_gprs.len().div_ceil(2) * 16)
1428 }
1429 fn pop_used_gpr(&mut self, used_gprs: &[GPR]) -> Result<(), CompileError> {
1430 for r in used_gprs.iter().rev() {
1431 self.emit_pop(Size::S64, Location::GPR(*r))?;
1432 }
1433 if used_gprs.len() % 2 == 1 {
1434 self.emit_pop(Size::S64, Location::GPR(GPR::XzrSp))?;
1435 }
1436 Ok(())
1437 }
1438
1439 fn pick_simd(&self) -> Option<NEON> {
1441 use NEON::*;
1442 static REGS: &[NEON] = &[V8, V9, V10, V11, V12];
1443 for r in REGS {
1444 if !self.used_simd_contains(r) {
1445 return Some(*r);
1446 }
1447 }
1448 None
1449 }
1450
1451 fn pick_temp_simd(&self) -> Option<NEON> {
1453 use NEON::*;
1454 static REGS: &[NEON] = &[V0, V1, V2, V3, V4, V5, V6, V7];
1455 for r in REGS {
1456 if !self.used_simd_contains(r) {
1457 return Some(*r);
1458 }
1459 }
1460 None
1461 }
1462
1463 fn acquire_temp_simd(&mut self) -> Option<NEON> {
1465 let simd = self.pick_temp_simd();
1466 if let Some(x) = simd {
1467 self.used_simd_insert(x);
1468 }
1469 simd
1470 }
1471
1472 fn reserve_simd(&mut self, simd: NEON) {
1473 self.used_simd_insert(simd);
1474 }
1475
1476 fn release_simd(&mut self, simd: NEON) {
1478 assert!(self.used_simd_remove(&simd));
1479 }
1480
1481 fn push_used_simd(&mut self, used_neons: &[NEON]) -> Result<usize, CompileError> {
1482 let stack_adjust = if used_neons.len() & 1 == 1 {
1483 (used_neons.len() * 8) as u32 + 8
1484 } else {
1485 (used_neons.len() * 8) as u32
1486 };
1487 self.adjust_stack(stack_adjust)?;
1488
1489 for (i, r) in used_neons.iter().enumerate() {
1490 self.assembler.emit_str(
1491 Size::S64,
1492 Location::SIMD(*r),
1493 Location::Memory(GPR::XzrSp, (i * 8) as i32),
1494 )?;
1495 }
1496 Ok(stack_adjust as usize)
1497 }
1498 fn pop_used_simd(&mut self, used_neons: &[NEON]) -> Result<(), CompileError> {
1499 for (i, r) in used_neons.iter().enumerate() {
1500 self.assembler.emit_ldr(
1501 Size::S64,
1502 Location::SIMD(*r),
1503 Location::Memory(GPR::XzrSp, (i * 8) as i32),
1504 )?;
1505 }
1506 let stack_adjust = if used_neons.len() & 1 == 1 {
1507 (used_neons.len() * 8) as u32 + 8
1508 } else {
1509 (used_neons.len() * 8) as u32
1510 };
1511 self.assembler.emit_add(
1512 Size::S64,
1513 Location::GPR(GPR::XzrSp),
1514 Location::Imm32(stack_adjust as _),
1515 Location::GPR(GPR::XzrSp),
1516 )
1517 }
1518
1519 fn set_srcloc(&mut self, offset: u32) {
1521 self.src_loc = offset;
1522 }
1523 fn mark_address_range_with_trap_code(&mut self, code: TrapCode, begin: usize, end: usize) {
1525 for i in begin..end {
1526 self.trap_table.offset_to_code.insert(i, code);
1527 }
1528 self.mark_instruction_address_end(begin);
1529 }
1530
1531 fn mark_address_with_trap_code(&mut self, code: TrapCode) {
1533 let offset = self.assembler.get_offset().0;
1534 self.trap_table.offset_to_code.insert(offset, code);
1535 self.mark_instruction_address_end(offset);
1536 }
1537 fn mark_instruction_with_trap_code(&mut self, code: TrapCode) -> usize {
1539 let offset = self.assembler.get_offset().0;
1540 self.trap_table.offset_to_code.insert(offset, code);
1541 offset
1542 }
1543 fn mark_instruction_address_end(&mut self, begin: usize) {
1546 self.instructions_address_map.push(InstructionAddressMap {
1547 srcloc: SourceLoc::new(self.src_loc),
1548 code_offset: begin,
1549 code_len: self.assembler.get_offset().0 - begin,
1550 });
1551 }
1552
1553 fn insert_stackoverflow(&mut self) {
1555 let offset = 0;
1556 self.trap_table
1557 .offset_to_code
1558 .insert(offset, TrapCode::StackOverflow);
1559 self.mark_instruction_address_end(offset);
1560 }
1561
1562 fn collect_trap_information(&self) -> Vec<TrapInformation> {
1564 self.trap_table
1565 .offset_to_code
1566 .clone()
1567 .into_iter()
1568 .map(|(offset, code)| TrapInformation {
1569 code_offset: offset as u32,
1570 trap_code: code,
1571 })
1572 .collect()
1573 }
1574
1575 fn instructions_address_map(&self) -> Vec<InstructionAddressMap> {
1576 self.instructions_address_map.clone()
1577 }
1578
1579 fn round_stack_adjust(&self, value: usize) -> usize {
1581 if value & 0xf != 0 {
1582 ((value >> 4) + 1) << 4
1583 } else {
1584 value
1585 }
1586 }
1587
1588 fn local_on_stack(&mut self, stack_offset: i32) -> Location {
1590 Location::Memory(GPR::X29, -stack_offset)
1591 }
1592
1593 fn adjust_stack(&mut self, delta_stack_offset: u32) -> Result<(), CompileError> {
1595 let delta = if self.compatible_imm(delta_stack_offset as _, ImmType::Bits12) {
1596 Location::Imm32(delta_stack_offset as _)
1597 } else {
1598 let tmp = GPR::X17;
1599 self.assembler
1600 .emit_mov_imm(Location::GPR(tmp), delta_stack_offset as u64)?;
1601 Location::GPR(tmp)
1602 };
1603 self.assembler.emit_sub(
1604 Size::S64,
1605 Location::GPR(GPR::XzrSp),
1606 delta,
1607 Location::GPR(GPR::XzrSp),
1608 )
1609 }
1610 fn restore_stack(&mut self, delta_stack_offset: u32) -> Result<(), CompileError> {
1612 let delta = if self.compatible_imm(delta_stack_offset as _, ImmType::Bits12) {
1613 Location::Imm32(delta_stack_offset as _)
1614 } else {
1615 let tmp = GPR::X17;
1616 self.assembler
1617 .emit_mov_imm(Location::GPR(tmp), delta_stack_offset as u64)?;
1618 Location::GPR(tmp)
1619 };
1620 self.assembler.emit_add(
1621 Size::S64,
1622 Location::GPR(GPR::XzrSp),
1623 delta,
1624 Location::GPR(GPR::XzrSp),
1625 )
1626 }
1627 fn pop_stack_locals(&mut self, delta_stack_offset: u32) -> Result<(), CompileError> {
1628 let real_delta = if delta_stack_offset & 15 != 0 {
1629 delta_stack_offset + 8
1630 } else {
1631 delta_stack_offset
1632 };
1633 let delta = if self.compatible_imm(real_delta as i64, ImmType::Bits12) {
1634 Location::Imm32(real_delta as _)
1635 } else {
1636 let tmp = GPR::X17;
1637 self.assembler
1638 .emit_mov_imm(Location::GPR(tmp), real_delta as u64)?;
1639 Location::GPR(tmp)
1640 };
1641 self.assembler.emit_add(
1642 Size::S64,
1643 Location::GPR(GPR::XzrSp),
1644 delta,
1645 Location::GPR(GPR::XzrSp),
1646 )
1647 }
1648 fn move_location_for_native(
1650 &mut self,
1651 size: Size,
1652 loc: Location,
1653 dest: Location,
1654 ) -> Result<(), CompileError> {
1655 match loc {
1656 Location::Imm64(_)
1657 | Location::Imm32(_)
1658 | Location::Imm8(_)
1659 | Location::Memory(_, _)
1660 | Location::Memory2(_, _, _, _) => {
1661 self.move_location(size, loc, Location::GPR(GPR::X17))?;
1662 self.move_location(size, Location::GPR(GPR::X17), dest)
1663 }
1664 _ => self.move_location(size, loc, dest),
1665 }
1666 }
1667
1668 fn zero_location(&mut self, size: Size, location: Location) -> Result<(), CompileError> {
1670 self.move_location(size, Location::GPR(GPR::XzrSp), location)
1671 }
1672
1673 fn local_pointer(&self) -> GPR {
1675 GPR::X29
1676 }
1677
1678 fn is_local_on_stack(&self, idx: usize) -> bool {
1680 idx > 7
1681 }
1682
1683 fn get_local_location(&self, idx: usize, callee_saved_regs_size: usize) -> Location {
1685 match idx {
1687 0 => Location::GPR(GPR::X19),
1688 1 => Location::GPR(GPR::X20),
1689 2 => Location::GPR(GPR::X21),
1690 3 => Location::GPR(GPR::X22),
1691 4 => Location::GPR(GPR::X23),
1692 5 => Location::GPR(GPR::X24),
1693 6 => Location::GPR(GPR::X25),
1694 7 => Location::GPR(GPR::X26),
1695 _ => Location::Memory(GPR::X29, -(((idx - 7) * 8 + callee_saved_regs_size) as i32)),
1696 }
1697 }
1698 fn move_local(&mut self, stack_offset: i32, location: Location) -> Result<(), CompileError> {
1700 if stack_offset < 256 {
1701 self.assembler
1702 .emit_stur(Size::S64, location, GPR::X29, -stack_offset)?;
1703 } else {
1704 let tmp = GPR::X17;
1705 if stack_offset < 0x1_0000 {
1706 self.assembler
1707 .emit_mov_imm(Location::GPR(tmp), (-stack_offset as i64) as u64)?;
1708 self.assembler.emit_str(
1709 Size::S64,
1710 location,
1711 Location::Memory2(GPR::X29, tmp, Multiplier::One, 0),
1712 )?;
1713 } else {
1714 self.assembler
1715 .emit_mov_imm(Location::GPR(tmp), (stack_offset as i64) as u64)?;
1716 self.assembler.emit_sub(
1717 Size::S64,
1718 Location::GPR(GPR::X29),
1719 Location::GPR(tmp),
1720 Location::GPR(tmp),
1721 )?;
1722 self.assembler
1723 .emit_str(Size::S64, location, Location::GPR(tmp))?;
1724 }
1725 }
1726 match location {
1727 Location::GPR(x) => self.emit_unwind_op(UnwindOps::SaveRegister {
1728 reg: UnwindRegister::<GPR, NEON>::GPR(x),
1729 bp_neg_offset: stack_offset,
1730 }),
1731 Location::SIMD(x) => self.emit_unwind_op(UnwindOps::SaveRegister {
1732 reg: UnwindRegister::FPR(x),
1733 bp_neg_offset: stack_offset,
1734 }),
1735 _ => (),
1736 }
1737 Ok(())
1738 }
1739
1740 fn list_to_save(&self, _calling_convention: CallingConvention) -> Vec<Location> {
1742 vec![]
1743 }
1744
1745 fn get_param_location(
1747 &self,
1748 idx: usize,
1749 sz: Size,
1750 stack_args: &mut usize,
1751 calling_convention: CallingConvention,
1752 ) -> Location {
1753 match calling_convention {
1754 CallingConvention::AppleAarch64 => match idx {
1755 0 => Location::GPR(GPR::X0),
1756 1 => Location::GPR(GPR::X1),
1757 2 => Location::GPR(GPR::X2),
1758 3 => Location::GPR(GPR::X3),
1759 4 => Location::GPR(GPR::X4),
1760 5 => Location::GPR(GPR::X5),
1761 6 => Location::GPR(GPR::X6),
1762 7 => Location::GPR(GPR::X7),
1763 _ => {
1764 let sz = 1
1765 << match sz {
1766 Size::S8 => 0,
1767 Size::S16 => 1,
1768 Size::S32 => 2,
1769 Size::S64 => 3,
1770 };
1771 if sz > 1 && *stack_args & (sz - 1) != 0 {
1773 *stack_args = (*stack_args + (sz - 1)) & !(sz - 1);
1774 }
1775 let loc = Location::Memory(GPR::XzrSp, *stack_args as i32);
1776 *stack_args += sz;
1777 loc
1778 }
1779 },
1780 _ => match idx {
1781 0 => Location::GPR(GPR::X0),
1782 1 => Location::GPR(GPR::X1),
1783 2 => Location::GPR(GPR::X2),
1784 3 => Location::GPR(GPR::X3),
1785 4 => Location::GPR(GPR::X4),
1786 5 => Location::GPR(GPR::X5),
1787 6 => Location::GPR(GPR::X6),
1788 7 => Location::GPR(GPR::X7),
1789 _ => {
1790 let loc = Location::Memory(GPR::XzrSp, *stack_args as i32);
1791 *stack_args += 8;
1792 loc
1793 }
1794 },
1795 }
1796 }
1797 fn get_call_param_location(
1799 &self,
1800 idx: usize,
1801 sz: Size,
1802 stack_args: &mut usize,
1803 calling_convention: CallingConvention,
1804 ) -> Location {
1805 match calling_convention {
1806 CallingConvention::AppleAarch64 => match idx {
1807 0 => Location::GPR(GPR::X0),
1808 1 => Location::GPR(GPR::X1),
1809 2 => Location::GPR(GPR::X2),
1810 3 => Location::GPR(GPR::X3),
1811 4 => Location::GPR(GPR::X4),
1812 5 => Location::GPR(GPR::X5),
1813 6 => Location::GPR(GPR::X6),
1814 7 => Location::GPR(GPR::X7),
1815 _ => {
1816 let sz = 1
1817 << match sz {
1818 Size::S8 => 0,
1819 Size::S16 => 1,
1820 Size::S32 => 2,
1821 Size::S64 => 3,
1822 };
1823 if sz > 1 && *stack_args & (sz - 1) != 0 {
1825 *stack_args = (*stack_args + (sz - 1)) & !(sz - 1);
1826 }
1827 let loc = Location::Memory(GPR::X29, 16 * 2 + *stack_args as i32);
1828 *stack_args += sz;
1829 loc
1830 }
1831 },
1832 _ => match idx {
1833 0 => Location::GPR(GPR::X0),
1834 1 => Location::GPR(GPR::X1),
1835 2 => Location::GPR(GPR::X2),
1836 3 => Location::GPR(GPR::X3),
1837 4 => Location::GPR(GPR::X4),
1838 5 => Location::GPR(GPR::X5),
1839 6 => Location::GPR(GPR::X6),
1840 7 => Location::GPR(GPR::X7),
1841 _ => {
1842 let loc = Location::Memory(GPR::X29, 16 * 2 + *stack_args as i32);
1843 *stack_args += 8;
1844 loc
1845 }
1846 },
1847 }
1848 }
1849 fn get_simple_param_location(
1851 &self,
1852 idx: usize,
1853 calling_convention: CallingConvention,
1854 ) -> Location {
1855 #[allow(clippy::match_single_binding)]
1856 match calling_convention {
1857 _ => match idx {
1858 0 => Location::GPR(GPR::X0),
1859 1 => Location::GPR(GPR::X1),
1860 2 => Location::GPR(GPR::X2),
1861 3 => Location::GPR(GPR::X3),
1862 4 => Location::GPR(GPR::X4),
1863 5 => Location::GPR(GPR::X5),
1864 6 => Location::GPR(GPR::X6),
1865 7 => Location::GPR(GPR::X7),
1866 _ => Location::Memory(GPR::X29, (16 * 2 + (idx - 8) * 8) as i32),
1867 },
1868 }
1869 }
1870 fn move_location(
1872 &mut self,
1873 size: Size,
1874 source: Location,
1875 dest: Location,
1876 ) -> Result<(), CompileError> {
1877 match source {
1878 Location::GPR(_) | Location::SIMD(_) => match dest {
1879 Location::GPR(_) | Location::SIMD(_) => self.assembler.emit_mov(size, source, dest),
1880 Location::Memory(addr, offs) => {
1881 if self.offset_is_ok(size, offs) {
1882 self.assembler.emit_str(size, source, dest)
1883 } else if self.compatible_imm(offs as i64, ImmType::UnscaledOffset) {
1884 self.assembler.emit_stur(size, source, addr, offs)
1885 } else {
1886 let tmp = GPR::X17;
1887 if offs < 0 {
1888 self.assembler
1889 .emit_mov_imm(Location::GPR(tmp), (-offs) as u64)?;
1890 self.assembler.emit_sub(
1891 Size::S64,
1892 Location::GPR(addr),
1893 Location::GPR(tmp),
1894 Location::GPR(tmp),
1895 )?;
1896 } else {
1897 self.assembler
1898 .emit_mov_imm(Location::GPR(tmp), offs as u64)?;
1899 self.assembler.emit_add(
1900 Size::S64,
1901 Location::GPR(addr),
1902 Location::GPR(tmp),
1903 Location::GPR(tmp),
1904 )?;
1905 }
1906 self.assembler
1907 .emit_str(size, source, Location::Memory(tmp, 0))
1908 }
1909 }
1910 _ => codegen_error!(
1911 "singlepass can't emit move_location {:?} {:?} => {:?}",
1912 size,
1913 source,
1914 dest
1915 ),
1916 },
1917 Location::Imm8(_) => match dest {
1918 Location::GPR(_) => self.assembler.emit_mov(size, source, dest),
1919 Location::Memory(_, _) => match size {
1920 Size::S64 => self.emit_relaxed_str64(source, dest),
1921 Size::S32 => self.emit_relaxed_str32(source, dest),
1922 Size::S16 => self.emit_relaxed_str16(source, dest),
1923 Size::S8 => self.emit_relaxed_str8(source, dest),
1924 },
1925 _ => codegen_error!(
1926 "singlepass can't emit move_location {:?} {:?} => {:?}",
1927 size,
1928 source,
1929 dest
1930 ),
1931 },
1932 Location::Imm32(val) => match dest {
1933 Location::GPR(_) => self.assembler.emit_mov_imm(dest, val as u64),
1934 Location::Memory(_, _) => match size {
1935 Size::S64 => self.emit_relaxed_str64(source, dest),
1936 Size::S32 => self.emit_relaxed_str32(source, dest),
1937 Size::S16 => self.emit_relaxed_str16(source, dest),
1938 Size::S8 => self.emit_relaxed_str8(source, dest),
1939 },
1940 _ => codegen_error!(
1941 "singlepass can't emit move_location {:?} {:?} => {:?}",
1942 size,
1943 source,
1944 dest
1945 ),
1946 },
1947 Location::Imm64(val) => match dest {
1948 Location::GPR(_) => self.assembler.emit_mov_imm(dest, val),
1949 Location::Memory(_, _) => match size {
1950 Size::S64 => self.emit_relaxed_str64(source, dest),
1951 Size::S32 => self.emit_relaxed_str32(source, dest),
1952 Size::S16 => self.emit_relaxed_str16(source, dest),
1953 Size::S8 => self.emit_relaxed_str8(source, dest),
1954 },
1955 _ => codegen_error!(
1956 "singlepass can't emit move_location {:?} {:?} => {:?}",
1957 size,
1958 source,
1959 dest
1960 ),
1961 },
1962 Location::Memory(addr, offs) => match dest {
1963 Location::GPR(_) | Location::SIMD(_) => {
1964 if self.offset_is_ok(size, offs) {
1965 self.assembler.emit_ldr(size, dest, source)
1966 } else if offs > -256 && offs < 256 {
1967 self.assembler.emit_ldur(size, dest, addr, offs)
1968 } else {
1969 let tmp = GPR::X17;
1970 if offs < 0 {
1971 self.assembler
1972 .emit_mov_imm(Location::GPR(tmp), (-offs) as u64)?;
1973 self.assembler.emit_sub(
1974 Size::S64,
1975 Location::GPR(addr),
1976 Location::GPR(tmp),
1977 Location::GPR(tmp),
1978 )?;
1979 } else {
1980 self.assembler
1981 .emit_mov_imm(Location::GPR(tmp), offs as u64)?;
1982 self.assembler.emit_add(
1983 Size::S64,
1984 Location::GPR(addr),
1985 Location::GPR(tmp),
1986 Location::GPR(tmp),
1987 )?;
1988 }
1989 self.assembler
1990 .emit_ldr(size, dest, Location::Memory(tmp, 0))
1991 }
1992 }
1993 _ => {
1994 let mut temps = vec![];
1995 let src =
1996 self.location_to_reg(size, source, &mut temps, ImmType::None, true, None)?;
1997 self.move_location(size, src, dest)?;
1998 for r in temps {
1999 self.release_gpr(r);
2000 }
2001 Ok(())
2002 }
2003 },
2004 _ => codegen_error!(
2005 "singlepass can't emit move_location {:?} {:?} => {:?}",
2006 size,
2007 source,
2008 dest
2009 ),
2010 }
2011 }
2012 fn move_location_extend(
2014 &mut self,
2015 size_val: Size,
2016 signed: bool,
2017 source: Location,
2018 size_op: Size,
2019 dest: Location,
2020 ) -> Result<(), CompileError> {
2021 if size_op != Size::S64 {
2022 codegen_error!("singlepass move_location_extend unreachable");
2023 }
2024 let mut temps = vec![];
2025 let dst = self.location_to_reg(size_op, dest, &mut temps, ImmType::None, false, None)?;
2026 let src = match (size_val, signed, source) {
2027 (Size::S64, _, _) => source,
2028 (Size::S32, false, Location::GPR(_)) => {
2029 self.assembler.emit_mov(size_val, source, dst)?;
2030 dst
2031 }
2032 (Size::S8, false, Location::GPR(_)) => {
2033 self.assembler.emit_uxtb(size_op, source, dst)?;
2034 dst
2035 }
2036 (Size::S16, false, Location::GPR(_)) => {
2037 self.assembler.emit_uxth(size_op, source, dst)?;
2038 dst
2039 }
2040 (Size::S8, true, Location::GPR(_)) => {
2041 self.assembler.emit_sxtb(size_op, source, dst)?;
2042 dst
2043 }
2044 (Size::S16, true, Location::GPR(_)) => {
2045 self.assembler.emit_sxth(size_op, source, dst)?;
2046 dst
2047 }
2048 (Size::S32, true, Location::GPR(_)) => {
2049 self.assembler.emit_sxtw(size_op, source, dst)?;
2050 dst
2051 }
2052 (Size::S32, false, Location::Memory(_, _)) => {
2053 self.emit_relaxed_ldr32(size_op, dst, source)?;
2054 dst
2055 }
2056 (Size::S32, true, Location::Memory(_, _)) => {
2057 self.emit_relaxed_ldr32s(size_op, dst, source)?;
2058 dst
2059 }
2060 (Size::S16, false, Location::Memory(_, _)) => {
2061 self.emit_relaxed_ldr16(size_op, dst, source)?;
2062 dst
2063 }
2064 (Size::S16, true, Location::Memory(_, _)) => {
2065 self.emit_relaxed_ldr16s(size_op, dst, source)?;
2066 dst
2067 }
2068 (Size::S8, false, Location::Memory(_, _)) => {
2069 self.emit_relaxed_ldr8(size_op, dst, source)?;
2070 dst
2071 }
2072 (Size::S8, true, Location::Memory(_, _)) => {
2073 self.emit_relaxed_ldr8s(size_op, dst, source)?;
2074 dst
2075 }
2076 _ => codegen_error!(
2077 "singlepass can't emit move_location_extend {:?} {:?} {:?} => {:?} {:?}",
2078 size_val,
2079 signed,
2080 source,
2081 size_op,
2082 dest
2083 ),
2084 };
2085 if src != dst {
2086 self.move_location(size_op, src, dst)?;
2087 }
2088 if dst != dest {
2089 self.move_location(size_op, dst, dest)?;
2090 }
2091 for r in temps {
2092 self.release_gpr(r);
2093 }
2094 Ok(())
2095 }
2096 fn load_address(
2097 &mut self,
2098 _size: Size,
2099 _reg: Location,
2100 _mem: Location,
2101 ) -> Result<(), CompileError> {
2102 codegen_error!("singlepass load_address unimplemented");
2103 }
2104 fn init_stack_loc(
2106 &mut self,
2107 init_stack_loc_cnt: u64,
2108 last_stack_loc: Location,
2109 ) -> Result<(), CompileError> {
2110 let label = self.assembler.get_label();
2111 let mut temps = vec![];
2112 let dest = self.acquire_temp_gpr().ok_or_else(|| {
2113 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2114 })?;
2115 temps.push(dest);
2116 let cnt = self.location_to_reg(
2117 Size::S64,
2118 Location::Imm64(init_stack_loc_cnt),
2119 &mut temps,
2120 ImmType::None,
2121 true,
2122 None,
2123 )?;
2124 let dest = match last_stack_loc {
2125 Location::GPR(_) => codegen_error!("singlepass init_stack_loc unreachable"),
2126 Location::SIMD(_) => codegen_error!("singlepass init_stack_loc unreachable"),
2127 Location::Memory(reg, offset) => {
2128 if offset < 0 {
2129 let offset = (-offset) as u32;
2130 if self.compatible_imm(offset as i64, ImmType::Bits12) {
2131 self.assembler.emit_sub(
2132 Size::S64,
2133 Location::GPR(reg),
2134 Location::Imm32(offset),
2135 Location::GPR(dest),
2136 )?;
2137 } else {
2138 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2139 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2140 })?;
2141 self.assembler
2142 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
2143 self.assembler.emit_sub(
2144 Size::S64,
2145 Location::GPR(reg),
2146 Location::GPR(tmp),
2147 Location::GPR(dest),
2148 )?;
2149 temps.push(tmp);
2150 }
2151 dest
2152 } else {
2153 let offset = offset as u32;
2154 if self.compatible_imm(offset as i64, ImmType::Bits12) {
2155 self.assembler.emit_add(
2156 Size::S64,
2157 Location::GPR(reg),
2158 Location::Imm32(offset),
2159 Location::GPR(dest),
2160 )?;
2161 } else {
2162 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2163 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2164 })?;
2165 self.assembler
2166 .emit_mov_imm(Location::GPR(tmp), (offset as i64) as u64)?;
2167 self.assembler.emit_add(
2168 Size::S64,
2169 Location::GPR(reg),
2170 Location::GPR(tmp),
2171 Location::GPR(dest),
2172 )?;
2173 temps.push(tmp);
2174 }
2175 dest
2176 }
2177 }
2178 _ => codegen_error!("singlepass can't emit init_stack_loc {:?}", last_stack_loc),
2179 };
2180 self.assembler.emit_label(label)?;
2181 self.assembler
2182 .emit_stria(Size::S64, Location::GPR(GPR::XzrSp), dest, 8)?;
2183 self.assembler
2184 .emit_sub(Size::S64, cnt, Location::Imm8(1), cnt)?;
2185 self.assembler.emit_cbnz_label(Size::S64, cnt, label)?;
2186 for r in temps {
2187 self.release_gpr(r);
2188 }
2189 Ok(())
2190 }
2191 fn restore_saved_area(&mut self, saved_area_offset: i32) -> Result<(), CompileError> {
2193 let real_delta = if saved_area_offset & 15 != 0 {
2194 self.pushed = true;
2195 saved_area_offset + 8
2196 } else {
2197 self.pushed = false;
2198 saved_area_offset
2199 };
2200 if self.compatible_imm(real_delta as _, ImmType::Bits12) {
2201 self.assembler.emit_sub(
2202 Size::S64,
2203 Location::GPR(GPR::X29),
2204 Location::Imm32(real_delta as _),
2205 Location::GPR(GPR::XzrSp),
2206 )?;
2207 } else {
2208 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2209 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2210 })?;
2211 self.assembler
2212 .emit_mov_imm(Location::GPR(tmp), real_delta as u64)?;
2213 self.assembler.emit_sub(
2214 Size::S64,
2215 Location::GPR(GPR::X29),
2216 Location::GPR(tmp),
2217 Location::GPR(GPR::XzrSp),
2218 )?;
2219 self.release_gpr(tmp);
2220 }
2221 Ok(())
2222 }
2223 fn pop_location(&mut self, location: Location) -> Result<(), CompileError> {
2225 self.emit_pop(Size::S64, location)
2226 }
2227 fn new_machine_state(&self) -> MachineState {
2229 new_machine_state()
2230 }
2231
2232 fn assembler_finalize(self) -> Result<Vec<u8>, CompileError> {
2234 self.assembler.finalize().map_err(|e| {
2235 CompileError::Codegen(format!("Assembler failed finalization with: {e:?}"))
2236 })
2237 }
2238
2239 fn get_offset(&self) -> Offset {
2240 self.assembler.get_offset()
2241 }
2242
2243 fn finalize_function(&mut self) -> Result<(), CompileError> {
2244 self.assembler.finalize_function();
2245 Ok(())
2246 }
2247
2248 fn emit_function_prolog(&mut self) -> Result<(), CompileError> {
2249 self.emit_double_push(Size::S64, Location::GPR(GPR::X29), Location::GPR(GPR::X30))?; self.emit_unwind_op(UnwindOps::Push2Regs {
2251 reg1: UnwindRegister::GPR(GPR::X29),
2252 reg2: UnwindRegister::GPR(GPR::X30),
2253 up_to_sp: 16,
2254 });
2255 self.emit_double_push(Size::S64, Location::GPR(GPR::X27), Location::GPR(GPR::X28))?;
2256 self.emit_unwind_op(UnwindOps::Push2Regs {
2257 reg1: UnwindRegister::GPR(GPR::X27),
2258 reg2: UnwindRegister::GPR(GPR::X28),
2259 up_to_sp: 32,
2260 });
2261 self.assembler.emit_add(
2263 Size::S64,
2264 Location::GPR(GPR::XzrSp),
2265 Location::Imm8(0),
2266 Location::GPR(GPR::X29),
2267 )?;
2268 self.emit_unwind_op(UnwindOps::DefineNewFrame);
2269 Ok(())
2270 }
2271
2272 fn emit_function_epilog(&mut self) -> Result<(), CompileError> {
2273 self.assembler.emit_add(
2275 Size::S64,
2276 Location::GPR(GPR::X29),
2277 Location::Imm8(0),
2278 Location::GPR(GPR::XzrSp),
2279 )?;
2280 self.pushed = false; self.emit_double_pop(Size::S64, Location::GPR(GPR::X27), Location::GPR(GPR::X28))?;
2282 self.emit_double_pop(Size::S64, Location::GPR(GPR::X29), Location::GPR(GPR::X30))?;
2283 Ok(())
2284 }
2285
2286 fn emit_function_return_value(
2287 &mut self,
2288 ty: WpType,
2289 canonicalize: bool,
2290 loc: Location,
2291 ) -> Result<(), CompileError> {
2292 if canonicalize {
2293 self.canonicalize_nan(
2294 match ty {
2295 WpType::F32 => Size::S32,
2296 WpType::F64 => Size::S64,
2297 _ => unreachable!(),
2298 },
2299 loc,
2300 Location::GPR(GPR::X0),
2301 )?;
2302 } else {
2303 self.emit_relaxed_mov(Size::S64, loc, Location::GPR(GPR::X0))?;
2304 }
2305 Ok(())
2306 }
2307
2308 fn emit_function_return_float(&mut self) -> Result<(), CompileError> {
2309 self.assembler
2310 .emit_mov(Size::S64, Location::GPR(GPR::X0), Location::SIMD(NEON::V0))
2311 }
2312
2313 fn arch_supports_canonicalize_nan(&self) -> bool {
2314 self.assembler.arch_supports_canonicalize_nan()
2315 }
2316 fn canonicalize_nan(
2317 &mut self,
2318 sz: Size,
2319 input: Location,
2320 output: Location,
2321 ) -> Result<(), CompileError> {
2322 let mut tempn = vec![];
2323 let mut temps = vec![];
2324 let old_fpcr = self.set_default_nan(&mut temps)?;
2325 match (sz, input, output) {
2327 (Size::S32, Location::SIMD(_), Location::SIMD(_)) => {
2328 self.assembler.emit_fmax(sz, input, input, output)?;
2329 }
2330 (Size::S64, Location::SIMD(_), Location::SIMD(_)) => {
2331 self.assembler.emit_fmax(sz, input, input, output)?;
2332 }
2333 (Size::S32, Location::SIMD(_), _) | (Size::S64, Location::SIMD(_), _) => {
2334 let tmp = self.location_to_neon(sz, output, &mut tempn, ImmType::None, false)?;
2335 self.assembler.emit_fmax(sz, input, input, tmp)?;
2336 self.move_location(sz, tmp, output)?;
2337 }
2338 (Size::S32, Location::Memory(_, _), _) | (Size::S64, Location::Memory(_, _), _) => {
2339 let src = self.location_to_neon(sz, input, &mut tempn, ImmType::None, true)?;
2340 let tmp = self.location_to_neon(sz, output, &mut tempn, ImmType::None, false)?;
2341 self.assembler.emit_fmax(sz, src, src, tmp)?;
2342 if tmp != output {
2343 self.move_location(sz, tmp, output)?;
2344 }
2345 }
2346 _ => codegen_error!(
2347 "singlepass can't emit canonicalize_nan {:?} {:?} {:?}",
2348 sz,
2349 input,
2350 output
2351 ),
2352 }
2353
2354 self.restore_fpcr(old_fpcr)?;
2355 for r in temps {
2356 self.release_gpr(r);
2357 }
2358 for r in tempn {
2359 self.release_simd(r);
2360 }
2361 Ok(())
2362 }
2363
2364 fn emit_illegal_op(&mut self, trap: TrapCode) -> Result<(), CompileError> {
2365 let offset = self.assembler.get_offset().0;
2366 self.assembler.emit_udf(0xc0 | (trap as u8) as u16)?;
2367 self.mark_instruction_address_end(offset);
2368 Ok(())
2369 }
2370 fn get_label(&mut self) -> Label {
2371 self.assembler.new_dynamic_label()
2372 }
2373 fn emit_label(&mut self, label: Label) -> Result<(), CompileError> {
2374 self.assembler.emit_label(label)
2375 }
2376 fn get_grp_for_call(&self) -> GPR {
2377 GPR::X27
2378 }
2379 fn emit_call_register(&mut self, reg: GPR) -> Result<(), CompileError> {
2380 self.assembler.emit_call_register(reg)
2381 }
2382 fn emit_call_label(&mut self, label: Label) -> Result<(), CompileError> {
2383 self.assembler.emit_call_label(label)
2384 }
2385 fn get_gpr_for_ret(&self) -> GPR {
2386 GPR::X0
2387 }
2388 fn get_simd_for_ret(&self) -> NEON {
2389 NEON::V0
2390 }
2391
2392 fn arch_requires_indirect_call_trampoline(&self) -> bool {
2393 self.assembler.arch_requires_indirect_call_trampoline()
2394 }
2395
2396 fn arch_emit_indirect_call_with_trampoline(
2397 &mut self,
2398 location: Location,
2399 ) -> Result<(), CompileError> {
2400 self.assembler
2401 .arch_emit_indirect_call_with_trampoline(location)
2402 }
2403
2404 fn emit_debug_breakpoint(&mut self) -> Result<(), CompileError> {
2405 self.assembler.emit_brk()
2406 }
2407
2408 fn emit_call_location(&mut self, location: Location) -> Result<(), CompileError> {
2409 let mut temps = vec![];
2410 let loc = self.location_to_reg(
2411 Size::S64,
2412 location,
2413 &mut temps,
2414 ImmType::None,
2415 true,
2416 Some(GPR::X27),
2417 )?;
2418 match loc {
2419 Location::GPR(reg) => self.assembler.emit_call_register(reg),
2420 _ => codegen_error!("singlepass can't emit CALL Location"),
2421 }?;
2422 for r in temps {
2423 self.release_gpr(r);
2424 }
2425 Ok(())
2426 }
2427
2428 fn location_address(
2429 &mut self,
2430 _size: Size,
2431 _source: Location,
2432 _dest: Location,
2433 ) -> Result<(), CompileError> {
2434 codegen_error!("singlepass location_address not implemented")
2435 }
2436 fn location_and(
2438 &mut self,
2439 _size: Size,
2440 _source: Location,
2441 _dest: Location,
2442 _flags: bool,
2443 ) -> Result<(), CompileError> {
2444 codegen_error!("singlepass location_and not implemented")
2445 }
2446 fn location_xor(
2447 &mut self,
2448 _size: Size,
2449 _source: Location,
2450 _dest: Location,
2451 _flags: bool,
2452 ) -> Result<(), CompileError> {
2453 codegen_error!("singlepass location_xor not implemented")
2454 }
2455 fn location_or(
2456 &mut self,
2457 _size: Size,
2458 _source: Location,
2459 _dest: Location,
2460 _flags: bool,
2461 ) -> Result<(), CompileError> {
2462 codegen_error!("singlepass location_or not implemented")
2463 }
2464 fn location_test(
2465 &mut self,
2466 _size: Size,
2467 _source: Location,
2468 _dest: Location,
2469 ) -> Result<(), CompileError> {
2470 codegen_error!("singlepass location_test not implemented")
2471 }
2472 fn location_add(
2474 &mut self,
2475 size: Size,
2476 source: Location,
2477 dest: Location,
2478 flags: bool,
2479 ) -> Result<(), CompileError> {
2480 let mut temps = vec![];
2481 let src = self.location_to_reg(size, source, &mut temps, ImmType::Bits12, true, None)?;
2482 let dst = self.location_to_reg(size, dest, &mut temps, ImmType::None, true, None)?;
2483 if flags {
2484 self.assembler.emit_adds(size, dst, src, dst)?;
2485 } else {
2486 self.assembler.emit_add(size, dst, src, dst)?;
2487 }
2488 if dst != dest {
2489 self.move_location(size, dst, dest)?;
2490 }
2491 for r in temps {
2492 self.release_gpr(r);
2493 }
2494 Ok(())
2495 }
2496 fn location_sub(
2497 &mut self,
2498 size: Size,
2499 source: Location,
2500 dest: Location,
2501 flags: bool,
2502 ) -> Result<(), CompileError> {
2503 let mut temps = vec![];
2504 let src = self.location_to_reg(size, source, &mut temps, ImmType::Bits12, true, None)?;
2505 let dst = self.location_to_reg(size, dest, &mut temps, ImmType::None, true, None)?;
2506 if flags {
2507 self.assembler.emit_subs(size, dst, src, dst)?;
2508 } else {
2509 self.assembler.emit_sub(size, dst, src, dst)?;
2510 }
2511 if dst != dest {
2512 self.move_location(size, dst, dest)?;
2513 }
2514 for r in temps {
2515 self.release_gpr(r);
2516 }
2517 Ok(())
2518 }
2519 fn location_cmp(
2520 &mut self,
2521 size: Size,
2522 source: Location,
2523 dest: Location,
2524 ) -> Result<(), CompileError> {
2525 self.emit_relaxed_binop(Assembler::emit_cmp, size, source, dest, false)
2526 }
2527 fn jmp_unconditionnal(&mut self, label: Label) -> Result<(), CompileError> {
2528 self.assembler.emit_b_label(label)
2529 }
2530
2531 fn jmp_on_condition(
2532 &mut self,
2533 cond: UnsignedCondition,
2534 size: Size,
2535 loc_a: AbstractLocation<Self::GPR, Self::SIMD>,
2536 loc_b: AbstractLocation<Self::GPR, Self::SIMD>,
2537 label: Label,
2538 ) -> Result<(), CompileError> {
2539 self.emit_relaxed_binop(Assembler::emit_cmp, size, loc_b, loc_a, false)?;
2540 let cond = match cond {
2541 UnsignedCondition::Equal => Condition::Eq,
2542 UnsignedCondition::NotEqual => Condition::Ne,
2543 UnsignedCondition::Above => Condition::Hi,
2544 UnsignedCondition::AboveEqual => Condition::Cs,
2545 UnsignedCondition::Below => Condition::Cc,
2546 UnsignedCondition::BelowEqual => Condition::Ls,
2547 };
2548 self.assembler.emit_bcond_label_far(cond, label)
2549 }
2550
2551 fn emit_jmp_to_jumptable(&mut self, label: Label, cond: Location) -> Result<(), CompileError> {
2553 let tmp1 = self.acquire_temp_gpr().ok_or_else(|| {
2554 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2555 })?;
2556 let tmp2 = self.acquire_temp_gpr().ok_or_else(|| {
2557 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2558 })?;
2559
2560 self.assembler.emit_load_label(tmp1, label)?;
2561 self.move_location(Size::S32, cond, Location::GPR(tmp2))?;
2562
2563 self.assembler.emit_add_lsl(
2564 Size::S64,
2565 Location::GPR(tmp1),
2566 Location::GPR(tmp2),
2567 2,
2568 Location::GPR(tmp2),
2569 )?;
2570 self.assembler.emit_b_register(tmp2)?;
2571 self.release_gpr(tmp2);
2572 self.release_gpr(tmp1);
2573 Ok(())
2574 }
2575
2576 fn align_for_loop(&mut self) -> Result<(), CompileError> {
2577 Ok(())
2579 }
2580
2581 fn emit_ret(&mut self) -> Result<(), CompileError> {
2582 self.assembler.emit_ret()
2583 }
2584
2585 fn emit_push(&mut self, size: Size, loc: Location) -> Result<(), CompileError> {
2586 self.emit_push(size, loc)
2587 }
2588 fn emit_pop(&mut self, size: Size, loc: Location) -> Result<(), CompileError> {
2589 self.emit_pop(size, loc)
2590 }
2591
2592 fn emit_memory_fence(&mut self) -> Result<(), CompileError> {
2593 self.assembler.emit_dmb()
2594 }
2595
2596 fn location_neg(
2597 &mut self,
2598 _size_val: Size, _signed: bool,
2600 _source: Location,
2601 _size_op: Size,
2602 _dest: Location,
2603 ) -> Result<(), CompileError> {
2604 codegen_error!("singlepass location_neg unimplemented");
2605 }
2606
2607 fn emit_imul_imm32(&mut self, size: Size, imm32: u32, gpr: GPR) -> Result<(), CompileError> {
2608 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2609 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2610 })?;
2611 self.assembler
2612 .emit_mov_imm(Location::GPR(tmp), imm32 as u64)?;
2613 self.assembler.emit_mul(
2614 size,
2615 Location::GPR(gpr),
2616 Location::GPR(tmp),
2617 Location::GPR(gpr),
2618 )?;
2619 self.release_gpr(tmp);
2620 Ok(())
2621 }
2622
2623 fn emit_relaxed_mov(
2625 &mut self,
2626 sz: Size,
2627 src: Location,
2628 dst: Location,
2629 ) -> Result<(), CompileError> {
2630 self.emit_relaxed_binop(Assembler::emit_mov, sz, src, dst, true)
2631 }
2632 fn emit_relaxed_cmp(
2633 &mut self,
2634 sz: Size,
2635 src: Location,
2636 dst: Location,
2637 ) -> Result<(), CompileError> {
2638 self.emit_relaxed_binop(Assembler::emit_cmp, sz, src, dst, false)
2639 }
2640 fn emit_relaxed_zero_extension(
2641 &mut self,
2642 _sz_src: Size,
2643 _src: Location,
2644 _sz_dst: Size,
2645 _dst: Location,
2646 ) -> Result<(), CompileError> {
2647 codegen_error!("singlepass emit_relaxed_zero_extension unimplemented");
2648 }
2649 fn emit_relaxed_sign_extension(
2650 &mut self,
2651 sz_src: Size,
2652 src: Location,
2653 sz_dst: Size,
2654 dst: Location,
2655 ) -> Result<(), CompileError> {
2656 match (src, dst) {
2657 (Location::Memory(_, _), Location::GPR(_)) => match sz_src {
2658 Size::S8 => self.emit_relaxed_ldr8s(sz_dst, dst, src),
2659 Size::S16 => self.emit_relaxed_ldr16s(sz_dst, dst, src),
2660 Size::S32 => self.emit_relaxed_ldr32s(sz_dst, dst, src),
2661 _ => codegen_error!("singlepass emit_relaxed_sign_extension unreachable"),
2662 },
2663 _ => {
2664 let mut temps = vec![];
2665 let src =
2666 self.location_to_reg(sz_src, src, &mut temps, ImmType::None, true, None)?;
2667 let dest =
2668 self.location_to_reg(sz_dst, dst, &mut temps, ImmType::None, false, None)?;
2669 match sz_src {
2670 Size::S8 => self.assembler.emit_sxtb(sz_dst, src, dest),
2671 Size::S16 => self.assembler.emit_sxth(sz_dst, src, dest),
2672 Size::S32 => self.assembler.emit_sxtw(sz_dst, src, dest),
2673 _ => codegen_error!("singlepass emit_relaxed_sign_extension unreachable"),
2674 }?;
2675 if dst != dest {
2676 self.move_location(sz_dst, dest, dst)?;
2677 }
2678 for r in temps {
2679 self.release_gpr(r);
2680 }
2681 Ok(())
2682 }
2683 }
2684 }
2685
2686 fn emit_binop_add32(
2687 &mut self,
2688 loc_a: Location,
2689 loc_b: Location,
2690 ret: Location,
2691 ) -> Result<(), CompileError> {
2692 self.emit_relaxed_binop3(
2693 Assembler::emit_add,
2694 Size::S32,
2695 loc_a,
2696 loc_b,
2697 ret,
2698 ImmType::Bits12,
2699 )
2700 }
2701 fn emit_binop_sub32(
2702 &mut self,
2703 loc_a: Location,
2704 loc_b: Location,
2705 ret: Location,
2706 ) -> Result<(), CompileError> {
2707 self.emit_relaxed_binop3(
2708 Assembler::emit_sub,
2709 Size::S32,
2710 loc_a,
2711 loc_b,
2712 ret,
2713 ImmType::Bits12,
2714 )
2715 }
2716 fn emit_binop_mul32(
2717 &mut self,
2718 loc_a: Location,
2719 loc_b: Location,
2720 ret: Location,
2721 ) -> Result<(), CompileError> {
2722 self.emit_relaxed_binop3(
2723 Assembler::emit_mul,
2724 Size::S32,
2725 loc_a,
2726 loc_b,
2727 ret,
2728 ImmType::None,
2729 )
2730 }
2731 fn emit_binop_udiv32(
2732 &mut self,
2733 loc_a: Location,
2734 loc_b: Location,
2735 ret: Location,
2736 integer_division_by_zero: Label,
2737 _integer_overflow: Label,
2738 ) -> Result<usize, CompileError> {
2739 let mut temps = vec![];
2740 let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
2741 let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
2742 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
2743
2744 self.assembler
2745 .emit_cbz_label_far(Size::S32, src2, integer_division_by_zero)?;
2746 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
2747 self.assembler.emit_udiv(Size::S32, src1, src2, dest)?;
2748 if ret != dest {
2749 self.move_location(Size::S32, dest, ret)?;
2750 }
2751 for r in temps {
2752 self.release_gpr(r);
2753 }
2754 Ok(offset)
2755 }
2756 fn emit_binop_sdiv32(
2757 &mut self,
2758 loc_a: Location,
2759 loc_b: Location,
2760 ret: Location,
2761 integer_division_by_zero: Label,
2762 integer_overflow: Label,
2763 ) -> Result<usize, CompileError> {
2764 let mut temps = vec![];
2765 let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
2766 let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
2767 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
2768
2769 self.assembler
2770 .emit_cbz_label_far(Size::S32, src2, integer_division_by_zero)?;
2771 let label_nooverflow = self.assembler.get_label();
2772 let tmp = self.location_to_reg(
2773 Size::S32,
2774 Location::Imm32(0x80000000),
2775 &mut temps,
2776 ImmType::None,
2777 true,
2778 None,
2779 )?;
2780 self.assembler.emit_cmp(Size::S32, tmp, src1)?;
2781 self.assembler
2782 .emit_bcond_label(Condition::Ne, label_nooverflow)?;
2783 self.assembler.emit_movn(Size::S32, tmp, 0)?;
2784 self.assembler.emit_cmp(Size::S32, tmp, src2)?;
2785 self.assembler
2786 .emit_bcond_label_far(Condition::Eq, integer_overflow)?;
2787 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
2788 self.assembler.emit_label(label_nooverflow)?;
2789 self.assembler.emit_sdiv(Size::S32, src1, src2, dest)?;
2790 if ret != dest {
2791 self.move_location(Size::S32, dest, ret)?;
2792 }
2793 for r in temps {
2794 self.release_gpr(r);
2795 }
2796 Ok(offset)
2797 }
2798 fn emit_binop_urem32(
2799 &mut self,
2800 loc_a: Location,
2801 loc_b: Location,
2802 ret: Location,
2803 integer_division_by_zero: Label,
2804 _integer_overflow: Label,
2805 ) -> Result<usize, CompileError> {
2806 let mut temps = vec![];
2807 let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
2808 let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
2809 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
2810 let dest = if dest == src1 || dest == src2 {
2811 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2812 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2813 })?;
2814 temps.push(tmp);
2815 self.assembler
2816 .emit_mov(Size::S32, dest, Location::GPR(tmp))?;
2817 Location::GPR(tmp)
2818 } else {
2819 dest
2820 };
2821 self.assembler
2822 .emit_cbz_label_far(Size::S32, src2, integer_division_by_zero)?;
2823 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
2824 self.assembler.emit_udiv(Size::S32, src1, src2, dest)?;
2825 self.assembler
2827 .emit_msub(Size::S32, dest, src2, src1, dest)?;
2828 if ret != dest {
2829 self.move_location(Size::S32, dest, ret)?;
2830 }
2831 for r in temps {
2832 self.release_gpr(r);
2833 }
2834 Ok(offset)
2835 }
2836 fn emit_binop_srem32(
2837 &mut self,
2838 loc_a: Location,
2839 loc_b: Location,
2840 ret: Location,
2841 integer_division_by_zero: Label,
2842 _integer_overflow: Label,
2843 ) -> Result<usize, CompileError> {
2844 let mut temps = vec![];
2845 let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
2846 let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
2847 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
2848 let dest = if dest == src1 || dest == src2 {
2849 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2850 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2851 })?;
2852 temps.push(tmp);
2853 self.assembler
2854 .emit_mov(Size::S32, dest, Location::GPR(tmp))?;
2855 Location::GPR(tmp)
2856 } else {
2857 dest
2858 };
2859 self.assembler
2860 .emit_cbz_label_far(Size::S32, src2, integer_division_by_zero)?;
2861 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
2862 self.assembler.emit_sdiv(Size::S32, src1, src2, dest)?;
2863 self.assembler
2865 .emit_msub(Size::S32, dest, src2, src1, dest)?;
2866 if ret != dest {
2867 self.move_location(Size::S32, dest, ret)?;
2868 }
2869 for r in temps {
2870 self.release_gpr(r);
2871 }
2872 Ok(offset)
2873 }
2874 fn emit_binop_and32(
2875 &mut self,
2876 loc_a: Location,
2877 loc_b: Location,
2878 ret: Location,
2879 ) -> Result<(), CompileError> {
2880 self.emit_relaxed_binop3(
2881 Assembler::emit_and,
2882 Size::S32,
2883 loc_a,
2884 loc_b,
2885 ret,
2886 ImmType::Logical32,
2887 )
2888 }
2889 fn emit_binop_or32(
2890 &mut self,
2891 loc_a: Location,
2892 loc_b: Location,
2893 ret: Location,
2894 ) -> Result<(), CompileError> {
2895 self.emit_relaxed_binop3(
2896 Assembler::emit_or,
2897 Size::S32,
2898 loc_a,
2899 loc_b,
2900 ret,
2901 ImmType::Logical32,
2902 )
2903 }
2904 fn emit_binop_xor32(
2905 &mut self,
2906 loc_a: Location,
2907 loc_b: Location,
2908 ret: Location,
2909 ) -> Result<(), CompileError> {
2910 self.emit_relaxed_binop3(
2911 Assembler::emit_eor,
2912 Size::S32,
2913 loc_a,
2914 loc_b,
2915 ret,
2916 ImmType::Logical32,
2917 )
2918 }
2919 fn i32_cmp_ge_s(
2920 &mut self,
2921 loc_a: Location,
2922 loc_b: Location,
2923 ret: Location,
2924 ) -> Result<(), CompileError> {
2925 self.emit_cmpop_i32_dynamic_b(Condition::Ge, loc_a, loc_b, ret)
2926 }
2927 fn i32_cmp_gt_s(
2928 &mut self,
2929 loc_a: Location,
2930 loc_b: Location,
2931 ret: Location,
2932 ) -> Result<(), CompileError> {
2933 self.emit_cmpop_i32_dynamic_b(Condition::Gt, loc_a, loc_b, ret)
2934 }
2935 fn i32_cmp_le_s(
2936 &mut self,
2937 loc_a: Location,
2938 loc_b: Location,
2939 ret: Location,
2940 ) -> Result<(), CompileError> {
2941 self.emit_cmpop_i32_dynamic_b(Condition::Le, loc_a, loc_b, ret)
2942 }
2943 fn i32_cmp_lt_s(
2944 &mut self,
2945 loc_a: Location,
2946 loc_b: Location,
2947 ret: Location,
2948 ) -> Result<(), CompileError> {
2949 self.emit_cmpop_i32_dynamic_b(Condition::Lt, loc_a, loc_b, ret)
2950 }
2951 fn i32_cmp_ge_u(
2952 &mut self,
2953 loc_a: Location,
2954 loc_b: Location,
2955 ret: Location,
2956 ) -> Result<(), CompileError> {
2957 self.emit_cmpop_i32_dynamic_b(Condition::Cs, loc_a, loc_b, ret)
2958 }
2959 fn i32_cmp_gt_u(
2960 &mut self,
2961 loc_a: Location,
2962 loc_b: Location,
2963 ret: Location,
2964 ) -> Result<(), CompileError> {
2965 self.emit_cmpop_i32_dynamic_b(Condition::Hi, loc_a, loc_b, ret)
2966 }
2967 fn i32_cmp_le_u(
2968 &mut self,
2969 loc_a: Location,
2970 loc_b: Location,
2971 ret: Location,
2972 ) -> Result<(), CompileError> {
2973 self.emit_cmpop_i32_dynamic_b(Condition::Ls, loc_a, loc_b, ret)
2974 }
2975 fn i32_cmp_lt_u(
2976 &mut self,
2977 loc_a: Location,
2978 loc_b: Location,
2979 ret: Location,
2980 ) -> Result<(), CompileError> {
2981 self.emit_cmpop_i32_dynamic_b(Condition::Cc, loc_a, loc_b, ret)
2982 }
2983 fn i32_cmp_ne(
2984 &mut self,
2985 loc_a: Location,
2986 loc_b: Location,
2987 ret: Location,
2988 ) -> Result<(), CompileError> {
2989 self.emit_cmpop_i32_dynamic_b(Condition::Ne, loc_a, loc_b, ret)
2990 }
2991 fn i32_cmp_eq(
2992 &mut self,
2993 loc_a: Location,
2994 loc_b: Location,
2995 ret: Location,
2996 ) -> Result<(), CompileError> {
2997 self.emit_cmpop_i32_dynamic_b(Condition::Eq, loc_a, loc_b, ret)
2998 }
2999 fn i32_clz(&mut self, src: Location, dst: Location) -> Result<(), CompileError> {
3000 self.emit_relaxed_binop(Assembler::emit_clz, Size::S32, src, dst, true)
3001 }
3002 fn i32_ctz(&mut self, src: Location, dst: Location) -> Result<(), CompileError> {
3003 let mut temps = vec![];
3004 let src = self.location_to_reg(Size::S32, src, &mut temps, ImmType::None, true, None)?;
3005 let dest = self.location_to_reg(Size::S32, dst, &mut temps, ImmType::None, false, None)?;
3006 self.assembler.emit_rbit(Size::S32, src, dest)?;
3007 self.assembler.emit_clz(Size::S32, dest, dest)?;
3008 if dst != dest {
3009 self.move_location(Size::S32, dest, dst)?;
3010 }
3011 for r in temps {
3012 self.release_gpr(r);
3013 }
3014 Ok(())
3015 }
3016 fn i32_popcnt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
3017 if self.has_neon {
3018 let mut temps = vec![];
3019
3020 let src_gpr =
3021 self.location_to_reg(Size::S32, loc, &mut temps, ImmType::None, true, None)?;
3022 let dst_gpr =
3023 self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3024
3025 let mut neon_temps = vec![];
3026 let neon_temp = self.acquire_temp_simd().ok_or_else(|| {
3027 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3028 })?;
3029 neon_temps.push(neon_temp);
3030
3031 self.assembler
3032 .emit_fmov(Size::S32, src_gpr, Size::S32, Location::SIMD(neon_temp))?;
3033 self.assembler.emit_cnt(neon_temp, neon_temp)?;
3034 self.assembler.emit_addv(neon_temp, neon_temp)?;
3035 self.assembler
3036 .emit_fmov(Size::S32, Location::SIMD(neon_temp), Size::S32, dst_gpr)?;
3037
3038 if ret != dst_gpr {
3039 self.move_location(Size::S32, dst_gpr, ret)?;
3040 }
3041
3042 for r in temps {
3043 self.release_gpr(r);
3044 }
3045
3046 for r in neon_temps {
3047 self.release_simd(r);
3048 }
3049 } else {
3050 let mut temps = vec![];
3051 let src =
3052 self.location_to_reg(Size::S32, loc, &mut temps, ImmType::None, true, None)?;
3053 let dest =
3054 self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3055 let src = if src == loc {
3056 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
3057 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3058 })?;
3059 temps.push(tmp);
3060 self.assembler
3061 .emit_mov(Size::S32, src, Location::GPR(tmp))?;
3062 Location::GPR(tmp)
3063 } else {
3064 src
3065 };
3066 let tmp = {
3067 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
3068 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3069 })?;
3070 temps.push(tmp);
3071 Location::GPR(tmp)
3072 };
3073 let label_loop = self.assembler.get_label();
3074 let label_exit = self.assembler.get_label();
3075 self.assembler
3076 .emit_mov(Size::S32, Location::GPR(GPR::XzrSp), dest)?; self.assembler.emit_cbz_label(Size::S32, src, label_exit)?; self.assembler.emit_label(label_loop)?; self.assembler
3080 .emit_add(Size::S32, dest, Location::Imm8(1), dest)?; self.assembler.emit_clz(Size::S32, src, tmp)?; self.assembler.emit_lsl(Size::S32, src, tmp, src)?; self.assembler
3084 .emit_lsl(Size::S32, src, Location::Imm8(1), src)?; self.assembler.emit_cbnz_label(Size::S32, src, label_loop)?; self.assembler.emit_label(label_exit)?;
3087 if ret != dest {
3088 self.move_location(Size::S32, dest, ret)?;
3089 }
3090 for r in temps {
3091 self.release_gpr(r);
3092 }
3093 }
3094 Ok(())
3095 }
3096 fn i32_shl(
3097 &mut self,
3098 loc_a: Location,
3099 loc_b: Location,
3100 ret: Location,
3101 ) -> Result<(), CompileError> {
3102 self.emit_relaxed_binop3(
3103 Assembler::emit_lsl,
3104 Size::S32,
3105 loc_a,
3106 loc_b,
3107 ret,
3108 ImmType::Shift32No0,
3109 )
3110 }
3111 fn i32_shr(
3112 &mut self,
3113 loc_a: Location,
3114 loc_b: Location,
3115 ret: Location,
3116 ) -> Result<(), CompileError> {
3117 self.emit_relaxed_binop3(
3118 Assembler::emit_lsr,
3119 Size::S32,
3120 loc_a,
3121 loc_b,
3122 ret,
3123 ImmType::Shift32No0,
3124 )
3125 }
3126 fn i32_sar(
3127 &mut self,
3128 loc_a: Location,
3129 loc_b: Location,
3130 ret: Location,
3131 ) -> Result<(), CompileError> {
3132 self.emit_relaxed_binop3(
3133 Assembler::emit_asr,
3134 Size::S32,
3135 loc_a,
3136 loc_b,
3137 ret,
3138 ImmType::Shift32No0,
3139 )
3140 }
3141 fn i32_rol(
3142 &mut self,
3143 loc_a: Location,
3144 loc_b: Location,
3145 ret: Location,
3146 ) -> Result<(), CompileError> {
3147 let mut temps = vec![];
3148 let src2 = match loc_b {
3149 Location::Imm8(imm) => Location::Imm8(32 - (imm & 31)),
3150 Location::Imm32(imm) => Location::Imm8(32 - (imm & 31) as u8),
3151 Location::Imm64(imm) => Location::Imm8(32 - (imm & 31) as u8),
3152 _ => {
3153 let tmp1 = self.location_to_reg(
3154 Size::S32,
3155 Location::Imm32(32),
3156 &mut temps,
3157 ImmType::None,
3158 true,
3159 None,
3160 )?;
3161 let tmp2 =
3162 self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
3163 self.assembler.emit_sub(Size::S32, tmp1, tmp2, tmp1)?;
3164 tmp1
3165 }
3166 };
3167 self.emit_relaxed_binop3(
3168 Assembler::emit_ror,
3169 Size::S32,
3170 loc_a,
3171 src2,
3172 ret,
3173 ImmType::Shift32No0,
3174 )?;
3175 for r in temps {
3176 self.release_gpr(r);
3177 }
3178 Ok(())
3179 }
3180 fn i32_ror(
3181 &mut self,
3182 loc_a: Location,
3183 loc_b: Location,
3184 ret: Location,
3185 ) -> Result<(), CompileError> {
3186 self.emit_relaxed_binop3(
3187 Assembler::emit_ror,
3188 Size::S32,
3189 loc_a,
3190 loc_b,
3191 ret,
3192 ImmType::Shift32No0,
3193 )
3194 }
3195 fn i32_load(
3196 &mut self,
3197 addr: Location,
3198 memarg: &MemArg,
3199 ret: Location,
3200 need_check: bool,
3201 imported_memories: bool,
3202 offset: i32,
3203 heap_access_oob: Label,
3204 unaligned_atomic: Label,
3205 ) -> Result<(), CompileError> {
3206 self.memory_op(
3207 addr,
3208 memarg,
3209 false,
3210 4,
3211 need_check,
3212 imported_memories,
3213 offset,
3214 heap_access_oob,
3215 unaligned_atomic,
3216 |this, addr| this.emit_relaxed_ldr32(Size::S32, ret, Location::Memory(addr, 0)),
3217 )
3218 }
3219 fn i32_load_8u(
3220 &mut self,
3221 addr: Location,
3222 memarg: &MemArg,
3223 ret: Location,
3224 need_check: bool,
3225 imported_memories: bool,
3226 offset: i32,
3227 heap_access_oob: Label,
3228 unaligned_atomic: Label,
3229 ) -> Result<(), CompileError> {
3230 self.memory_op(
3231 addr,
3232 memarg,
3233 false,
3234 1,
3235 need_check,
3236 imported_memories,
3237 offset,
3238 heap_access_oob,
3239 unaligned_atomic,
3240 |this, addr| this.emit_relaxed_ldr8(Size::S32, ret, Location::Memory(addr, 0)),
3241 )
3242 }
3243 fn i32_load_8s(
3244 &mut self,
3245 addr: Location,
3246 memarg: &MemArg,
3247 ret: Location,
3248 need_check: bool,
3249 imported_memories: bool,
3250 offset: i32,
3251 heap_access_oob: Label,
3252 unaligned_atomic: Label,
3253 ) -> Result<(), CompileError> {
3254 self.memory_op(
3255 addr,
3256 memarg,
3257 false,
3258 1,
3259 need_check,
3260 imported_memories,
3261 offset,
3262 heap_access_oob,
3263 unaligned_atomic,
3264 |this, addr| this.emit_relaxed_ldr8s(Size::S32, ret, Location::Memory(addr, 0)),
3265 )
3266 }
3267 fn i32_load_16u(
3268 &mut self,
3269 addr: Location,
3270 memarg: &MemArg,
3271 ret: Location,
3272 need_check: bool,
3273 imported_memories: bool,
3274 offset: i32,
3275 heap_access_oob: Label,
3276 unaligned_atomic: Label,
3277 ) -> Result<(), CompileError> {
3278 self.memory_op(
3279 addr,
3280 memarg,
3281 false,
3282 2,
3283 need_check,
3284 imported_memories,
3285 offset,
3286 heap_access_oob,
3287 unaligned_atomic,
3288 |this, addr| this.emit_relaxed_ldr16(Size::S32, ret, Location::Memory(addr, 0)),
3289 )
3290 }
3291 fn i32_load_16s(
3292 &mut self,
3293 addr: Location,
3294 memarg: &MemArg,
3295 ret: Location,
3296 need_check: bool,
3297 imported_memories: bool,
3298 offset: i32,
3299 heap_access_oob: Label,
3300 unaligned_atomic: Label,
3301 ) -> Result<(), CompileError> {
3302 self.memory_op(
3303 addr,
3304 memarg,
3305 false,
3306 2,
3307 need_check,
3308 imported_memories,
3309 offset,
3310 heap_access_oob,
3311 unaligned_atomic,
3312 |this, addr| this.emit_relaxed_ldr16s(Size::S32, ret, Location::Memory(addr, 0)),
3313 )
3314 }
3315 fn i32_atomic_load(
3316 &mut self,
3317 addr: Location,
3318 memarg: &MemArg,
3319 ret: Location,
3320 need_check: bool,
3321 imported_memories: bool,
3322 offset: i32,
3323 heap_access_oob: Label,
3324 unaligned_atomic: Label,
3325 ) -> Result<(), CompileError> {
3326 self.memory_op(
3327 addr,
3328 memarg,
3329 true,
3330 4,
3331 need_check,
3332 imported_memories,
3333 offset,
3334 heap_access_oob,
3335 unaligned_atomic,
3336 |this, addr| this.emit_relaxed_ldr32(Size::S32, ret, Location::Memory(addr, 0)),
3337 )
3338 }
3339 fn i32_atomic_load_8u(
3340 &mut self,
3341 addr: Location,
3342 memarg: &MemArg,
3343 ret: Location,
3344 need_check: bool,
3345 imported_memories: bool,
3346 offset: i32,
3347 heap_access_oob: Label,
3348 unaligned_atomic: Label,
3349 ) -> Result<(), CompileError> {
3350 self.memory_op(
3351 addr,
3352 memarg,
3353 true,
3354 1,
3355 need_check,
3356 imported_memories,
3357 offset,
3358 heap_access_oob,
3359 unaligned_atomic,
3360 |this, addr| this.emit_relaxed_ldr8(Size::S32, ret, Location::Memory(addr, 0)),
3361 )
3362 }
3363 fn i32_atomic_load_16u(
3364 &mut self,
3365 addr: Location,
3366 memarg: &MemArg,
3367 ret: Location,
3368 need_check: bool,
3369 imported_memories: bool,
3370 offset: i32,
3371 heap_access_oob: Label,
3372 unaligned_atomic: Label,
3373 ) -> Result<(), CompileError> {
3374 self.memory_op(
3375 addr,
3376 memarg,
3377 true,
3378 2,
3379 need_check,
3380 imported_memories,
3381 offset,
3382 heap_access_oob,
3383 unaligned_atomic,
3384 |this, addr| this.emit_relaxed_ldr16(Size::S32, ret, Location::Memory(addr, 0)),
3385 )
3386 }
3387 fn i32_save(
3388 &mut self,
3389 target_value: Location,
3390 memarg: &MemArg,
3391 target_addr: Location,
3392 need_check: bool,
3393 imported_memories: bool,
3394 offset: i32,
3395 heap_access_oob: Label,
3396 unaligned_atomic: Label,
3397 ) -> Result<(), CompileError> {
3398 self.memory_op(
3399 target_addr,
3400 memarg,
3401 false,
3402 4,
3403 need_check,
3404 imported_memories,
3405 offset,
3406 heap_access_oob,
3407 unaligned_atomic,
3408 |this, addr| this.emit_relaxed_str32(target_value, Location::Memory(addr, 0)),
3409 )
3410 }
3411 fn i32_save_8(
3412 &mut self,
3413 target_value: Location,
3414 memarg: &MemArg,
3415 target_addr: Location,
3416 need_check: bool,
3417 imported_memories: bool,
3418 offset: i32,
3419 heap_access_oob: Label,
3420 unaligned_atomic: Label,
3421 ) -> Result<(), CompileError> {
3422 self.memory_op(
3423 target_addr,
3424 memarg,
3425 false,
3426 4,
3427 need_check,
3428 imported_memories,
3429 offset,
3430 heap_access_oob,
3431 unaligned_atomic,
3432 |this, addr| this.emit_relaxed_str8(target_value, Location::Memory(addr, 0)),
3433 )
3434 }
3435 fn i32_save_16(
3436 &mut self,
3437 target_value: Location,
3438 memarg: &MemArg,
3439 target_addr: Location,
3440 need_check: bool,
3441 imported_memories: bool,
3442 offset: i32,
3443 heap_access_oob: Label,
3444 unaligned_atomic: Label,
3445 ) -> Result<(), CompileError> {
3446 self.memory_op(
3447 target_addr,
3448 memarg,
3449 false,
3450 4,
3451 need_check,
3452 imported_memories,
3453 offset,
3454 heap_access_oob,
3455 unaligned_atomic,
3456 |this, addr| this.emit_relaxed_str16(target_value, Location::Memory(addr, 0)),
3457 )
3458 }
3459 fn i32_atomic_save(
3460 &mut self,
3461 target_value: Location,
3462 memarg: &MemArg,
3463 target_addr: Location,
3464 need_check: bool,
3465 imported_memories: bool,
3466 offset: i32,
3467 heap_access_oob: Label,
3468 unaligned_atomic: Label,
3469 ) -> Result<(), CompileError> {
3470 self.memory_op(
3471 target_addr,
3472 memarg,
3473 true,
3474 4,
3475 need_check,
3476 imported_memories,
3477 offset,
3478 heap_access_oob,
3479 unaligned_atomic,
3480 |this, addr| this.emit_relaxed_str32(target_value, Location::Memory(addr, 0)),
3481 )?;
3482 self.assembler.emit_dmb()
3483 }
3484 fn i32_atomic_save_8(
3485 &mut self,
3486 target_value: Location,
3487 memarg: &MemArg,
3488 target_addr: Location,
3489 need_check: bool,
3490 imported_memories: bool,
3491 offset: i32,
3492 heap_access_oob: Label,
3493 unaligned_atomic: Label,
3494 ) -> Result<(), CompileError> {
3495 self.memory_op(
3496 target_addr,
3497 memarg,
3498 true,
3499 1,
3500 need_check,
3501 imported_memories,
3502 offset,
3503 heap_access_oob,
3504 unaligned_atomic,
3505 |this, addr| this.emit_relaxed_str8(target_value, Location::Memory(addr, 0)),
3506 )?;
3507 self.assembler.emit_dmb()
3508 }
3509 fn i32_atomic_save_16(
3510 &mut self,
3511 target_value: Location,
3512 memarg: &MemArg,
3513 target_addr: Location,
3514 need_check: bool,
3515 imported_memories: bool,
3516 offset: i32,
3517 heap_access_oob: Label,
3518 unaligned_atomic: Label,
3519 ) -> Result<(), CompileError> {
3520 self.memory_op(
3521 target_addr,
3522 memarg,
3523 true,
3524 2,
3525 need_check,
3526 imported_memories,
3527 offset,
3528 heap_access_oob,
3529 unaligned_atomic,
3530 |this, addr| this.emit_relaxed_str16(target_value, Location::Memory(addr, 0)),
3531 )?;
3532 self.assembler.emit_dmb()
3533 }
3534 fn i32_atomic_add(
3536 &mut self,
3537 loc: Location,
3538 target: Location,
3539 memarg: &MemArg,
3540 ret: Location,
3541 need_check: bool,
3542 imported_memories: bool,
3543 offset: i32,
3544 heap_access_oob: Label,
3545 unaligned_atomic: Label,
3546 ) -> Result<(), CompileError> {
3547 self.memory_op(
3548 target,
3549 memarg,
3550 true,
3551 4,
3552 need_check,
3553 imported_memories,
3554 offset,
3555 heap_access_oob,
3556 unaligned_atomic,
3557 |this, addr| {
3558 let mut temps = vec![];
3559 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3560 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3561 })?;
3562 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3563 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3564 })?;
3565 let dst =
3566 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3567 let reread = this.get_label();
3568
3569 this.emit_label(reread)?;
3570 this.assembler
3571 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
3572 this.emit_binop_add32(dst, loc, Location::GPR(tmp1))?;
3573 this.assembler.emit_stlxr(
3574 Size::S32,
3575 Location::GPR(tmp2),
3576 Location::GPR(tmp1),
3577 Location::GPR(addr),
3578 )?;
3579 this.assembler
3580 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3581 this.assembler.emit_dmb()?;
3582
3583 if dst != ret {
3584 this.move_location(Size::S32, ret, dst)?;
3585 }
3586 for r in temps {
3587 this.release_gpr(r);
3588 }
3589 this.release_gpr(tmp1);
3590 this.release_gpr(tmp2);
3591 Ok(())
3592 },
3593 )
3594 }
3595 fn i32_atomic_add_8u(
3597 &mut self,
3598 loc: Location,
3599 target: Location,
3600 memarg: &MemArg,
3601 ret: Location,
3602 need_check: bool,
3603 imported_memories: bool,
3604 offset: i32,
3605 heap_access_oob: Label,
3606 unaligned_atomic: Label,
3607 ) -> Result<(), CompileError> {
3608 self.memory_op(
3609 target,
3610 memarg,
3611 true,
3612 1,
3613 need_check,
3614 imported_memories,
3615 offset,
3616 heap_access_oob,
3617 unaligned_atomic,
3618 |this, addr| {
3619 let mut temps = vec![];
3620 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3621 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3622 })?;
3623 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3624 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3625 })?;
3626 let dst =
3627 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3628 let reread = this.get_label();
3629
3630 this.emit_label(reread)?;
3631 this.assembler
3632 .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?;
3633 this.emit_binop_add32(dst, loc, Location::GPR(tmp1))?;
3634 this.assembler.emit_stlxrb(
3635 Size::S32,
3636 Location::GPR(tmp2),
3637 Location::GPR(tmp1),
3638 Location::GPR(addr),
3639 )?;
3640 this.assembler
3641 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3642 this.assembler.emit_dmb()?;
3643
3644 if dst != ret {
3645 this.move_location(Size::S32, ret, dst)?;
3646 }
3647 for r in temps {
3648 this.release_gpr(r);
3649 }
3650 this.release_gpr(tmp1);
3651 this.release_gpr(tmp2);
3652 Ok(())
3653 },
3654 )
3655 }
3656 fn i32_atomic_add_16u(
3658 &mut self,
3659 loc: Location,
3660 target: Location,
3661 memarg: &MemArg,
3662 ret: Location,
3663 need_check: bool,
3664 imported_memories: bool,
3665 offset: i32,
3666 heap_access_oob: Label,
3667 unaligned_atomic: Label,
3668 ) -> Result<(), CompileError> {
3669 self.memory_op(
3670 target,
3671 memarg,
3672 true,
3673 2,
3674 need_check,
3675 imported_memories,
3676 offset,
3677 heap_access_oob,
3678 unaligned_atomic,
3679 |this, addr| {
3680 let mut temps = vec![];
3681 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3682 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3683 })?;
3684 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3685 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3686 })?;
3687 let dst =
3688 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3689 let reread = this.get_label();
3690
3691 this.emit_label(reread)?;
3692 this.assembler
3693 .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?;
3694 this.emit_binop_add32(dst, loc, Location::GPR(tmp1))?;
3695 this.assembler.emit_stlxrh(
3696 Size::S32,
3697 Location::GPR(tmp2),
3698 Location::GPR(tmp1),
3699 Location::GPR(addr),
3700 )?;
3701 this.assembler
3702 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3703 this.assembler.emit_dmb()?;
3704
3705 if dst != ret {
3706 this.move_location(Size::S32, ret, dst)?;
3707 }
3708 for r in temps {
3709 this.release_gpr(r);
3710 }
3711 this.release_gpr(tmp1);
3712 this.release_gpr(tmp2);
3713 Ok(())
3714 },
3715 )
3716 }
3717 fn i32_atomic_sub(
3719 &mut self,
3720 loc: Location,
3721 target: Location,
3722 memarg: &MemArg,
3723 ret: Location,
3724 need_check: bool,
3725 imported_memories: bool,
3726 offset: i32,
3727 heap_access_oob: Label,
3728 unaligned_atomic: Label,
3729 ) -> Result<(), CompileError> {
3730 self.memory_op(
3731 target,
3732 memarg,
3733 true,
3734 4,
3735 need_check,
3736 imported_memories,
3737 offset,
3738 heap_access_oob,
3739 unaligned_atomic,
3740 |this, addr| {
3741 let mut temps = vec![];
3742 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3743 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3744 })?;
3745 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3746 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3747 })?;
3748 let dst =
3749 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3750 let reread = this.get_label();
3751
3752 this.emit_label(reread)?;
3753 this.assembler
3754 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
3755 this.emit_binop_sub32(dst, loc, Location::GPR(tmp1))?;
3756 this.assembler.emit_stlxr(
3757 Size::S32,
3758 Location::GPR(tmp2),
3759 Location::GPR(tmp1),
3760 Location::GPR(addr),
3761 )?;
3762 this.assembler
3763 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3764 this.assembler.emit_dmb()?;
3765
3766 if dst != ret {
3767 this.move_location(Size::S32, ret, dst)?;
3768 }
3769 for r in temps {
3770 this.release_gpr(r);
3771 }
3772 this.release_gpr(tmp1);
3773 this.release_gpr(tmp2);
3774 Ok(())
3775 },
3776 )
3777 }
3778 fn i32_atomic_sub_8u(
3780 &mut self,
3781 loc: Location,
3782 target: Location,
3783 memarg: &MemArg,
3784 ret: Location,
3785 need_check: bool,
3786 imported_memories: bool,
3787 offset: i32,
3788 heap_access_oob: Label,
3789 unaligned_atomic: Label,
3790 ) -> Result<(), CompileError> {
3791 self.memory_op(
3792 target,
3793 memarg,
3794 true,
3795 1,
3796 need_check,
3797 imported_memories,
3798 offset,
3799 heap_access_oob,
3800 unaligned_atomic,
3801 |this, addr| {
3802 let mut temps = vec![];
3803 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3804 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3805 })?;
3806 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3807 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3808 })?;
3809 let dst =
3810 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3811 let reread = this.get_label();
3812
3813 this.emit_label(reread)?;
3814 this.assembler
3815 .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?;
3816 this.emit_binop_sub32(dst, loc, Location::GPR(tmp1))?;
3817 this.assembler.emit_stlxrb(
3818 Size::S32,
3819 Location::GPR(tmp2),
3820 Location::GPR(tmp1),
3821 Location::GPR(addr),
3822 )?;
3823 this.assembler
3824 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3825 this.assembler.emit_dmb()?;
3826
3827 if dst != ret {
3828 this.move_location(Size::S32, ret, dst)?;
3829 }
3830 for r in temps {
3831 this.release_gpr(r);
3832 }
3833 this.release_gpr(tmp1);
3834 this.release_gpr(tmp2);
3835 Ok(())
3836 },
3837 )
3838 }
3839 fn i32_atomic_sub_16u(
3841 &mut self,
3842 loc: Location,
3843 target: Location,
3844 memarg: &MemArg,
3845 ret: Location,
3846 need_check: bool,
3847 imported_memories: bool,
3848 offset: i32,
3849 heap_access_oob: Label,
3850 unaligned_atomic: Label,
3851 ) -> Result<(), CompileError> {
3852 self.memory_op(
3853 target,
3854 memarg,
3855 true,
3856 2,
3857 need_check,
3858 imported_memories,
3859 offset,
3860 heap_access_oob,
3861 unaligned_atomic,
3862 |this, addr| {
3863 let mut temps = vec![];
3864 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3865 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3866 })?;
3867 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3868 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3869 })?;
3870 let dst =
3871 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3872 let reread = this.get_label();
3873
3874 this.emit_label(reread)?;
3875 this.assembler
3876 .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?;
3877 this.emit_binop_sub32(dst, loc, Location::GPR(tmp1))?;
3878 this.assembler.emit_stlxrh(
3879 Size::S32,
3880 Location::GPR(tmp2),
3881 Location::GPR(tmp1),
3882 Location::GPR(addr),
3883 )?;
3884 this.assembler
3885 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3886 this.assembler.emit_dmb()?;
3887
3888 if dst != ret {
3889 this.move_location(Size::S32, ret, dst)?;
3890 }
3891 for r in temps {
3892 this.release_gpr(r);
3893 }
3894 this.release_gpr(tmp1);
3895 this.release_gpr(tmp2);
3896 Ok(())
3897 },
3898 )
3899 }
3900 fn i32_atomic_and(
3902 &mut self,
3903 loc: Location,
3904 target: Location,
3905 memarg: &MemArg,
3906 ret: Location,
3907 need_check: bool,
3908 imported_memories: bool,
3909 offset: i32,
3910 heap_access_oob: Label,
3911 unaligned_atomic: Label,
3912 ) -> Result<(), CompileError> {
3913 self.memory_op(
3914 target,
3915 memarg,
3916 true,
3917 4,
3918 need_check,
3919 imported_memories,
3920 offset,
3921 heap_access_oob,
3922 unaligned_atomic,
3923 |this, addr| {
3924 let mut temps = vec![];
3925 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3926 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3927 })?;
3928 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3929 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3930 })?;
3931 let dst =
3932 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3933 let reread = this.get_label();
3934
3935 this.emit_label(reread)?;
3936 this.assembler
3937 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
3938 this.emit_binop_and32(dst, loc, Location::GPR(tmp1))?;
3939 this.assembler.emit_stlxr(
3940 Size::S32,
3941 Location::GPR(tmp2),
3942 Location::GPR(tmp1),
3943 Location::GPR(addr),
3944 )?;
3945 this.assembler
3946 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
3947 this.assembler.emit_dmb()?;
3948
3949 if dst != ret {
3950 this.move_location(Size::S32, ret, dst)?;
3951 }
3952 for r in temps {
3953 this.release_gpr(r);
3954 }
3955 this.release_gpr(tmp1);
3956 this.release_gpr(tmp2);
3957 Ok(())
3958 },
3959 )
3960 }
3961 fn i32_atomic_and_8u(
3963 &mut self,
3964 loc: Location,
3965 target: Location,
3966 memarg: &MemArg,
3967 ret: Location,
3968 need_check: bool,
3969 imported_memories: bool,
3970 offset: i32,
3971 heap_access_oob: Label,
3972 unaligned_atomic: Label,
3973 ) -> Result<(), CompileError> {
3974 self.memory_op(
3975 target,
3976 memarg,
3977 true,
3978 1,
3979 need_check,
3980 imported_memories,
3981 offset,
3982 heap_access_oob,
3983 unaligned_atomic,
3984 |this, addr| {
3985 let mut temps = vec![];
3986 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
3987 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3988 })?;
3989 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
3990 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3991 })?;
3992 let dst =
3993 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3994 let reread = this.get_label();
3995
3996 this.emit_label(reread)?;
3997 this.assembler
3998 .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?;
3999 this.emit_binop_and32(dst, loc, Location::GPR(tmp1))?;
4000 this.assembler.emit_stlxrb(
4001 Size::S32,
4002 Location::GPR(tmp2),
4003 Location::GPR(tmp1),
4004 Location::GPR(addr),
4005 )?;
4006 this.assembler
4007 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
4008 this.assembler.emit_dmb()?;
4009
4010 if dst != ret {
4011 this.move_location(Size::S32, ret, dst)?;
4012 }
4013 for r in temps {
4014 this.release_gpr(r);
4015 }
4016 this.release_gpr(tmp1);
4017 this.release_gpr(tmp2);
4018 Ok(())
4019 },
4020 )
4021 }
4022 fn i32_atomic_and_16u(
4024 &mut self,
4025 loc: Location,
4026 target: Location,
4027 memarg: &MemArg,
4028 ret: Location,
4029 need_check: bool,
4030 imported_memories: bool,
4031 offset: i32,
4032 heap_access_oob: Label,
4033 unaligned_atomic: Label,
4034 ) -> Result<(), CompileError> {
4035 self.memory_op(
4036 target,
4037 memarg,
4038 true,
4039 2,
4040 need_check,
4041 imported_memories,
4042 offset,
4043 heap_access_oob,
4044 unaligned_atomic,
4045 |this, addr| {
4046 let mut temps = vec![];
4047 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
4048 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4049 })?;
4050 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
4051 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4052 })?;
4053 let dst =
4054 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4055 let reread = this.get_label();
4056
4057 this.emit_label(reread)?;
4058 this.assembler
4059 .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?;
4060 this.emit_binop_and32(dst, loc, Location::GPR(tmp1))?;
4061 this.assembler.emit_stlxrh(
4062 Size::S32,
4063 Location::GPR(tmp2),
4064 Location::GPR(tmp1),
4065 Location::GPR(addr),
4066 )?;
4067 this.assembler
4068 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
4069 this.assembler.emit_dmb()?;
4070
4071 if dst != ret {
4072 this.move_location(Size::S32, ret, dst)?;
4073 }
4074 for r in temps {
4075 this.release_gpr(r);
4076 }
4077 this.release_gpr(tmp1);
4078 this.release_gpr(tmp2);
4079 Ok(())
4080 },
4081 )
4082 }
4083 fn i32_atomic_or(
4085 &mut self,
4086 loc: Location,
4087 target: Location,
4088 memarg: &MemArg,
4089 ret: Location,
4090 need_check: bool,
4091 imported_memories: bool,
4092 offset: i32,
4093 heap_access_oob: Label,
4094 unaligned_atomic: Label,
4095 ) -> Result<(), CompileError> {
4096 self.memory_op(
4097 target,
4098 memarg,
4099 true,
4100 4,
4101 need_check,
4102 imported_memories,
4103 offset,
4104 heap_access_oob,
4105 unaligned_atomic,
4106 |this, addr| {
4107 let mut temps = vec![];
4108 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
4109 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4110 })?;
4111 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
4112 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4113 })?;
4114 let dst =
4115 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4116 let reread = this.get_label();
4117
4118 this.emit_label(reread)?;
4119 this.assembler
4120 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
4121 this.emit_binop_or32(dst, loc, Location::GPR(tmp1))?;
4122 this.assembler.emit_stlxr(
4123 Size::S32,
4124 Location::GPR(tmp2),
4125 Location::GPR(tmp1),
4126 Location::GPR(addr),
4127 )?;
4128 this.assembler
4129 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
4130 this.assembler.emit_dmb()?;
4131
4132 if dst != ret {
4133 this.move_location(Size::S32, ret, dst)?;
4134 }
4135 for r in temps {
4136 this.release_gpr(r);
4137 }
4138 this.release_gpr(tmp1);
4139 this.release_gpr(tmp2);
4140 Ok(())
4141 },
4142 )
4143 }
4144 fn i32_atomic_or_8u(
4146 &mut self,
4147 loc: Location,
4148 target: Location,
4149 memarg: &MemArg,
4150 ret: Location,
4151 need_check: bool,
4152 imported_memories: bool,
4153 offset: i32,
4154 heap_access_oob: Label,
4155 unaligned_atomic: Label,
4156 ) -> Result<(), CompileError> {
4157 self.memory_op(
4158 target,
4159 memarg,
4160 true,
4161 1,
4162 need_check,
4163 imported_memories,
4164 offset,
4165 heap_access_oob,
4166 unaligned_atomic,
4167 |this, addr| {
4168 let mut temps = vec![];
4169 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
4170 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4171 })?;
4172 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
4173 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4174 })?;
4175 let dst =
4176 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4177 let reread = this.get_label();
4178
4179 this.emit_label(reread)?;
4180 this.assembler
4181 .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?;
4182 this.emit_binop_or32(dst, loc, Location::GPR(tmp1))?;
4183 this.assembler.emit_stlxrb(
4184 Size::S32,
4185 Location::GPR(tmp2),
4186 Location::GPR(tmp1),
4187 Location::GPR(addr),
4188 )?;
4189 this.assembler
4190 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
4191 this.assembler.emit_dmb()?;
4192
4193 if dst != ret {
4194 this.move_location(Size::S32, ret, dst)?;
4195 }
4196 for r in temps {
4197 this.release_gpr(r);
4198 }
4199 this.release_gpr(tmp1);
4200 this.release_gpr(tmp2);
4201 Ok(())
4202 },
4203 )
4204 }
4205 fn i32_atomic_or_16u(
4207 &mut self,
4208 loc: Location,
4209 target: Location,
4210 memarg: &MemArg,
4211 ret: Location,
4212 need_check: bool,
4213 imported_memories: bool,
4214 offset: i32,
4215 heap_access_oob: Label,
4216 unaligned_atomic: Label,
4217 ) -> Result<(), CompileError> {
4218 self.memory_op(
4219 target,
4220 memarg,
4221 true,
4222 2,
4223 need_check,
4224 imported_memories,
4225 offset,
4226 heap_access_oob,
4227 unaligned_atomic,
4228 |this, addr| {
4229 let mut temps = vec![];
4230 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
4231 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4232 })?;
4233 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
4234 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4235 })?;
4236 let dst =
4237 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4238 let reread = this.get_label();
4239
4240 this.emit_label(reread)?;
4241 this.assembler
4242 .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?;
4243 this.emit_binop_or32(dst, loc, Location::GPR(tmp1))?;
4244 this.assembler.emit_stlxrh(
4245 Size::S32,
4246 Location::GPR(tmp2),
4247 Location::GPR(tmp1),
4248 Location::GPR(addr),
4249 )?;
4250 this.assembler
4251 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
4252 this.assembler.emit_dmb()?;
4253
4254 if dst != ret {
4255 this.move_location(Size::S32, ret, dst)?;
4256 }
4257 for r in temps {
4258 this.release_gpr(r);
4259 }
4260 this.release_gpr(tmp1);
4261 this.release_gpr(tmp2);
4262 Ok(())
4263 },
4264 )
4265 }
4266 fn i32_atomic_xor(
4268 &mut self,
4269 loc: Location,
4270 target: Location,
4271 memarg: &MemArg,
4272 ret: Location,
4273 need_check: bool,
4274 imported_memories: bool,
4275 offset: i32,
4276 heap_access_oob: Label,
4277 unaligned_atomic: Label,
4278 ) -> Result<(), CompileError> {
4279 self.memory_op(
4280 target,
4281 memarg,
4282 true,
4283 4,
4284 need_check,
4285 imported_memories,
4286 offset,
4287 heap_access_oob,
4288 unaligned_atomic,
4289 |this, addr| {
4290 let mut temps = vec![];
4291 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
4292 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4293 })?;
4294 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
4295 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4296 })?;
4297 let dst =
4298 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4299 let reread = this.get_label();
4300
4301 this.emit_label(reread)?;
4302 this.assembler
4303 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
4304 this.emit_binop_xor32(dst, loc, Location::GPR(tmp1))?;
4305 this.assembler.emit_stlxr(
4306 Size::S32,
4307 Location::GPR(tmp2),
4308 Location::GPR(tmp1),
4309 Location::GPR(addr),
4310 )?;
4311 this.assembler
4312 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
4313 this.assembler.emit_dmb()?;
4314
4315 if dst != ret {
4316 this.move_location(Size::S32, ret, dst)?;
4317 }
4318 for r in temps {
4319 this.release_gpr(r);
4320 }
4321 this.release_gpr(tmp1);
4322 this.release_gpr(tmp2);
4323 Ok(())
4324 },
4325 )
4326 }
4327 fn i32_atomic_xor_8u(
4329 &mut self,
4330 loc: Location,
4331 target: Location,
4332 memarg: &MemArg,
4333 ret: Location,
4334 need_check: bool,
4335 imported_memories: bool,
4336 offset: i32,
4337 heap_access_oob: Label,
4338 unaligned_atomic: Label,
4339 ) -> Result<(), CompileError> {
4340 self.memory_op(
4341 target,
4342 memarg,
4343 true,
4344 1,
4345 need_check,
4346 imported_memories,
4347 offset,
4348 heap_access_oob,
4349 unaligned_atomic,
4350 |this, addr| {
4351 let mut temps = vec![];
4352 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
4353 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4354 })?;
4355 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
4356 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4357 })?;
4358 let dst =
4359 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4360 let reread = this.get_label();
4361
4362 this.emit_label(reread)?;
4363 this.assembler
4364 .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?;
4365 this.emit_binop_xor32(dst, loc, Location::GPR(tmp1))?;
4366 this.assembler.emit_stlxrb(
4367 Size::S32,
4368 Location::GPR(tmp2),
4369 Location::GPR(tmp1),
4370 Location::GPR(addr),
4371 )?;
4372 this.assembler
4373 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
4374 this.assembler.emit_dmb()?;
4375
4376 if dst != ret {
4377 this.move_location(Size::S32, ret, dst)?;
4378 }
4379 for r in temps {
4380 this.release_gpr(r);
4381 }
4382 this.release_gpr(tmp1);
4383 this.release_gpr(tmp2);
4384 Ok(())
4385 },
4386 )
4387 }
4388 fn i32_atomic_xor_16u(
4390 &mut self,
4391 loc: Location,
4392 target: Location,
4393 memarg: &MemArg,
4394 ret: Location,
4395 need_check: bool,
4396 imported_memories: bool,
4397 offset: i32,
4398 heap_access_oob: Label,
4399 unaligned_atomic: Label,
4400 ) -> Result<(), CompileError> {
4401 self.memory_op(
4402 target,
4403 memarg,
4404 true,
4405 2,
4406 need_check,
4407 imported_memories,
4408 offset,
4409 heap_access_oob,
4410 unaligned_atomic,
4411 |this, addr| {
4412 let mut temps = vec![];
4413 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
4414 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4415 })?;
4416 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
4417 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4418 })?;
4419 let dst =
4420 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4421 let reread = this.get_label();
4422
4423 this.emit_label(reread)?;
4424 this.assembler
4425 .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?;
4426 this.emit_binop_xor32(dst, loc, Location::GPR(tmp1))?;
4427 this.assembler.emit_stlxrh(
4428 Size::S32,
4429 Location::GPR(tmp2),
4430 Location::GPR(tmp1),
4431 Location::GPR(addr),
4432 )?;
4433 this.assembler
4434 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
4435 this.assembler.emit_dmb()?;
4436
4437 if dst != ret {
4438 this.move_location(Size::S32, ret, dst)?;
4439 }
4440 for r in temps {
4441 this.release_gpr(r);
4442 }
4443 this.release_gpr(tmp1);
4444 this.release_gpr(tmp2);
4445 Ok(())
4446 },
4447 )
4448 }
4449 fn i32_atomic_xchg(
4451 &mut self,
4452 loc: Location,
4453 target: Location,
4454 memarg: &MemArg,
4455 ret: Location,
4456 need_check: bool,
4457 imported_memories: bool,
4458 offset: i32,
4459 heap_access_oob: Label,
4460 unaligned_atomic: Label,
4461 ) -> Result<(), CompileError> {
4462 self.memory_op(
4463 target,
4464 memarg,
4465 true,
4466 4,
4467 need_check,
4468 imported_memories,
4469 offset,
4470 heap_access_oob,
4471 unaligned_atomic,
4472 |this, addr| {
4473 let mut temps = vec![];
4474 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
4475 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4476 })?;
4477 let dst =
4478 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4479 let org =
4480 this.location_to_reg(Size::S32, loc, &mut temps, ImmType::None, false, None)?;
4481 let reread = this.get_label();
4482
4483 this.emit_label(reread)?;
4484 this.assembler
4485 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
4486 this.assembler.emit_stlxr(
4487 Size::S32,
4488 Location::GPR(tmp),
4489 org,
4490 Location::GPR(addr),
4491 )?;
4492 this.assembler
4493 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
4494 this.assembler.emit_dmb()?;
4495
4496 if dst != ret {
4497 this.move_location(Size::S32, ret, dst)?;
4498 }
4499 for r in temps {
4500 this.release_gpr(r);
4501 }
4502 this.release_gpr(tmp);
4503 Ok(())
4504 },
4505 )
4506 }
4507 fn i32_atomic_xchg_8u(
4509 &mut self,
4510 loc: Location,
4511 target: Location,
4512 memarg: &MemArg,
4513 ret: Location,
4514 need_check: bool,
4515 imported_memories: bool,
4516 offset: i32,
4517 heap_access_oob: Label,
4518 unaligned_atomic: Label,
4519 ) -> Result<(), CompileError> {
4520 self.memory_op(
4521 target,
4522 memarg,
4523 true,
4524 1,
4525 need_check,
4526 imported_memories,
4527 offset,
4528 heap_access_oob,
4529 unaligned_atomic,
4530 |this, addr| {
4531 let mut temps = vec![];
4532 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
4533 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4534 })?;
4535 let dst =
4536 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4537 let org =
4538 this.location_to_reg(Size::S32, loc, &mut temps, ImmType::None, false, None)?;
4539 let reread = this.get_label();
4540
4541 this.emit_label(reread)?;
4542 this.assembler
4543 .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?;
4544 this.assembler.emit_stlxrb(
4545 Size::S32,
4546 Location::GPR(tmp),
4547 org,
4548 Location::GPR(addr),
4549 )?;
4550 this.assembler
4551 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
4552 this.assembler.emit_dmb()?;
4553
4554 if dst != ret {
4555 this.move_location(Size::S32, ret, dst)?;
4556 }
4557 for r in temps {
4558 this.release_gpr(r);
4559 }
4560 this.release_gpr(tmp);
4561 Ok(())
4562 },
4563 )
4564 }
4565 fn i32_atomic_xchg_16u(
4567 &mut self,
4568 loc: Location,
4569 target: Location,
4570 memarg: &MemArg,
4571 ret: Location,
4572 need_check: bool,
4573 imported_memories: bool,
4574 offset: i32,
4575 heap_access_oob: Label,
4576 unaligned_atomic: Label,
4577 ) -> Result<(), CompileError> {
4578 self.memory_op(
4579 target,
4580 memarg,
4581 true,
4582 2,
4583 need_check,
4584 imported_memories,
4585 offset,
4586 heap_access_oob,
4587 unaligned_atomic,
4588 |this, addr| {
4589 let mut temps = vec![];
4590 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
4591 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4592 })?;
4593 let dst =
4594 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4595 let org =
4596 this.location_to_reg(Size::S32, loc, &mut temps, ImmType::None, false, None)?;
4597 let reread = this.get_label();
4598
4599 this.emit_label(reread)?;
4600 this.assembler
4601 .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?;
4602 this.assembler.emit_stlxrh(
4603 Size::S32,
4604 Location::GPR(tmp),
4605 org,
4606 Location::GPR(addr),
4607 )?;
4608 this.assembler
4609 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
4610 this.assembler.emit_dmb()?;
4611
4612 if dst != ret {
4613 this.move_location(Size::S32, ret, dst)?;
4614 }
4615 for r in temps {
4616 this.release_gpr(r);
4617 }
4618 this.release_gpr(tmp);
4619 Ok(())
4620 },
4621 )
4622 }
4623 fn i32_atomic_cmpxchg(
4625 &mut self,
4626 new: Location,
4627 cmp: Location,
4628 target: Location,
4629 memarg: &MemArg,
4630 ret: Location,
4631 need_check: bool,
4632 imported_memories: bool,
4633 offset: i32,
4634 heap_access_oob: Label,
4635 unaligned_atomic: Label,
4636 ) -> Result<(), CompileError> {
4637 self.memory_op(
4638 target,
4639 memarg,
4640 true,
4641 4,
4642 need_check,
4643 imported_memories,
4644 offset,
4645 heap_access_oob,
4646 unaligned_atomic,
4647 |this, addr| {
4648 let mut temps = vec![];
4649 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
4650 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4651 })?;
4652 let dst =
4653 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4654 let org =
4655 this.location_to_reg(Size::S32, new, &mut temps, ImmType::None, false, None)?;
4656 let reread = this.get_label();
4657 let nosame = this.get_label();
4658
4659 this.emit_label(reread)?;
4660 this.assembler
4661 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
4662 this.emit_relaxed_cmp(Size::S32, dst, cmp)?;
4663 this.assembler.emit_bcond_label(Condition::Ne, nosame)?;
4664 this.assembler.emit_stlxr(
4665 Size::S32,
4666 Location::GPR(tmp),
4667 org,
4668 Location::GPR(addr),
4669 )?;
4670 this.assembler
4671 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
4672 this.assembler.emit_dmb()?;
4673
4674 this.emit_label(nosame)?;
4675 if dst != ret {
4676 this.move_location(Size::S32, ret, dst)?;
4677 }
4678 for r in temps {
4679 this.release_gpr(r);
4680 }
4681 this.release_gpr(tmp);
4682 Ok(())
4683 },
4684 )
4685 }
4686 fn i32_atomic_cmpxchg_8u(
4688 &mut self,
4689 new: Location,
4690 cmp: Location,
4691 target: Location,
4692 memarg: &MemArg,
4693 ret: Location,
4694 need_check: bool,
4695 imported_memories: bool,
4696 offset: i32,
4697 heap_access_oob: Label,
4698 unaligned_atomic: Label,
4699 ) -> Result<(), CompileError> {
4700 self.memory_op(
4701 target,
4702 memarg,
4703 true,
4704 1,
4705 need_check,
4706 imported_memories,
4707 offset,
4708 heap_access_oob,
4709 unaligned_atomic,
4710 |this, addr| {
4711 let mut temps = vec![];
4712 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
4713 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4714 })?;
4715 let dst =
4716 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4717 let org =
4718 this.location_to_reg(Size::S32, new, &mut temps, ImmType::None, false, None)?;
4719 let reread = this.get_label();
4720 let nosame = this.get_label();
4721
4722 this.emit_label(reread)?;
4723 this.assembler
4724 .emit_ldaxrb(Size::S32, dst, Location::GPR(addr))?;
4725 this.emit_relaxed_cmp(Size::S32, dst, cmp)?;
4726 this.assembler.emit_bcond_label(Condition::Ne, nosame)?;
4727 this.assembler.emit_stlxrb(
4728 Size::S32,
4729 Location::GPR(tmp),
4730 org,
4731 Location::GPR(addr),
4732 )?;
4733 this.assembler
4734 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
4735 this.assembler.emit_dmb()?;
4736
4737 this.emit_label(nosame)?;
4738 if dst != ret {
4739 this.move_location(Size::S32, ret, dst)?;
4740 }
4741 for r in temps {
4742 this.release_gpr(r);
4743 }
4744 this.release_gpr(tmp);
4745 Ok(())
4746 },
4747 )
4748 }
4749 fn i32_atomic_cmpxchg_16u(
4751 &mut self,
4752 new: Location,
4753 cmp: Location,
4754 target: Location,
4755 memarg: &MemArg,
4756 ret: Location,
4757 need_check: bool,
4758 imported_memories: bool,
4759 offset: i32,
4760 heap_access_oob: Label,
4761 unaligned_atomic: Label,
4762 ) -> Result<(), CompileError> {
4763 self.memory_op(
4764 target,
4765 memarg,
4766 true,
4767 2,
4768 need_check,
4769 imported_memories,
4770 offset,
4771 heap_access_oob,
4772 unaligned_atomic,
4773 |this, addr| {
4774 let mut temps = vec![];
4775 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
4776 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4777 })?;
4778 let dst =
4779 this.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
4780 let org =
4781 this.location_to_reg(Size::S32, new, &mut temps, ImmType::None, false, None)?;
4782 let reread = this.get_label();
4783 let nosame = this.get_label();
4784
4785 this.emit_label(reread)?;
4786 this.assembler
4787 .emit_ldaxrh(Size::S32, dst, Location::GPR(addr))?;
4788 this.emit_relaxed_cmp(Size::S32, dst, cmp)?;
4789 this.assembler.emit_bcond_label(Condition::Ne, nosame)?;
4790 this.assembler.emit_stlxrh(
4791 Size::S32,
4792 Location::GPR(tmp),
4793 org,
4794 Location::GPR(addr),
4795 )?;
4796 this.assembler
4797 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
4798 this.assembler.emit_dmb()?;
4799
4800 this.emit_label(nosame)?;
4801 if dst != ret {
4802 this.move_location(Size::S32, ret, dst)?;
4803 }
4804 for r in temps {
4805 this.release_gpr(r);
4806 }
4807 this.release_gpr(tmp);
4808 Ok(())
4809 },
4810 )
4811 }
4812
4813 fn emit_call_with_reloc(
4814 &mut self,
4815 _calling_convention: CallingConvention,
4816 reloc_target: RelocationTarget,
4817 ) -> Result<Vec<Relocation>, CompileError> {
4818 let mut relocations = vec![];
4819 let next = self.get_label();
4820 let reloc_at = self.assembler.get_offset().0;
4821 self.emit_label(next)?; self.assembler.emit_call_label(next)?;
4823 relocations.push(Relocation {
4824 kind: RelocationKind::Arm64Call,
4825 reloc_target,
4826 offset: reloc_at as u32,
4827 addend: 0,
4828 });
4829 Ok(relocations)
4830 }
4831
4832 fn emit_binop_add64(
4833 &mut self,
4834 loc_a: Location,
4835 loc_b: Location,
4836 ret: Location,
4837 ) -> Result<(), CompileError> {
4838 self.emit_relaxed_binop3(
4839 Assembler::emit_add,
4840 Size::S64,
4841 loc_a,
4842 loc_b,
4843 ret,
4844 ImmType::Bits12,
4845 )
4846 }
4847 fn emit_binop_sub64(
4848 &mut self,
4849 loc_a: Location,
4850 loc_b: Location,
4851 ret: Location,
4852 ) -> Result<(), CompileError> {
4853 self.emit_relaxed_binop3(
4854 Assembler::emit_sub,
4855 Size::S64,
4856 loc_a,
4857 loc_b,
4858 ret,
4859 ImmType::Bits12,
4860 )
4861 }
4862 fn emit_binop_mul64(
4863 &mut self,
4864 loc_a: Location,
4865 loc_b: Location,
4866 ret: Location,
4867 ) -> Result<(), CompileError> {
4868 self.emit_relaxed_binop3(
4869 Assembler::emit_mul,
4870 Size::S64,
4871 loc_a,
4872 loc_b,
4873 ret,
4874 ImmType::None,
4875 )
4876 }
4877 fn emit_binop_udiv64(
4878 &mut self,
4879 loc_a: Location,
4880 loc_b: Location,
4881 ret: Location,
4882 integer_division_by_zero: Label,
4883 _integer_overflow: Label,
4884 ) -> Result<usize, CompileError> {
4885 let mut temps = vec![];
4886 let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4887 let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4888 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4889
4890 self.assembler
4891 .emit_cbz_label(Size::S64, src2, integer_division_by_zero)?;
4892 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
4893 self.assembler.emit_udiv(Size::S64, src1, src2, dest)?;
4894 if ret != dest {
4895 self.move_location(Size::S64, dest, ret)?;
4896 }
4897 for r in temps {
4898 self.release_gpr(r);
4899 }
4900 Ok(offset)
4901 }
4902 fn emit_binop_sdiv64(
4903 &mut self,
4904 loc_a: Location,
4905 loc_b: Location,
4906 ret: Location,
4907 integer_division_by_zero: Label,
4908 integer_overflow: Label,
4909 ) -> Result<usize, CompileError> {
4910 let mut temps = vec![];
4911 let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4912 let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4913 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4914
4915 self.assembler
4916 .emit_cbz_label_far(Size::S64, src2, integer_division_by_zero)?;
4917 let label_nooverflow = self.assembler.get_label();
4918 let tmp = self.location_to_reg(
4919 Size::S64,
4920 Location::Imm64(0x8000000000000000),
4921 &mut temps,
4922 ImmType::None,
4923 true,
4924 None,
4925 )?;
4926 self.assembler.emit_cmp(Size::S64, tmp, src1)?;
4927 self.assembler
4928 .emit_bcond_label(Condition::Ne, label_nooverflow)?;
4929 self.assembler.emit_movn(Size::S64, tmp, 0)?;
4930 self.assembler.emit_cmp(Size::S64, tmp, src2)?;
4931 self.assembler
4932 .emit_bcond_label_far(Condition::Eq, integer_overflow)?;
4933 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
4934 self.assembler.emit_label(label_nooverflow)?;
4935 self.assembler.emit_sdiv(Size::S64, src1, src2, dest)?;
4936 if ret != dest {
4937 self.move_location(Size::S64, dest, ret)?;
4938 }
4939 for r in temps {
4940 self.release_gpr(r);
4941 }
4942 Ok(offset)
4943 }
4944 fn emit_binop_urem64(
4945 &mut self,
4946 loc_a: Location,
4947 loc_b: Location,
4948 ret: Location,
4949 integer_division_by_zero: Label,
4950 _integer_overflow: Label,
4951 ) -> Result<usize, CompileError> {
4952 let mut temps = vec![];
4953 let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4954 let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4955 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4956 let dest = if dest == src1 || dest == src2 {
4957 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
4958 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4959 })?;
4960 temps.push(tmp);
4961 self.assembler
4962 .emit_mov(Size::S32, dest, Location::GPR(tmp))?;
4963 Location::GPR(tmp)
4964 } else {
4965 dest
4966 };
4967 self.assembler
4968 .emit_cbz_label_far(Size::S64, src2, integer_division_by_zero)?;
4969 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
4970 self.assembler.emit_udiv(Size::S64, src1, src2, dest)?;
4971 self.assembler
4973 .emit_msub(Size::S64, dest, src2, src1, dest)?;
4974 if ret != dest {
4975 self.move_location(Size::S64, dest, ret)?;
4976 }
4977 for r in temps {
4978 self.release_gpr(r);
4979 }
4980 Ok(offset)
4981 }
4982 fn emit_binop_srem64(
4983 &mut self,
4984 loc_a: Location,
4985 loc_b: Location,
4986 ret: Location,
4987 integer_division_by_zero: Label,
4988 _integer_overflow: Label,
4989 ) -> Result<usize, CompileError> {
4990 let mut temps = vec![];
4991 let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4992 let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4993 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4994 let dest = if dest == src1 || dest == src2 {
4995 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
4996 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4997 })?;
4998 temps.push(tmp);
4999 self.assembler
5000 .emit_mov(Size::S64, dest, Location::GPR(tmp))?;
5001 Location::GPR(tmp)
5002 } else {
5003 dest
5004 };
5005 self.assembler
5006 .emit_cbz_label_far(Size::S64, src2, integer_division_by_zero)?;
5007 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
5008 self.assembler.emit_sdiv(Size::S64, src1, src2, dest)?;
5009 self.assembler
5011 .emit_msub(Size::S64, dest, src2, src1, dest)?;
5012 if ret != dest {
5013 self.move_location(Size::S64, dest, ret)?;
5014 }
5015 for r in temps {
5016 self.release_gpr(r);
5017 }
5018 Ok(offset)
5019 }
5020 fn emit_binop_and64(
5021 &mut self,
5022 loc_a: Location,
5023 loc_b: Location,
5024 ret: Location,
5025 ) -> Result<(), CompileError> {
5026 self.emit_relaxed_binop3(
5027 Assembler::emit_and,
5028 Size::S64,
5029 loc_a,
5030 loc_b,
5031 ret,
5032 ImmType::Logical64,
5033 )
5034 }
5035 fn emit_binop_or64(
5036 &mut self,
5037 loc_a: Location,
5038 loc_b: Location,
5039 ret: Location,
5040 ) -> Result<(), CompileError> {
5041 self.emit_relaxed_binop3(
5042 Assembler::emit_or,
5043 Size::S64,
5044 loc_a,
5045 loc_b,
5046 ret,
5047 ImmType::Logical64,
5048 )
5049 }
5050 fn emit_binop_xor64(
5051 &mut self,
5052 loc_a: Location,
5053 loc_b: Location,
5054 ret: Location,
5055 ) -> Result<(), CompileError> {
5056 self.emit_relaxed_binop3(
5057 Assembler::emit_eor,
5058 Size::S64,
5059 loc_a,
5060 loc_b,
5061 ret,
5062 ImmType::Logical64,
5063 )
5064 }
5065 fn i64_cmp_ge_s(
5066 &mut self,
5067 loc_a: Location,
5068 loc_b: Location,
5069 ret: Location,
5070 ) -> Result<(), CompileError> {
5071 self.emit_cmpop_i64_dynamic_b(Condition::Ge, loc_a, loc_b, ret)
5072 }
5073 fn i64_cmp_gt_s(
5074 &mut self,
5075 loc_a: Location,
5076 loc_b: Location,
5077 ret: Location,
5078 ) -> Result<(), CompileError> {
5079 self.emit_cmpop_i64_dynamic_b(Condition::Gt, loc_a, loc_b, ret)
5080 }
5081 fn i64_cmp_le_s(
5082 &mut self,
5083 loc_a: Location,
5084 loc_b: Location,
5085 ret: Location,
5086 ) -> Result<(), CompileError> {
5087 self.emit_cmpop_i64_dynamic_b(Condition::Le, loc_a, loc_b, ret)
5088 }
5089 fn i64_cmp_lt_s(
5090 &mut self,
5091 loc_a: Location,
5092 loc_b: Location,
5093 ret: Location,
5094 ) -> Result<(), CompileError> {
5095 self.emit_cmpop_i64_dynamic_b(Condition::Lt, loc_a, loc_b, ret)
5096 }
5097 fn i64_cmp_ge_u(
5098 &mut self,
5099 loc_a: Location,
5100 loc_b: Location,
5101 ret: Location,
5102 ) -> Result<(), CompileError> {
5103 self.emit_cmpop_i64_dynamic_b(Condition::Cs, loc_a, loc_b, ret)
5104 }
5105 fn i64_cmp_gt_u(
5106 &mut self,
5107 loc_a: Location,
5108 loc_b: Location,
5109 ret: Location,
5110 ) -> Result<(), CompileError> {
5111 self.emit_cmpop_i64_dynamic_b(Condition::Hi, loc_a, loc_b, ret)
5112 }
5113 fn i64_cmp_le_u(
5114 &mut self,
5115 loc_a: Location,
5116 loc_b: Location,
5117 ret: Location,
5118 ) -> Result<(), CompileError> {
5119 self.emit_cmpop_i64_dynamic_b(Condition::Ls, loc_a, loc_b, ret)
5120 }
5121 fn i64_cmp_lt_u(
5122 &mut self,
5123 loc_a: Location,
5124 loc_b: Location,
5125 ret: Location,
5126 ) -> Result<(), CompileError> {
5127 self.emit_cmpop_i64_dynamic_b(Condition::Cc, loc_a, loc_b, ret)
5128 }
5129 fn i64_cmp_ne(
5130 &mut self,
5131 loc_a: Location,
5132 loc_b: Location,
5133 ret: Location,
5134 ) -> Result<(), CompileError> {
5135 self.emit_cmpop_i64_dynamic_b(Condition::Ne, loc_a, loc_b, ret)
5136 }
5137 fn i64_cmp_eq(
5138 &mut self,
5139 loc_a: Location,
5140 loc_b: Location,
5141 ret: Location,
5142 ) -> Result<(), CompileError> {
5143 self.emit_cmpop_i64_dynamic_b(Condition::Eq, loc_a, loc_b, ret)
5144 }
5145 fn i64_clz(&mut self, src: Location, dst: Location) -> Result<(), CompileError> {
5146 self.emit_relaxed_binop(Assembler::emit_clz, Size::S64, src, dst, true)
5147 }
5148 fn i64_ctz(&mut self, src: Location, dst: Location) -> Result<(), CompileError> {
5149 let mut temps = vec![];
5150 let src = self.location_to_reg(Size::S64, src, &mut temps, ImmType::None, true, None)?;
5151 let dest = self.location_to_reg(Size::S64, dst, &mut temps, ImmType::None, false, None)?;
5152 self.assembler.emit_rbit(Size::S64, src, dest)?;
5153 self.assembler.emit_clz(Size::S64, dest, dest)?;
5154 if dst != dest {
5155 self.move_location(Size::S64, dest, dst)?;
5156 }
5157 for r in temps {
5158 self.release_gpr(r);
5159 }
5160 Ok(())
5161 }
5162 fn i64_popcnt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
5163 if self.has_neon {
5164 let mut temps = vec![];
5165
5166 let src_gpr =
5167 self.location_to_reg(Size::S64, loc, &mut temps, ImmType::None, true, None)?;
5168 let dst_gpr =
5169 self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
5170
5171 let mut neon_temps = vec![];
5172 let neon_temp = self.acquire_temp_simd().ok_or_else(|| {
5173 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5174 })?;
5175 neon_temps.push(neon_temp);
5176
5177 self.assembler
5178 .emit_fmov(Size::S64, src_gpr, Size::S64, Location::SIMD(neon_temp))?;
5179 self.assembler.emit_cnt(neon_temp, neon_temp)?;
5180 self.assembler.emit_addv(neon_temp, neon_temp)?;
5181 self.assembler
5182 .emit_fmov(Size::S64, Location::SIMD(neon_temp), Size::S64, dst_gpr)?;
5183
5184 if ret != dst_gpr {
5185 self.move_location(Size::S64, dst_gpr, ret)?;
5186 }
5187
5188 for r in temps {
5189 self.release_gpr(r);
5190 }
5191
5192 for r in neon_temps {
5193 self.release_simd(r);
5194 }
5195 } else {
5196 let mut temps = vec![];
5197 let src =
5198 self.location_to_reg(Size::S64, loc, &mut temps, ImmType::None, true, None)?;
5199 let dest =
5200 self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
5201 let src = if src == loc {
5202 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
5203 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5204 })?;
5205 temps.push(tmp);
5206 self.assembler
5207 .emit_mov(Size::S64, src, Location::GPR(tmp))?;
5208 Location::GPR(tmp)
5209 } else {
5210 src
5211 };
5212 let tmp = {
5213 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
5214 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5215 })?;
5216 temps.push(tmp);
5217 Location::GPR(tmp)
5218 };
5219 let label_loop = self.assembler.get_label();
5220 let label_exit = self.assembler.get_label();
5221 self.assembler
5222 .emit_mov(Size::S32, Location::GPR(GPR::XzrSp), dest)?; self.assembler.emit_cbz_label(Size::S64, src, label_exit)?; self.assembler.emit_label(label_loop)?;
5225 self.assembler
5226 .emit_add(Size::S32, dest, Location::Imm8(1), dest)?; self.assembler.emit_clz(Size::S64, src, tmp)?; self.assembler.emit_lsl(Size::S64, src, tmp, src)?; self.assembler
5230 .emit_lsl(Size::S64, src, Location::Imm8(1), src)?; self.assembler.emit_cbnz_label(Size::S64, src, label_loop)?; self.assembler.emit_label(label_exit)?;
5233 if ret != dest {
5234 self.move_location(Size::S64, dest, ret)?;
5235 }
5236 for r in temps {
5237 self.release_gpr(r);
5238 }
5239 }
5240
5241 Ok(())
5242 }
5243 fn i64_shl(
5244 &mut self,
5245 loc_a: Location,
5246 loc_b: Location,
5247 ret: Location,
5248 ) -> Result<(), CompileError> {
5249 self.emit_relaxed_binop3(
5250 Assembler::emit_lsl,
5251 Size::S64,
5252 loc_a,
5253 loc_b,
5254 ret,
5255 ImmType::Shift64No0,
5256 )
5257 }
5258 fn i64_shr(
5259 &mut self,
5260 loc_a: Location,
5261 loc_b: Location,
5262 ret: Location,
5263 ) -> Result<(), CompileError> {
5264 self.emit_relaxed_binop3(
5265 Assembler::emit_lsr,
5266 Size::S64,
5267 loc_a,
5268 loc_b,
5269 ret,
5270 ImmType::Shift64No0,
5271 )
5272 }
5273 fn i64_sar(
5274 &mut self,
5275 loc_a: Location,
5276 loc_b: Location,
5277 ret: Location,
5278 ) -> Result<(), CompileError> {
5279 self.emit_relaxed_binop3(
5280 Assembler::emit_asr,
5281 Size::S64,
5282 loc_a,
5283 loc_b,
5284 ret,
5285 ImmType::Shift64No0,
5286 )
5287 }
5288 fn i64_rol(
5289 &mut self,
5290 loc_a: Location,
5291 loc_b: Location,
5292 ret: Location,
5293 ) -> Result<(), CompileError> {
5294 let mut temps = vec![];
5296 let src2 = match loc_b {
5297 Location::Imm8(imm) => Location::Imm8(64 - (imm & 63)),
5298 Location::Imm32(imm) => Location::Imm8(64 - (imm & 63) as u8),
5299 Location::Imm64(imm) => Location::Imm8(64 - (imm & 63) as u8),
5300 _ => {
5301 let tmp1 = self.location_to_reg(
5302 Size::S64,
5303 Location::Imm32(64),
5304 &mut temps,
5305 ImmType::None,
5306 true,
5307 None,
5308 )?;
5309 let tmp2 =
5310 self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
5311 self.assembler.emit_sub(Size::S64, tmp1, tmp2, tmp1)?;
5312 tmp1
5313 }
5314 };
5315 self.emit_relaxed_binop3(
5316 Assembler::emit_ror,
5317 Size::S64,
5318 loc_a,
5319 src2,
5320 ret,
5321 ImmType::Shift64No0,
5322 )?;
5323 for r in temps {
5324 self.release_gpr(r);
5325 }
5326 Ok(())
5327 }
5328 fn i64_ror(
5329 &mut self,
5330 loc_a: Location,
5331 loc_b: Location,
5332 ret: Location,
5333 ) -> Result<(), CompileError> {
5334 self.emit_relaxed_binop3(
5335 Assembler::emit_ror,
5336 Size::S64,
5337 loc_a,
5338 loc_b,
5339 ret,
5340 ImmType::Shift64No0,
5341 )
5342 }
5343 fn i64_load(
5344 &mut self,
5345 addr: Location,
5346 memarg: &MemArg,
5347 ret: Location,
5348 need_check: bool,
5349 imported_memories: bool,
5350 offset: i32,
5351 heap_access_oob: Label,
5352 unaligned_atomic: Label,
5353 ) -> Result<(), CompileError> {
5354 self.memory_op(
5355 addr,
5356 memarg,
5357 false,
5358 8,
5359 need_check,
5360 imported_memories,
5361 offset,
5362 heap_access_oob,
5363 unaligned_atomic,
5364 |this, addr| this.emit_relaxed_ldr64(Size::S64, ret, Location::Memory(addr, 0)),
5365 )
5366 }
5367 fn i64_load_8u(
5368 &mut self,
5369 addr: Location,
5370 memarg: &MemArg,
5371 ret: Location,
5372 need_check: bool,
5373 imported_memories: bool,
5374 offset: i32,
5375 heap_access_oob: Label,
5376 unaligned_atomic: Label,
5377 ) -> Result<(), CompileError> {
5378 self.memory_op(
5379 addr,
5380 memarg,
5381 false,
5382 1,
5383 need_check,
5384 imported_memories,
5385 offset,
5386 heap_access_oob,
5387 unaligned_atomic,
5388 |this, addr| this.emit_relaxed_ldr8(Size::S64, ret, Location::Memory(addr, 0)),
5389 )
5390 }
5391 fn i64_load_8s(
5392 &mut self,
5393 addr: Location,
5394 memarg: &MemArg,
5395 ret: Location,
5396 need_check: bool,
5397 imported_memories: bool,
5398 offset: i32,
5399 heap_access_oob: Label,
5400 unaligned_atomic: Label,
5401 ) -> Result<(), CompileError> {
5402 self.memory_op(
5403 addr,
5404 memarg,
5405 false,
5406 1,
5407 need_check,
5408 imported_memories,
5409 offset,
5410 heap_access_oob,
5411 unaligned_atomic,
5412 |this, addr| this.emit_relaxed_ldr8s(Size::S64, ret, Location::Memory(addr, 0)),
5413 )
5414 }
5415 fn i64_load_16u(
5416 &mut self,
5417 addr: Location,
5418 memarg: &MemArg,
5419 ret: Location,
5420 need_check: bool,
5421 imported_memories: bool,
5422 offset: i32,
5423 heap_access_oob: Label,
5424 unaligned_atomic: Label,
5425 ) -> Result<(), CompileError> {
5426 self.memory_op(
5427 addr,
5428 memarg,
5429 false,
5430 2,
5431 need_check,
5432 imported_memories,
5433 offset,
5434 heap_access_oob,
5435 unaligned_atomic,
5436 |this, addr| this.emit_relaxed_ldr16(Size::S64, ret, Location::Memory(addr, 0)),
5437 )
5438 }
5439 fn i64_load_16s(
5440 &mut self,
5441 addr: Location,
5442 memarg: &MemArg,
5443 ret: Location,
5444 need_check: bool,
5445 imported_memories: bool,
5446 offset: i32,
5447 heap_access_oob: Label,
5448 unaligned_atomic: Label,
5449 ) -> Result<(), CompileError> {
5450 self.memory_op(
5451 addr,
5452 memarg,
5453 false,
5454 2,
5455 need_check,
5456 imported_memories,
5457 offset,
5458 heap_access_oob,
5459 unaligned_atomic,
5460 |this, addr| this.emit_relaxed_ldr16s(Size::S64, ret, Location::Memory(addr, 0)),
5461 )
5462 }
5463 fn i64_load_32u(
5464 &mut self,
5465 addr: Location,
5466 memarg: &MemArg,
5467 ret: Location,
5468 need_check: bool,
5469 imported_memories: bool,
5470 offset: i32,
5471 heap_access_oob: Label,
5472 unaligned_atomic: Label,
5473 ) -> Result<(), CompileError> {
5474 self.memory_op(
5475 addr,
5476 memarg,
5477 false,
5478 4,
5479 need_check,
5480 imported_memories,
5481 offset,
5482 heap_access_oob,
5483 unaligned_atomic,
5484 |this, addr| this.emit_relaxed_ldr32(Size::S64, ret, Location::Memory(addr, 0)),
5485 )
5486 }
5487 fn i64_load_32s(
5488 &mut self,
5489 addr: Location,
5490 memarg: &MemArg,
5491 ret: Location,
5492 need_check: bool,
5493 imported_memories: bool,
5494 offset: i32,
5495 heap_access_oob: Label,
5496 unaligned_atomic: Label,
5497 ) -> Result<(), CompileError> {
5498 self.memory_op(
5499 addr,
5500 memarg,
5501 false,
5502 4,
5503 need_check,
5504 imported_memories,
5505 offset,
5506 heap_access_oob,
5507 unaligned_atomic,
5508 |this, addr| this.emit_relaxed_ldr32s(Size::S64, ret, Location::Memory(addr, 0)),
5509 )
5510 }
5511 fn i64_atomic_load(
5512 &mut self,
5513 addr: Location,
5514 memarg: &MemArg,
5515 ret: Location,
5516 need_check: bool,
5517 imported_memories: bool,
5518 offset: i32,
5519 heap_access_oob: Label,
5520 unaligned_atomic: Label,
5521 ) -> Result<(), CompileError> {
5522 self.memory_op(
5523 addr,
5524 memarg,
5525 true,
5526 8,
5527 need_check,
5528 imported_memories,
5529 offset,
5530 heap_access_oob,
5531 unaligned_atomic,
5532 |this, addr| this.emit_relaxed_ldr64(Size::S64, ret, Location::Memory(addr, 0)),
5533 )
5534 }
5535 fn i64_atomic_load_8u(
5536 &mut self,
5537 addr: Location,
5538 memarg: &MemArg,
5539 ret: Location,
5540 need_check: bool,
5541 imported_memories: bool,
5542 offset: i32,
5543 heap_access_oob: Label,
5544 unaligned_atomic: Label,
5545 ) -> Result<(), CompileError> {
5546 self.memory_op(
5547 addr,
5548 memarg,
5549 true,
5550 1,
5551 need_check,
5552 imported_memories,
5553 offset,
5554 heap_access_oob,
5555 unaligned_atomic,
5556 |this, addr| this.emit_relaxed_ldr8(Size::S64, ret, Location::Memory(addr, 0)),
5557 )
5558 }
5559 fn i64_atomic_load_16u(
5560 &mut self,
5561 addr: Location,
5562 memarg: &MemArg,
5563 ret: Location,
5564 need_check: bool,
5565 imported_memories: bool,
5566 offset: i32,
5567 heap_access_oob: Label,
5568 unaligned_atomic: Label,
5569 ) -> Result<(), CompileError> {
5570 self.memory_op(
5571 addr,
5572 memarg,
5573 true,
5574 2,
5575 need_check,
5576 imported_memories,
5577 offset,
5578 heap_access_oob,
5579 unaligned_atomic,
5580 |this, addr| this.emit_relaxed_ldr16(Size::S64, ret, Location::Memory(addr, 0)),
5581 )
5582 }
5583 fn i64_atomic_load_32u(
5584 &mut self,
5585 addr: Location,
5586 memarg: &MemArg,
5587 ret: Location,
5588 need_check: bool,
5589 imported_memories: bool,
5590 offset: i32,
5591 heap_access_oob: Label,
5592 unaligned_atomic: Label,
5593 ) -> Result<(), CompileError> {
5594 self.memory_op(
5595 addr,
5596 memarg,
5597 true,
5598 4,
5599 need_check,
5600 imported_memories,
5601 offset,
5602 heap_access_oob,
5603 unaligned_atomic,
5604 |this, addr| this.emit_relaxed_ldr32(Size::S64, ret, Location::Memory(addr, 0)),
5605 )
5606 }
5607 fn i64_save(
5608 &mut self,
5609 target_value: Location,
5610 memarg: &MemArg,
5611 target_addr: Location,
5612 need_check: bool,
5613 imported_memories: bool,
5614 offset: i32,
5615 heap_access_oob: Label,
5616 unaligned_atomic: Label,
5617 ) -> Result<(), CompileError> {
5618 self.memory_op(
5619 target_addr,
5620 memarg,
5621 false,
5622 8,
5623 need_check,
5624 imported_memories,
5625 offset,
5626 heap_access_oob,
5627 unaligned_atomic,
5628 |this, addr| this.emit_relaxed_str64(target_value, Location::Memory(addr, 0)),
5629 )
5630 }
5631 fn i64_save_8(
5632 &mut self,
5633 target_value: Location,
5634 memarg: &MemArg,
5635 target_addr: Location,
5636 need_check: bool,
5637 imported_memories: bool,
5638 offset: i32,
5639 heap_access_oob: Label,
5640 unaligned_atomic: Label,
5641 ) -> Result<(), CompileError> {
5642 self.memory_op(
5643 target_addr,
5644 memarg,
5645 false,
5646 1,
5647 need_check,
5648 imported_memories,
5649 offset,
5650 heap_access_oob,
5651 unaligned_atomic,
5652 |this, addr| this.emit_relaxed_str8(target_value, Location::Memory(addr, 0)),
5653 )
5654 }
5655 fn i64_save_16(
5656 &mut self,
5657 target_value: Location,
5658 memarg: &MemArg,
5659 target_addr: Location,
5660 need_check: bool,
5661 imported_memories: bool,
5662 offset: i32,
5663 heap_access_oob: Label,
5664 unaligned_atomic: Label,
5665 ) -> Result<(), CompileError> {
5666 self.memory_op(
5667 target_addr,
5668 memarg,
5669 false,
5670 2,
5671 need_check,
5672 imported_memories,
5673 offset,
5674 heap_access_oob,
5675 unaligned_atomic,
5676 |this, addr| this.emit_relaxed_str16(target_value, Location::Memory(addr, 0)),
5677 )
5678 }
5679 fn i64_save_32(
5680 &mut self,
5681 target_value: Location,
5682 memarg: &MemArg,
5683 target_addr: Location,
5684 need_check: bool,
5685 imported_memories: bool,
5686 offset: i32,
5687 heap_access_oob: Label,
5688 unaligned_atomic: Label,
5689 ) -> Result<(), CompileError> {
5690 self.memory_op(
5691 target_addr,
5692 memarg,
5693 false,
5694 4,
5695 need_check,
5696 imported_memories,
5697 offset,
5698 heap_access_oob,
5699 unaligned_atomic,
5700 |this, addr| this.emit_relaxed_str32(target_value, Location::Memory(addr, 0)),
5701 )
5702 }
5703 fn i64_atomic_save(
5704 &mut self,
5705 target_value: Location,
5706 memarg: &MemArg,
5707 target_addr: Location,
5708 need_check: bool,
5709 imported_memories: bool,
5710 offset: i32,
5711 heap_access_oob: Label,
5712 unaligned_atomic: Label,
5713 ) -> Result<(), CompileError> {
5714 self.memory_op(
5715 target_addr,
5716 memarg,
5717 true,
5718 8,
5719 need_check,
5720 imported_memories,
5721 offset,
5722 heap_access_oob,
5723 unaligned_atomic,
5724 |this, addr| this.emit_relaxed_str64(target_value, Location::Memory(addr, 0)),
5725 )?;
5726 self.assembler.emit_dmb()
5727 }
5728 fn i64_atomic_save_8(
5729 &mut self,
5730 target_value: Location,
5731 memarg: &MemArg,
5732 target_addr: Location,
5733 need_check: bool,
5734 imported_memories: bool,
5735 offset: i32,
5736 heap_access_oob: Label,
5737 unaligned_atomic: Label,
5738 ) -> Result<(), CompileError> {
5739 self.memory_op(
5740 target_addr,
5741 memarg,
5742 true,
5743 1,
5744 need_check,
5745 imported_memories,
5746 offset,
5747 heap_access_oob,
5748 unaligned_atomic,
5749 |this, addr| this.emit_relaxed_str8(target_value, Location::Memory(addr, 0)),
5750 )?;
5751 self.assembler.emit_dmb()
5752 }
5753 fn i64_atomic_save_16(
5754 &mut self,
5755 target_value: Location,
5756 memarg: &MemArg,
5757 target_addr: Location,
5758 need_check: bool,
5759 imported_memories: bool,
5760 offset: i32,
5761 heap_access_oob: Label,
5762 unaligned_atomic: Label,
5763 ) -> Result<(), CompileError> {
5764 self.memory_op(
5765 target_addr,
5766 memarg,
5767 true,
5768 2,
5769 need_check,
5770 imported_memories,
5771 offset,
5772 heap_access_oob,
5773 unaligned_atomic,
5774 |this, addr| this.emit_relaxed_str16(target_value, Location::Memory(addr, 0)),
5775 )?;
5776 self.assembler.emit_dmb()
5777 }
5778 fn i64_atomic_save_32(
5779 &mut self,
5780 target_value: Location,
5781 memarg: &MemArg,
5782 target_addr: Location,
5783 need_check: bool,
5784 imported_memories: bool,
5785 offset: i32,
5786 heap_access_oob: Label,
5787 unaligned_atomic: Label,
5788 ) -> Result<(), CompileError> {
5789 self.memory_op(
5790 target_addr,
5791 memarg,
5792 true,
5793 4,
5794 need_check,
5795 imported_memories,
5796 offset,
5797 heap_access_oob,
5798 unaligned_atomic,
5799 |this, addr| this.emit_relaxed_str32(target_value, Location::Memory(addr, 0)),
5800 )?;
5801 self.assembler.emit_dmb()
5802 }
5803 fn i64_atomic_add(
5805 &mut self,
5806 loc: Location,
5807 target: Location,
5808 memarg: &MemArg,
5809 ret: Location,
5810 need_check: bool,
5811 imported_memories: bool,
5812 offset: i32,
5813 heap_access_oob: Label,
5814 unaligned_atomic: Label,
5815 ) -> Result<(), CompileError> {
5816 self.memory_op(
5817 target,
5818 memarg,
5819 true,
5820 8,
5821 need_check,
5822 imported_memories,
5823 offset,
5824 heap_access_oob,
5825 unaligned_atomic,
5826 |this, addr| {
5827 let mut temps = vec![];
5828 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
5829 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5830 })?;
5831 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
5832 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5833 })?;
5834 let dst =
5835 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
5836 let reread = this.get_label();
5837
5838 this.emit_label(reread)?;
5839 this.assembler
5840 .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?;
5841 this.emit_binop_add64(dst, loc, Location::GPR(tmp1))?;
5842 this.assembler.emit_stlxr(
5843 Size::S64,
5844 Location::GPR(tmp2),
5845 Location::GPR(tmp1),
5846 Location::GPR(addr),
5847 )?;
5848 this.assembler
5849 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
5850 this.assembler.emit_dmb()?;
5851
5852 if dst != ret {
5853 this.move_location(Size::S64, ret, dst)?;
5854 }
5855 for r in temps {
5856 this.release_gpr(r);
5857 }
5858 this.release_gpr(tmp1);
5859 this.release_gpr(tmp2);
5860 Ok(())
5861 },
5862 )
5863 }
5864 fn i64_atomic_add_8u(
5866 &mut self,
5867 loc: Location,
5868 target: Location,
5869 memarg: &MemArg,
5870 ret: Location,
5871 need_check: bool,
5872 imported_memories: bool,
5873 offset: i32,
5874 heap_access_oob: Label,
5875 unaligned_atomic: Label,
5876 ) -> Result<(), CompileError> {
5877 self.memory_op(
5878 target,
5879 memarg,
5880 true,
5881 1,
5882 need_check,
5883 imported_memories,
5884 offset,
5885 heap_access_oob,
5886 unaligned_atomic,
5887 |this, addr| {
5888 let mut temps = vec![];
5889 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
5890 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5891 })?;
5892 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
5893 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5894 })?;
5895 let dst =
5896 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
5897 let reread = this.get_label();
5898
5899 this.emit_label(reread)?;
5900 this.assembler
5901 .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?;
5902 this.emit_binop_add64(dst, loc, Location::GPR(tmp1))?;
5903 this.assembler.emit_stlxrb(
5904 Size::S64,
5905 Location::GPR(tmp2),
5906 Location::GPR(tmp1),
5907 Location::GPR(addr),
5908 )?;
5909 this.assembler
5910 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
5911 this.assembler.emit_dmb()?;
5912
5913 if dst != ret {
5914 this.move_location(Size::S64, ret, dst)?;
5915 }
5916 for r in temps {
5917 this.release_gpr(r);
5918 }
5919 this.release_gpr(tmp1);
5920 this.release_gpr(tmp2);
5921 Ok(())
5922 },
5923 )
5924 }
5925 fn i64_atomic_add_16u(
5927 &mut self,
5928 loc: Location,
5929 target: Location,
5930 memarg: &MemArg,
5931 ret: Location,
5932 need_check: bool,
5933 imported_memories: bool,
5934 offset: i32,
5935 heap_access_oob: Label,
5936 unaligned_atomic: Label,
5937 ) -> Result<(), CompileError> {
5938 self.memory_op(
5939 target,
5940 memarg,
5941 true,
5942 2,
5943 need_check,
5944 imported_memories,
5945 offset,
5946 heap_access_oob,
5947 unaligned_atomic,
5948 |this, addr| {
5949 let mut temps = vec![];
5950 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
5951 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5952 })?;
5953 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
5954 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5955 })?;
5956 let dst =
5957 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
5958 let reread = this.get_label();
5959
5960 this.emit_label(reread)?;
5961 this.assembler
5962 .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?;
5963 this.emit_binop_add64(dst, loc, Location::GPR(tmp1))?;
5964 this.assembler.emit_stlxrh(
5965 Size::S64,
5966 Location::GPR(tmp2),
5967 Location::GPR(tmp1),
5968 Location::GPR(addr),
5969 )?;
5970 this.assembler
5971 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
5972 this.assembler.emit_dmb()?;
5973
5974 if dst != ret {
5975 this.move_location(Size::S64, ret, dst)?;
5976 }
5977 for r in temps {
5978 this.release_gpr(r);
5979 }
5980 this.release_gpr(tmp1);
5981 this.release_gpr(tmp2);
5982 Ok(())
5983 },
5984 )
5985 }
5986 fn i64_atomic_add_32u(
5988 &mut self,
5989 loc: Location,
5990 target: Location,
5991 memarg: &MemArg,
5992 ret: Location,
5993 need_check: bool,
5994 imported_memories: bool,
5995 offset: i32,
5996 heap_access_oob: Label,
5997 unaligned_atomic: Label,
5998 ) -> Result<(), CompileError> {
5999 self.memory_op(
6000 target,
6001 memarg,
6002 true,
6003 4,
6004 need_check,
6005 imported_memories,
6006 offset,
6007 heap_access_oob,
6008 unaligned_atomic,
6009 |this, addr| {
6010 let mut temps = vec![];
6011 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6012 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6013 })?;
6014 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6015 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6016 })?;
6017 let dst =
6018 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6019 let reread = this.get_label();
6020
6021 this.emit_label(reread)?;
6022 this.assembler
6023 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
6024 this.emit_binop_add64(dst, loc, Location::GPR(tmp1))?;
6025 this.assembler.emit_stlxr(
6026 Size::S32,
6027 Location::GPR(tmp2),
6028 Location::GPR(tmp1),
6029 Location::GPR(addr),
6030 )?;
6031 this.assembler
6032 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6033 this.assembler.emit_dmb()?;
6034
6035 if dst != ret {
6036 this.move_location(Size::S64, ret, dst)?;
6037 }
6038 for r in temps {
6039 this.release_gpr(r);
6040 }
6041 this.release_gpr(tmp1);
6042 this.release_gpr(tmp2);
6043 Ok(())
6044 },
6045 )
6046 }
6047 fn i64_atomic_sub(
6049 &mut self,
6050 loc: Location,
6051 target: Location,
6052 memarg: &MemArg,
6053 ret: Location,
6054 need_check: bool,
6055 imported_memories: bool,
6056 offset: i32,
6057 heap_access_oob: Label,
6058 unaligned_atomic: Label,
6059 ) -> Result<(), CompileError> {
6060 self.memory_op(
6061 target,
6062 memarg,
6063 true,
6064 8,
6065 need_check,
6066 imported_memories,
6067 offset,
6068 heap_access_oob,
6069 unaligned_atomic,
6070 |this, addr| {
6071 let mut temps = vec![];
6072 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6073 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6074 })?;
6075 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6076 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6077 })?;
6078 let dst =
6079 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6080 let reread = this.get_label();
6081
6082 this.emit_label(reread)?;
6083 this.assembler
6084 .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?;
6085 this.emit_binop_sub64(dst, loc, Location::GPR(tmp1))?;
6086 this.assembler.emit_stlxr(
6087 Size::S64,
6088 Location::GPR(tmp2),
6089 Location::GPR(tmp1),
6090 Location::GPR(addr),
6091 )?;
6092 this.assembler
6093 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6094 this.assembler.emit_dmb()?;
6095
6096 if dst != ret {
6097 this.move_location(Size::S64, ret, dst)?;
6098 }
6099 for r in temps {
6100 this.release_gpr(r);
6101 }
6102 this.release_gpr(tmp1);
6103 this.release_gpr(tmp2);
6104 Ok(())
6105 },
6106 )
6107 }
6108 fn i64_atomic_sub_8u(
6110 &mut self,
6111 loc: Location,
6112 target: Location,
6113 memarg: &MemArg,
6114 ret: Location,
6115 need_check: bool,
6116 imported_memories: bool,
6117 offset: i32,
6118 heap_access_oob: Label,
6119 unaligned_atomic: Label,
6120 ) -> Result<(), CompileError> {
6121 self.memory_op(
6122 target,
6123 memarg,
6124 true,
6125 1,
6126 need_check,
6127 imported_memories,
6128 offset,
6129 heap_access_oob,
6130 unaligned_atomic,
6131 |this, addr| {
6132 let mut temps = vec![];
6133 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6134 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6135 })?;
6136 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6137 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6138 })?;
6139 let dst =
6140 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6141 let reread = this.get_label();
6142
6143 this.emit_label(reread)?;
6144 this.assembler
6145 .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?;
6146 this.emit_binop_sub64(dst, loc, Location::GPR(tmp1))?;
6147 this.assembler.emit_stlxrb(
6148 Size::S64,
6149 Location::GPR(tmp2),
6150 Location::GPR(tmp1),
6151 Location::GPR(addr),
6152 )?;
6153 this.assembler
6154 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6155 this.assembler.emit_dmb()?;
6156
6157 if dst != ret {
6158 this.move_location(Size::S64, ret, dst)?;
6159 }
6160 for r in temps {
6161 this.release_gpr(r);
6162 }
6163 this.release_gpr(tmp1);
6164 this.release_gpr(tmp2);
6165 Ok(())
6166 },
6167 )
6168 }
6169 fn i64_atomic_sub_16u(
6171 &mut self,
6172 loc: Location,
6173 target: Location,
6174 memarg: &MemArg,
6175 ret: Location,
6176 need_check: bool,
6177 imported_memories: bool,
6178 offset: i32,
6179 heap_access_oob: Label,
6180 unaligned_atomic: Label,
6181 ) -> Result<(), CompileError> {
6182 self.memory_op(
6183 target,
6184 memarg,
6185 true,
6186 2,
6187 need_check,
6188 imported_memories,
6189 offset,
6190 heap_access_oob,
6191 unaligned_atomic,
6192 |this, addr| {
6193 let mut temps = vec![];
6194 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6195 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6196 })?;
6197 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6198 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6199 })?;
6200 let dst =
6201 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6202 let reread = this.get_label();
6203
6204 this.emit_label(reread)?;
6205 this.assembler
6206 .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?;
6207 this.emit_binop_sub64(dst, loc, Location::GPR(tmp1))?;
6208 this.assembler.emit_stlxrh(
6209 Size::S64,
6210 Location::GPR(tmp2),
6211 Location::GPR(tmp1),
6212 Location::GPR(addr),
6213 )?;
6214 this.assembler
6215 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6216 this.assembler.emit_dmb()?;
6217
6218 if dst != ret {
6219 this.move_location(Size::S64, ret, dst)?;
6220 }
6221 for r in temps {
6222 this.release_gpr(r);
6223 }
6224 this.release_gpr(tmp1);
6225 this.release_gpr(tmp2);
6226 Ok(())
6227 },
6228 )
6229 }
6230 fn i64_atomic_sub_32u(
6232 &mut self,
6233 loc: Location,
6234 target: Location,
6235 memarg: &MemArg,
6236 ret: Location,
6237 need_check: bool,
6238 imported_memories: bool,
6239 offset: i32,
6240 heap_access_oob: Label,
6241 unaligned_atomic: Label,
6242 ) -> Result<(), CompileError> {
6243 self.memory_op(
6244 target,
6245 memarg,
6246 true,
6247 4,
6248 need_check,
6249 imported_memories,
6250 offset,
6251 heap_access_oob,
6252 unaligned_atomic,
6253 |this, addr| {
6254 let mut temps = vec![];
6255 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6256 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6257 })?;
6258 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6259 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6260 })?;
6261 let dst =
6262 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6263 let reread = this.get_label();
6264
6265 this.emit_label(reread)?;
6266 this.assembler
6267 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
6268 this.emit_binop_sub64(dst, loc, Location::GPR(tmp1))?;
6269 this.assembler.emit_stlxr(
6270 Size::S32,
6271 Location::GPR(tmp2),
6272 Location::GPR(tmp1),
6273 Location::GPR(addr),
6274 )?;
6275 this.assembler
6276 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6277 this.assembler.emit_dmb()?;
6278
6279 if dst != ret {
6280 this.move_location(Size::S64, ret, dst)?;
6281 }
6282 for r in temps {
6283 this.release_gpr(r);
6284 }
6285 this.release_gpr(tmp1);
6286 this.release_gpr(tmp2);
6287 Ok(())
6288 },
6289 )
6290 }
6291 fn i64_atomic_and(
6293 &mut self,
6294 loc: Location,
6295 target: Location,
6296 memarg: &MemArg,
6297 ret: Location,
6298 need_check: bool,
6299 imported_memories: bool,
6300 offset: i32,
6301 heap_access_oob: Label,
6302 unaligned_atomic: Label,
6303 ) -> Result<(), CompileError> {
6304 self.memory_op(
6305 target,
6306 memarg,
6307 true,
6308 8,
6309 need_check,
6310 imported_memories,
6311 offset,
6312 heap_access_oob,
6313 unaligned_atomic,
6314 |this, addr| {
6315 let mut temps = vec![];
6316 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6317 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6318 })?;
6319 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6320 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6321 })?;
6322 let dst =
6323 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6324 let reread = this.get_label();
6325
6326 this.emit_label(reread)?;
6327 this.assembler
6328 .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?;
6329 this.emit_binop_and64(dst, loc, Location::GPR(tmp1))?;
6330 this.assembler.emit_stlxr(
6331 Size::S64,
6332 Location::GPR(tmp2),
6333 Location::GPR(tmp1),
6334 Location::GPR(addr),
6335 )?;
6336 this.assembler
6337 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6338 this.assembler.emit_dmb()?;
6339
6340 if dst != ret {
6341 this.move_location(Size::S64, ret, dst)?;
6342 }
6343 for r in temps {
6344 this.release_gpr(r);
6345 }
6346 this.release_gpr(tmp1);
6347 this.release_gpr(tmp2);
6348 Ok(())
6349 },
6350 )
6351 }
6352 fn i64_atomic_and_8u(
6354 &mut self,
6355 loc: Location,
6356 target: Location,
6357 memarg: &MemArg,
6358 ret: Location,
6359 need_check: bool,
6360 imported_memories: bool,
6361 offset: i32,
6362 heap_access_oob: Label,
6363 unaligned_atomic: Label,
6364 ) -> Result<(), CompileError> {
6365 self.memory_op(
6366 target,
6367 memarg,
6368 true,
6369 1,
6370 need_check,
6371 imported_memories,
6372 offset,
6373 heap_access_oob,
6374 unaligned_atomic,
6375 |this, addr| {
6376 let mut temps = vec![];
6377 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6378 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6379 })?;
6380 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6381 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6382 })?;
6383 let dst =
6384 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6385 let reread = this.get_label();
6386
6387 this.emit_label(reread)?;
6388 this.assembler
6389 .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?;
6390 this.emit_binop_and64(dst, loc, Location::GPR(tmp1))?;
6391 this.assembler.emit_stlxrb(
6392 Size::S64,
6393 Location::GPR(tmp2),
6394 Location::GPR(tmp1),
6395 Location::GPR(addr),
6396 )?;
6397 this.assembler
6398 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6399 this.assembler.emit_dmb()?;
6400
6401 if dst != ret {
6402 this.move_location(Size::S64, ret, dst)?;
6403 }
6404 for r in temps {
6405 this.release_gpr(r);
6406 }
6407 this.release_gpr(tmp1);
6408 this.release_gpr(tmp2);
6409 Ok(())
6410 },
6411 )
6412 }
6413 fn i64_atomic_and_16u(
6415 &mut self,
6416 loc: Location,
6417 target: Location,
6418 memarg: &MemArg,
6419 ret: Location,
6420 need_check: bool,
6421 imported_memories: bool,
6422 offset: i32,
6423 heap_access_oob: Label,
6424 unaligned_atomic: Label,
6425 ) -> Result<(), CompileError> {
6426 self.memory_op(
6427 target,
6428 memarg,
6429 true,
6430 2,
6431 need_check,
6432 imported_memories,
6433 offset,
6434 heap_access_oob,
6435 unaligned_atomic,
6436 |this, addr| {
6437 let mut temps = vec![];
6438 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6439 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6440 })?;
6441 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6442 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6443 })?;
6444 let dst =
6445 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6446 let reread = this.get_label();
6447
6448 this.emit_label(reread)?;
6449 this.assembler
6450 .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?;
6451 this.emit_binop_and64(dst, loc, Location::GPR(tmp1))?;
6452 this.assembler.emit_stlxrh(
6453 Size::S64,
6454 Location::GPR(tmp2),
6455 Location::GPR(tmp1),
6456 Location::GPR(addr),
6457 )?;
6458 this.assembler
6459 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6460 this.assembler.emit_dmb()?;
6461
6462 if dst != ret {
6463 this.move_location(Size::S64, ret, dst)?;
6464 }
6465 for r in temps {
6466 this.release_gpr(r);
6467 }
6468 this.release_gpr(tmp1);
6469 this.release_gpr(tmp2);
6470 Ok(())
6471 },
6472 )
6473 }
6474 fn i64_atomic_and_32u(
6476 &mut self,
6477 loc: Location,
6478 target: Location,
6479 memarg: &MemArg,
6480 ret: Location,
6481 need_check: bool,
6482 imported_memories: bool,
6483 offset: i32,
6484 heap_access_oob: Label,
6485 unaligned_atomic: Label,
6486 ) -> Result<(), CompileError> {
6487 self.memory_op(
6488 target,
6489 memarg,
6490 true,
6491 4,
6492 need_check,
6493 imported_memories,
6494 offset,
6495 heap_access_oob,
6496 unaligned_atomic,
6497 |this, addr| {
6498 let mut temps = vec![];
6499 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6500 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6501 })?;
6502 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6503 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6504 })?;
6505 let dst =
6506 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6507 let reread = this.get_label();
6508
6509 this.emit_label(reread)?;
6510 this.assembler
6511 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
6512 this.emit_binop_and64(dst, loc, Location::GPR(tmp1))?;
6513 this.assembler.emit_stlxr(
6514 Size::S32,
6515 Location::GPR(tmp2),
6516 Location::GPR(tmp1),
6517 Location::GPR(addr),
6518 )?;
6519 this.assembler
6520 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6521 this.assembler.emit_dmb()?;
6522
6523 if dst != ret {
6524 this.move_location(Size::S64, ret, dst)?;
6525 }
6526 for r in temps {
6527 this.release_gpr(r);
6528 }
6529 this.release_gpr(tmp1);
6530 this.release_gpr(tmp2);
6531 Ok(())
6532 },
6533 )
6534 }
6535 fn i64_atomic_or(
6537 &mut self,
6538 loc: Location,
6539 target: Location,
6540 memarg: &MemArg,
6541 ret: Location,
6542 need_check: bool,
6543 imported_memories: bool,
6544 offset: i32,
6545 heap_access_oob: Label,
6546 unaligned_atomic: Label,
6547 ) -> Result<(), CompileError> {
6548 self.memory_op(
6549 target,
6550 memarg,
6551 true,
6552 8,
6553 need_check,
6554 imported_memories,
6555 offset,
6556 heap_access_oob,
6557 unaligned_atomic,
6558 |this, addr| {
6559 let mut temps = vec![];
6560 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6561 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6562 })?;
6563 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6564 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6565 })?;
6566 let dst =
6567 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6568 let reread = this.get_label();
6569
6570 this.emit_label(reread)?;
6571 this.assembler
6572 .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?;
6573 this.emit_binop_or64(dst, loc, Location::GPR(tmp1))?;
6574 this.assembler.emit_stlxr(
6575 Size::S64,
6576 Location::GPR(tmp2),
6577 Location::GPR(tmp1),
6578 Location::GPR(addr),
6579 )?;
6580 this.assembler
6581 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6582 this.assembler.emit_dmb()?;
6583
6584 if dst != ret {
6585 this.move_location(Size::S64, ret, dst)?;
6586 }
6587 for r in temps {
6588 this.release_gpr(r);
6589 }
6590 this.release_gpr(tmp1);
6591 this.release_gpr(tmp2);
6592 Ok(())
6593 },
6594 )
6595 }
6596 fn i64_atomic_or_8u(
6598 &mut self,
6599 loc: Location,
6600 target: Location,
6601 memarg: &MemArg,
6602 ret: Location,
6603 need_check: bool,
6604 imported_memories: bool,
6605 offset: i32,
6606 heap_access_oob: Label,
6607 unaligned_atomic: Label,
6608 ) -> Result<(), CompileError> {
6609 self.memory_op(
6610 target,
6611 memarg,
6612 true,
6613 1,
6614 need_check,
6615 imported_memories,
6616 offset,
6617 heap_access_oob,
6618 unaligned_atomic,
6619 |this, addr| {
6620 let mut temps = vec![];
6621 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6622 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6623 })?;
6624 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6625 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6626 })?;
6627 let dst =
6628 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6629 let reread = this.get_label();
6630
6631 this.emit_label(reread)?;
6632 this.assembler
6633 .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?;
6634 this.emit_binop_or64(dst, loc, Location::GPR(tmp1))?;
6635 this.assembler.emit_stlxrb(
6636 Size::S64,
6637 Location::GPR(tmp2),
6638 Location::GPR(tmp1),
6639 Location::GPR(addr),
6640 )?;
6641 this.assembler
6642 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6643 this.assembler.emit_dmb()?;
6644
6645 if dst != ret {
6646 this.move_location(Size::S64, ret, dst)?;
6647 }
6648 for r in temps {
6649 this.release_gpr(r);
6650 }
6651 this.release_gpr(tmp1);
6652 this.release_gpr(tmp2);
6653 Ok(())
6654 },
6655 )
6656 }
6657 fn i64_atomic_or_16u(
6659 &mut self,
6660 loc: Location,
6661 target: Location,
6662 memarg: &MemArg,
6663 ret: Location,
6664 need_check: bool,
6665 imported_memories: bool,
6666 offset: i32,
6667 heap_access_oob: Label,
6668 unaligned_atomic: Label,
6669 ) -> Result<(), CompileError> {
6670 self.memory_op(
6671 target,
6672 memarg,
6673 true,
6674 2,
6675 need_check,
6676 imported_memories,
6677 offset,
6678 heap_access_oob,
6679 unaligned_atomic,
6680 |this, addr| {
6681 let mut temps = vec![];
6682 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6683 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6684 })?;
6685 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6686 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6687 })?;
6688 let dst =
6689 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6690 let reread = this.get_label();
6691
6692 this.emit_label(reread)?;
6693 this.assembler
6694 .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?;
6695 this.emit_binop_or64(dst, loc, Location::GPR(tmp1))?;
6696 this.assembler.emit_stlxrh(
6697 Size::S64,
6698 Location::GPR(tmp2),
6699 Location::GPR(tmp1),
6700 Location::GPR(addr),
6701 )?;
6702 this.assembler
6703 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6704 this.assembler.emit_dmb()?;
6705
6706 if dst != ret {
6707 this.move_location(Size::S64, ret, dst)?;
6708 }
6709 for r in temps {
6710 this.release_gpr(r);
6711 }
6712 this.release_gpr(tmp1);
6713 this.release_gpr(tmp2);
6714 Ok(())
6715 },
6716 )
6717 }
6718 fn i64_atomic_or_32u(
6720 &mut self,
6721 loc: Location,
6722 target: Location,
6723 memarg: &MemArg,
6724 ret: Location,
6725 need_check: bool,
6726 imported_memories: bool,
6727 offset: i32,
6728 heap_access_oob: Label,
6729 unaligned_atomic: Label,
6730 ) -> Result<(), CompileError> {
6731 self.memory_op(
6732 target,
6733 memarg,
6734 true,
6735 4,
6736 need_check,
6737 imported_memories,
6738 offset,
6739 heap_access_oob,
6740 unaligned_atomic,
6741 |this, addr| {
6742 let mut temps = vec![];
6743 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6744 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6745 })?;
6746 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6747 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6748 })?;
6749 let dst =
6750 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6751 let reread = this.get_label();
6752
6753 this.emit_label(reread)?;
6754 this.assembler
6755 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
6756 this.emit_binop_or64(dst, loc, Location::GPR(tmp1))?;
6757 this.assembler.emit_stlxr(
6758 Size::S32,
6759 Location::GPR(tmp2),
6760 Location::GPR(tmp1),
6761 Location::GPR(addr),
6762 )?;
6763 this.assembler
6764 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6765 this.assembler.emit_dmb()?;
6766
6767 if dst != ret {
6768 this.move_location(Size::S64, ret, dst)?;
6769 }
6770 for r in temps {
6771 this.release_gpr(r);
6772 }
6773 this.release_gpr(tmp1);
6774 this.release_gpr(tmp2);
6775 Ok(())
6776 },
6777 )
6778 }
6779 fn i64_atomic_xor(
6781 &mut self,
6782 loc: Location,
6783 target: Location,
6784 memarg: &MemArg,
6785 ret: Location,
6786 need_check: bool,
6787 imported_memories: bool,
6788 offset: i32,
6789 heap_access_oob: Label,
6790 unaligned_atomic: Label,
6791 ) -> Result<(), CompileError> {
6792 self.memory_op(
6793 target,
6794 memarg,
6795 true,
6796 8,
6797 need_check,
6798 imported_memories,
6799 offset,
6800 heap_access_oob,
6801 unaligned_atomic,
6802 |this, addr| {
6803 let mut temps = vec![];
6804 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6805 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6806 })?;
6807 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6808 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6809 })?;
6810 let dst =
6811 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6812 let reread = this.get_label();
6813
6814 this.emit_label(reread)?;
6815 this.assembler
6816 .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?;
6817 this.emit_binop_xor64(dst, loc, Location::GPR(tmp1))?;
6818 this.assembler.emit_stlxr(
6819 Size::S64,
6820 Location::GPR(tmp2),
6821 Location::GPR(tmp1),
6822 Location::GPR(addr),
6823 )?;
6824 this.assembler
6825 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6826 this.assembler.emit_dmb()?;
6827
6828 if dst != ret {
6829 this.move_location(Size::S64, ret, dst)?;
6830 }
6831 for r in temps {
6832 this.release_gpr(r);
6833 }
6834 this.release_gpr(tmp1);
6835 this.release_gpr(tmp2);
6836 Ok(())
6837 },
6838 )
6839 }
6840 fn i64_atomic_xor_8u(
6842 &mut self,
6843 loc: Location,
6844 target: Location,
6845 memarg: &MemArg,
6846 ret: Location,
6847 need_check: bool,
6848 imported_memories: bool,
6849 offset: i32,
6850 heap_access_oob: Label,
6851 unaligned_atomic: Label,
6852 ) -> Result<(), CompileError> {
6853 self.memory_op(
6854 target,
6855 memarg,
6856 true,
6857 1,
6858 need_check,
6859 imported_memories,
6860 offset,
6861 heap_access_oob,
6862 unaligned_atomic,
6863 |this, addr| {
6864 let mut temps = vec![];
6865 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6866 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6867 })?;
6868 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6869 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6870 })?;
6871 let dst =
6872 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6873 let reread = this.get_label();
6874
6875 this.emit_label(reread)?;
6876 this.assembler
6877 .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?;
6878 this.emit_binop_xor64(dst, loc, Location::GPR(tmp1))?;
6879 this.assembler.emit_stlxrb(
6880 Size::S64,
6881 Location::GPR(tmp2),
6882 Location::GPR(tmp1),
6883 Location::GPR(addr),
6884 )?;
6885 this.assembler
6886 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6887 this.assembler.emit_dmb()?;
6888
6889 if dst != ret {
6890 this.move_location(Size::S64, ret, dst)?;
6891 }
6892 for r in temps {
6893 this.release_gpr(r);
6894 }
6895 this.release_gpr(tmp1);
6896 this.release_gpr(tmp2);
6897 Ok(())
6898 },
6899 )
6900 }
6901 fn i64_atomic_xor_16u(
6903 &mut self,
6904 loc: Location,
6905 target: Location,
6906 memarg: &MemArg,
6907 ret: Location,
6908 need_check: bool,
6909 imported_memories: bool,
6910 offset: i32,
6911 heap_access_oob: Label,
6912 unaligned_atomic: Label,
6913 ) -> Result<(), CompileError> {
6914 self.memory_op(
6915 target,
6916 memarg,
6917 true,
6918 2,
6919 need_check,
6920 imported_memories,
6921 offset,
6922 heap_access_oob,
6923 unaligned_atomic,
6924 |this, addr| {
6925 let mut temps = vec![];
6926 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6927 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6928 })?;
6929 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6930 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6931 })?;
6932 let dst =
6933 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6934 let reread = this.get_label();
6935
6936 this.emit_label(reread)?;
6937 this.assembler
6938 .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?;
6939 this.emit_binop_xor64(dst, loc, Location::GPR(tmp1))?;
6940 this.assembler.emit_stlxrh(
6941 Size::S64,
6942 Location::GPR(tmp2),
6943 Location::GPR(tmp1),
6944 Location::GPR(addr),
6945 )?;
6946 this.assembler
6947 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
6948 this.assembler.emit_dmb()?;
6949
6950 if dst != ret {
6951 this.move_location(Size::S64, ret, dst)?;
6952 }
6953 for r in temps {
6954 this.release_gpr(r);
6955 }
6956 this.release_gpr(tmp1);
6957 this.release_gpr(tmp2);
6958 Ok(())
6959 },
6960 )
6961 }
6962 fn i64_atomic_xor_32u(
6964 &mut self,
6965 loc: Location,
6966 target: Location,
6967 memarg: &MemArg,
6968 ret: Location,
6969 need_check: bool,
6970 imported_memories: bool,
6971 offset: i32,
6972 heap_access_oob: Label,
6973 unaligned_atomic: Label,
6974 ) -> Result<(), CompileError> {
6975 self.memory_op(
6976 target,
6977 memarg,
6978 true,
6979 4,
6980 need_check,
6981 imported_memories,
6982 offset,
6983 heap_access_oob,
6984 unaligned_atomic,
6985 |this, addr| {
6986 let mut temps = vec![];
6987 let tmp1 = this.acquire_temp_gpr().ok_or_else(|| {
6988 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6989 })?;
6990 let tmp2 = this.acquire_temp_gpr().ok_or_else(|| {
6991 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6992 })?;
6993 let dst =
6994 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
6995 let reread = this.get_label();
6996
6997 this.emit_label(reread)?;
6998 this.assembler
6999 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
7000 this.emit_binop_xor64(dst, loc, Location::GPR(tmp1))?;
7001 this.assembler.emit_stlxr(
7002 Size::S32,
7003 Location::GPR(tmp2),
7004 Location::GPR(tmp1),
7005 Location::GPR(addr),
7006 )?;
7007 this.assembler
7008 .emit_cbnz_label(Size::S32, Location::GPR(tmp2), reread)?;
7009 this.assembler.emit_dmb()?;
7010
7011 if dst != ret {
7012 this.move_location(Size::S64, ret, dst)?;
7013 }
7014 for r in temps {
7015 this.release_gpr(r);
7016 }
7017 this.release_gpr(tmp1);
7018 this.release_gpr(tmp2);
7019 Ok(())
7020 },
7021 )
7022 }
7023 fn i64_atomic_xchg(
7025 &mut self,
7026 loc: Location,
7027 target: Location,
7028 memarg: &MemArg,
7029 ret: Location,
7030 need_check: bool,
7031 imported_memories: bool,
7032 offset: i32,
7033 heap_access_oob: Label,
7034 unaligned_atomic: Label,
7035 ) -> Result<(), CompileError> {
7036 self.memory_op(
7037 target,
7038 memarg,
7039 true,
7040 8,
7041 need_check,
7042 imported_memories,
7043 offset,
7044 heap_access_oob,
7045 unaligned_atomic,
7046 |this, addr| {
7047 let mut temps = vec![];
7048 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
7049 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7050 })?;
7051 let dst =
7052 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7053 let org =
7054 this.location_to_reg(Size::S64, loc, &mut temps, ImmType::None, false, None)?;
7055 let reread = this.get_label();
7056
7057 this.emit_label(reread)?;
7058 this.assembler
7059 .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?;
7060 this.assembler.emit_stlxr(
7061 Size::S64,
7062 Location::GPR(tmp),
7063 org,
7064 Location::GPR(addr),
7065 )?;
7066 this.assembler
7067 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7068 this.assembler.emit_dmb()?;
7069
7070 if dst != ret {
7071 this.move_location(Size::S64, ret, dst)?;
7072 }
7073 for r in temps {
7074 this.release_gpr(r);
7075 }
7076 this.release_gpr(tmp);
7077 Ok(())
7078 },
7079 )
7080 }
7081 fn i64_atomic_xchg_8u(
7083 &mut self,
7084 loc: Location,
7085 target: Location,
7086 memarg: &MemArg,
7087 ret: Location,
7088 need_check: bool,
7089 imported_memories: bool,
7090 offset: i32,
7091 heap_access_oob: Label,
7092 unaligned_atomic: Label,
7093 ) -> Result<(), CompileError> {
7094 self.memory_op(
7095 target,
7096 memarg,
7097 true,
7098 1,
7099 need_check,
7100 imported_memories,
7101 offset,
7102 heap_access_oob,
7103 unaligned_atomic,
7104 |this, addr| {
7105 let mut temps = vec![];
7106 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
7107 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7108 })?;
7109 let dst =
7110 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7111 let org =
7112 this.location_to_reg(Size::S64, loc, &mut temps, ImmType::None, false, None)?;
7113 let reread = this.get_label();
7114
7115 this.emit_label(reread)?;
7116 this.assembler
7117 .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?;
7118 this.assembler.emit_stlxrb(
7119 Size::S64,
7120 Location::GPR(tmp),
7121 org,
7122 Location::GPR(addr),
7123 )?;
7124 this.assembler
7125 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7126 this.assembler.emit_dmb()?;
7127
7128 if dst != ret {
7129 this.move_location(Size::S64, ret, dst)?;
7130 }
7131 for r in temps {
7132 this.release_gpr(r);
7133 }
7134 this.release_gpr(tmp);
7135 Ok(())
7136 },
7137 )
7138 }
7139 fn i64_atomic_xchg_16u(
7141 &mut self,
7142 loc: Location,
7143 target: Location,
7144 memarg: &MemArg,
7145 ret: Location,
7146 need_check: bool,
7147 imported_memories: bool,
7148 offset: i32,
7149 heap_access_oob: Label,
7150 unaligned_atomic: Label,
7151 ) -> Result<(), CompileError> {
7152 self.memory_op(
7153 target,
7154 memarg,
7155 true,
7156 2,
7157 need_check,
7158 imported_memories,
7159 offset,
7160 heap_access_oob,
7161 unaligned_atomic,
7162 |this, addr| {
7163 let mut temps = vec![];
7164 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
7165 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7166 })?;
7167 let dst =
7168 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7169 let org =
7170 this.location_to_reg(Size::S64, loc, &mut temps, ImmType::None, false, None)?;
7171 let reread = this.get_label();
7172
7173 this.emit_label(reread)?;
7174 this.assembler
7175 .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?;
7176 this.assembler.emit_stlxrh(
7177 Size::S64,
7178 Location::GPR(tmp),
7179 org,
7180 Location::GPR(addr),
7181 )?;
7182 this.assembler
7183 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7184 this.assembler.emit_dmb()?;
7185
7186 if dst != ret {
7187 this.move_location(Size::S64, ret, dst)?;
7188 }
7189 for r in temps {
7190 this.release_gpr(r);
7191 }
7192 this.release_gpr(tmp);
7193 Ok(())
7194 },
7195 )
7196 }
7197 fn i64_atomic_xchg_32u(
7199 &mut self,
7200 loc: Location,
7201 target: Location,
7202 memarg: &MemArg,
7203 ret: Location,
7204 need_check: bool,
7205 imported_memories: bool,
7206 offset: i32,
7207 heap_access_oob: Label,
7208 unaligned_atomic: Label,
7209 ) -> Result<(), CompileError> {
7210 self.memory_op(
7211 target,
7212 memarg,
7213 true,
7214 4,
7215 need_check,
7216 imported_memories,
7217 offset,
7218 heap_access_oob,
7219 unaligned_atomic,
7220 |this, addr| {
7221 let mut temps = vec![];
7222 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
7223 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7224 })?;
7225 let dst =
7226 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7227 let org =
7228 this.location_to_reg(Size::S64, loc, &mut temps, ImmType::None, false, None)?;
7229 let reread = this.get_label();
7230
7231 this.emit_label(reread)?;
7232 this.assembler
7233 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
7234 this.assembler.emit_stlxr(
7235 Size::S32,
7236 Location::GPR(tmp),
7237 org,
7238 Location::GPR(addr),
7239 )?;
7240 this.assembler
7241 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7242 this.assembler.emit_dmb()?;
7243
7244 if dst != ret {
7245 this.move_location(Size::S64, ret, dst)?;
7246 }
7247 for r in temps {
7248 this.release_gpr(r);
7249 }
7250 this.release_gpr(tmp);
7251 Ok(())
7252 },
7253 )
7254 }
7255 fn i64_atomic_cmpxchg(
7257 &mut self,
7258 new: Location,
7259 cmp: Location,
7260 target: Location,
7261 memarg: &MemArg,
7262 ret: Location,
7263 need_check: bool,
7264 imported_memories: bool,
7265 offset: i32,
7266 heap_access_oob: Label,
7267 unaligned_atomic: Label,
7268 ) -> Result<(), CompileError> {
7269 self.memory_op(
7270 target,
7271 memarg,
7272 true,
7273 8,
7274 need_check,
7275 imported_memories,
7276 offset,
7277 heap_access_oob,
7278 unaligned_atomic,
7279 |this, addr| {
7280 let mut temps = vec![];
7281 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
7282 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7283 })?;
7284 let dst =
7285 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7286 let org =
7287 this.location_to_reg(Size::S64, new, &mut temps, ImmType::None, false, None)?;
7288 let reread = this.get_label();
7289 let nosame = this.get_label();
7290
7291 this.emit_label(reread)?;
7292 this.assembler
7293 .emit_ldaxr(Size::S64, dst, Location::GPR(addr))?;
7294 this.emit_relaxed_cmp(Size::S64, dst, cmp)?;
7295 this.assembler.emit_bcond_label(Condition::Ne, nosame)?;
7296 this.assembler.emit_stlxr(
7297 Size::S64,
7298 Location::GPR(tmp),
7299 org,
7300 Location::GPR(addr),
7301 )?;
7302 this.assembler
7303 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7304 this.assembler.emit_dmb()?;
7305
7306 this.emit_label(nosame)?;
7307 if dst != ret {
7308 this.move_location(Size::S64, ret, dst)?;
7309 }
7310 for r in temps {
7311 this.release_gpr(r);
7312 }
7313 this.release_gpr(tmp);
7314 Ok(())
7315 },
7316 )
7317 }
7318 fn i64_atomic_cmpxchg_8u(
7320 &mut self,
7321 new: Location,
7322 cmp: Location,
7323 target: Location,
7324 memarg: &MemArg,
7325 ret: Location,
7326 need_check: bool,
7327 imported_memories: bool,
7328 offset: i32,
7329 heap_access_oob: Label,
7330 unaligned_atomic: Label,
7331 ) -> Result<(), CompileError> {
7332 self.memory_op(
7333 target,
7334 memarg,
7335 true,
7336 1,
7337 need_check,
7338 imported_memories,
7339 offset,
7340 heap_access_oob,
7341 unaligned_atomic,
7342 |this, addr| {
7343 let mut temps = vec![];
7344 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
7345 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7346 })?;
7347 let dst =
7348 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7349 let org =
7350 this.location_to_reg(Size::S64, new, &mut temps, ImmType::None, false, None)?;
7351 let reread = this.get_label();
7352 let nosame = this.get_label();
7353
7354 this.emit_label(reread)?;
7355 this.assembler
7356 .emit_ldaxrb(Size::S64, dst, Location::GPR(addr))?;
7357 this.emit_relaxed_cmp(Size::S64, dst, cmp)?;
7358 this.assembler.emit_bcond_label(Condition::Ne, nosame)?;
7359 this.assembler.emit_stlxrb(
7360 Size::S64,
7361 Location::GPR(tmp),
7362 org,
7363 Location::GPR(addr),
7364 )?;
7365 this.assembler
7366 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7367 this.assembler.emit_dmb()?;
7368
7369 this.emit_label(nosame)?;
7370 if dst != ret {
7371 this.move_location(Size::S64, ret, dst)?;
7372 }
7373 for r in temps {
7374 this.release_gpr(r);
7375 }
7376 this.release_gpr(tmp);
7377 Ok(())
7378 },
7379 )
7380 }
7381 fn i64_atomic_cmpxchg_16u(
7383 &mut self,
7384 new: Location,
7385 cmp: Location,
7386 target: Location,
7387 memarg: &MemArg,
7388 ret: Location,
7389 need_check: bool,
7390 imported_memories: bool,
7391 offset: i32,
7392 heap_access_oob: Label,
7393 unaligned_atomic: Label,
7394 ) -> Result<(), CompileError> {
7395 self.memory_op(
7396 target,
7397 memarg,
7398 true,
7399 2,
7400 need_check,
7401 imported_memories,
7402 offset,
7403 heap_access_oob,
7404 unaligned_atomic,
7405 |this, addr| {
7406 let mut temps = vec![];
7407 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
7408 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7409 })?;
7410 let dst =
7411 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7412 let org =
7413 this.location_to_reg(Size::S64, new, &mut temps, ImmType::None, false, None)?;
7414 let reread = this.get_label();
7415 let nosame = this.get_label();
7416
7417 this.emit_label(reread)?;
7418 this.assembler
7419 .emit_ldaxrh(Size::S64, dst, Location::GPR(addr))?;
7420 this.emit_relaxed_cmp(Size::S64, dst, cmp)?;
7421 this.assembler.emit_bcond_label(Condition::Ne, nosame)?;
7422 this.assembler.emit_stlxrh(
7423 Size::S64,
7424 Location::GPR(tmp),
7425 org,
7426 Location::GPR(addr),
7427 )?;
7428 this.assembler
7429 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7430 this.assembler.emit_dmb()?;
7431
7432 this.emit_label(nosame)?;
7433 if dst != ret {
7434 this.move_location(Size::S64, ret, dst)?;
7435 }
7436 for r in temps {
7437 this.release_gpr(r);
7438 }
7439 this.release_gpr(tmp);
7440 Ok(())
7441 },
7442 )
7443 }
7444 fn i64_atomic_cmpxchg_32u(
7446 &mut self,
7447 new: Location,
7448 cmp: Location,
7449 target: Location,
7450 memarg: &MemArg,
7451 ret: Location,
7452 need_check: bool,
7453 imported_memories: bool,
7454 offset: i32,
7455 heap_access_oob: Label,
7456 unaligned_atomic: Label,
7457 ) -> Result<(), CompileError> {
7458 self.memory_op(
7459 target,
7460 memarg,
7461 true,
7462 4,
7463 need_check,
7464 imported_memories,
7465 offset,
7466 heap_access_oob,
7467 unaligned_atomic,
7468 |this, addr| {
7469 let mut temps = vec![];
7470 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
7471 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7472 })?;
7473 let dst =
7474 this.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7475 let org =
7476 this.location_to_reg(Size::S64, new, &mut temps, ImmType::None, false, None)?;
7477 let reread = this.get_label();
7478 let nosame = this.get_label();
7479
7480 this.emit_label(reread)?;
7481 this.assembler
7482 .emit_ldaxr(Size::S32, dst, Location::GPR(addr))?;
7483 this.emit_relaxed_cmp(Size::S64, dst, cmp)?;
7484 this.assembler.emit_bcond_label(Condition::Ne, nosame)?;
7485 this.assembler.emit_stlxr(
7486 Size::S32,
7487 Location::GPR(tmp),
7488 org,
7489 Location::GPR(addr),
7490 )?;
7491 this.assembler
7492 .emit_cbnz_label(Size::S32, Location::GPR(tmp), reread)?;
7493 this.assembler.emit_dmb()?;
7494
7495 this.emit_label(nosame)?;
7496 if dst != ret {
7497 this.move_location(Size::S64, ret, dst)?;
7498 }
7499 for r in temps {
7500 this.release_gpr(r);
7501 }
7502 this.release_gpr(tmp);
7503 Ok(())
7504 },
7505 )
7506 }
7507
7508 fn f32_load(
7509 &mut self,
7510 addr: Location,
7511 memarg: &MemArg,
7512 ret: Location,
7513 need_check: bool,
7514 imported_memories: bool,
7515 offset: i32,
7516 heap_access_oob: Label,
7517 unaligned_atomic: Label,
7518 ) -> Result<(), CompileError> {
7519 self.memory_op(
7520 addr,
7521 memarg,
7522 false,
7523 4,
7524 need_check,
7525 imported_memories,
7526 offset,
7527 heap_access_oob,
7528 unaligned_atomic,
7529 |this, addr| this.emit_relaxed_ldr32(Size::S32, ret, Location::Memory(addr, 0)),
7530 )
7531 }
7532 fn f32_save(
7533 &mut self,
7534 target_value: Location,
7535 memarg: &MemArg,
7536 target_addr: Location,
7537 canonicalize: bool,
7538 need_check: bool,
7539 imported_memories: bool,
7540 offset: i32,
7541 heap_access_oob: Label,
7542 unaligned_atomic: Label,
7543 ) -> Result<(), CompileError> {
7544 let canonicalize = canonicalize && self.arch_supports_canonicalize_nan();
7545 self.memory_op(
7546 target_addr,
7547 memarg,
7548 false,
7549 4,
7550 need_check,
7551 imported_memories,
7552 offset,
7553 heap_access_oob,
7554 unaligned_atomic,
7555 |this, addr| {
7556 if !canonicalize {
7557 this.emit_relaxed_str32(target_value, Location::Memory(addr, 0))
7558 } else {
7559 this.canonicalize_nan(Size::S32, target_value, Location::Memory(addr, 0))
7560 }
7561 },
7562 )
7563 }
7564 fn f64_load(
7565 &mut self,
7566 addr: Location,
7567 memarg: &MemArg,
7568 ret: Location,
7569 need_check: bool,
7570 imported_memories: bool,
7571 offset: i32,
7572 heap_access_oob: Label,
7573 unaligned_atomic: Label,
7574 ) -> Result<(), CompileError> {
7575 self.memory_op(
7576 addr,
7577 memarg,
7578 false,
7579 8,
7580 need_check,
7581 imported_memories,
7582 offset,
7583 heap_access_oob,
7584 unaligned_atomic,
7585 |this, addr| this.emit_relaxed_ldr64(Size::S64, ret, Location::Memory(addr, 0)),
7586 )
7587 }
7588 fn f64_save(
7589 &mut self,
7590 target_value: Location,
7591 memarg: &MemArg,
7592 target_addr: Location,
7593 canonicalize: bool,
7594 need_check: bool,
7595 imported_memories: bool,
7596 offset: i32,
7597 heap_access_oob: Label,
7598 unaligned_atomic: Label,
7599 ) -> Result<(), CompileError> {
7600 let canonicalize = canonicalize && self.arch_supports_canonicalize_nan();
7601 self.memory_op(
7602 target_addr,
7603 memarg,
7604 false,
7605 8,
7606 need_check,
7607 imported_memories,
7608 offset,
7609 heap_access_oob,
7610 unaligned_atomic,
7611 |this, addr| {
7612 if !canonicalize {
7613 this.emit_relaxed_str64(target_value, Location::Memory(addr, 0))
7614 } else {
7615 this.canonicalize_nan(Size::S64, target_value, Location::Memory(addr, 0))
7616 }
7617 },
7618 )
7619 }
7620
7621 fn convert_f64_i64(
7622 &mut self,
7623 loc: Location,
7624 signed: bool,
7625 ret: Location,
7626 ) -> Result<(), CompileError> {
7627 let mut gprs = vec![];
7628 let mut neons = vec![];
7629 let src = self.location_to_reg(Size::S64, loc, &mut gprs, ImmType::NoneXzr, true, None)?;
7630 let dest = self.location_to_neon(Size::S64, ret, &mut neons, ImmType::None, false)?;
7631 if signed {
7632 self.assembler.emit_scvtf(Size::S64, src, Size::S64, dest)?;
7633 } else {
7634 self.assembler.emit_ucvtf(Size::S64, src, Size::S64, dest)?;
7635 }
7636 if ret != dest {
7637 self.move_location(Size::S64, dest, ret)?;
7638 }
7639 for r in gprs {
7640 self.release_gpr(r);
7641 }
7642 for r in neons {
7643 self.release_simd(r);
7644 }
7645 Ok(())
7646 }
7647 fn convert_f64_i32(
7648 &mut self,
7649 loc: Location,
7650 signed: bool,
7651 ret: Location,
7652 ) -> Result<(), CompileError> {
7653 let mut gprs = vec![];
7654 let mut neons = vec![];
7655 let src = self.location_to_reg(Size::S32, loc, &mut gprs, ImmType::NoneXzr, true, None)?;
7656 let dest = self.location_to_neon(Size::S64, ret, &mut neons, ImmType::None, false)?;
7657 if signed {
7658 self.assembler.emit_scvtf(Size::S32, src, Size::S64, dest)?;
7659 } else {
7660 self.assembler.emit_ucvtf(Size::S32, src, Size::S64, dest)?;
7661 }
7662 if ret != dest {
7663 self.move_location(Size::S64, dest, ret)?;
7664 }
7665 for r in gprs {
7666 self.release_gpr(r);
7667 }
7668 for r in neons {
7669 self.release_simd(r);
7670 }
7671 Ok(())
7672 }
7673 fn convert_f32_i64(
7674 &mut self,
7675 loc: Location,
7676 signed: bool,
7677 ret: Location,
7678 ) -> Result<(), CompileError> {
7679 let mut gprs = vec![];
7680 let mut neons = vec![];
7681 let src = self.location_to_reg(Size::S64, loc, &mut gprs, ImmType::NoneXzr, true, None)?;
7682 let dest = self.location_to_neon(Size::S32, ret, &mut neons, ImmType::None, false)?;
7683 if signed {
7684 self.assembler.emit_scvtf(Size::S64, src, Size::S32, dest)?;
7685 } else {
7686 self.assembler.emit_ucvtf(Size::S64, src, Size::S32, dest)?;
7687 }
7688 if ret != dest {
7689 self.move_location(Size::S32, dest, ret)?;
7690 }
7691 for r in gprs {
7692 self.release_gpr(r);
7693 }
7694 for r in neons {
7695 self.release_simd(r);
7696 }
7697 Ok(())
7698 }
7699 fn convert_f32_i32(
7700 &mut self,
7701 loc: Location,
7702 signed: bool,
7703 ret: Location,
7704 ) -> Result<(), CompileError> {
7705 let mut gprs = vec![];
7706 let mut neons = vec![];
7707 let src = self.location_to_reg(Size::S32, loc, &mut gprs, ImmType::NoneXzr, true, None)?;
7708 let dest = self.location_to_neon(Size::S32, ret, &mut neons, ImmType::None, false)?;
7709 if signed {
7710 self.assembler.emit_scvtf(Size::S32, src, Size::S32, dest)?;
7711 } else {
7712 self.assembler.emit_ucvtf(Size::S32, src, Size::S32, dest)?;
7713 }
7714 if ret != dest {
7715 self.move_location(Size::S32, dest, ret)?;
7716 }
7717 for r in gprs {
7718 self.release_gpr(r);
7719 }
7720 for r in neons {
7721 self.release_simd(r);
7722 }
7723 Ok(())
7724 }
7725 fn convert_i64_f64(
7726 &mut self,
7727 loc: Location,
7728 ret: Location,
7729 signed: bool,
7730 sat: bool,
7731 ) -> Result<(), CompileError> {
7732 let mut gprs = vec![];
7733 let mut neons = vec![];
7734 let src = self.location_to_neon(Size::S64, loc, &mut neons, ImmType::None, true)?;
7735 let dest = self.location_to_reg(Size::S64, ret, &mut gprs, ImmType::None, false, None)?;
7736 let old_fpcr = if !sat {
7737 self.reset_exception_fpsr()?;
7738 self.set_trap_enabled(&mut gprs)?
7739 } else {
7740 GPR::XzrSp
7741 };
7742 if signed {
7743 self.assembler
7744 .emit_fcvtzs(Size::S64, src, Size::S64, dest)?;
7745 } else {
7746 self.assembler
7747 .emit_fcvtzu(Size::S64, src, Size::S64, dest)?;
7748 }
7749 if !sat {
7750 self.trap_float_convertion_errors(old_fpcr, Size::S64, src, &mut gprs)?;
7751 }
7752 if ret != dest {
7753 self.move_location(Size::S64, dest, ret)?;
7754 }
7755 for r in gprs {
7756 self.release_gpr(r);
7757 }
7758 for r in neons {
7759 self.release_simd(r);
7760 }
7761 Ok(())
7762 }
7763 fn convert_i32_f64(
7764 &mut self,
7765 loc: Location,
7766 ret: Location,
7767 signed: bool,
7768 sat: bool,
7769 ) -> Result<(), CompileError> {
7770 let mut gprs = vec![];
7771 let mut neons = vec![];
7772 let src = self.location_to_neon(Size::S64, loc, &mut neons, ImmType::None, true)?;
7773 let dest = self.location_to_reg(Size::S32, ret, &mut gprs, ImmType::None, false, None)?;
7774 let old_fpcr = if !sat {
7775 self.reset_exception_fpsr()?;
7776 self.set_trap_enabled(&mut gprs)?
7777 } else {
7778 GPR::XzrSp
7779 };
7780 if signed {
7781 self.assembler
7782 .emit_fcvtzs(Size::S64, src, Size::S32, dest)?;
7783 } else {
7784 self.assembler
7785 .emit_fcvtzu(Size::S64, src, Size::S32, dest)?;
7786 }
7787 if !sat {
7788 self.trap_float_convertion_errors(old_fpcr, Size::S64, src, &mut gprs)?;
7789 }
7790 if ret != dest {
7791 self.move_location(Size::S32, dest, ret)?;
7792 }
7793 for r in gprs {
7794 self.release_gpr(r);
7795 }
7796 for r in neons {
7797 self.release_simd(r);
7798 }
7799 Ok(())
7800 }
7801 fn convert_i64_f32(
7802 &mut self,
7803 loc: Location,
7804 ret: Location,
7805 signed: bool,
7806 sat: bool,
7807 ) -> Result<(), CompileError> {
7808 let mut gprs = vec![];
7809 let mut neons = vec![];
7810 let src = self.location_to_neon(Size::S32, loc, &mut neons, ImmType::None, true)?;
7811 let dest = self.location_to_reg(Size::S64, ret, &mut gprs, ImmType::None, false, None)?;
7812 let old_fpcr = if !sat {
7813 self.reset_exception_fpsr()?;
7814 self.set_trap_enabled(&mut gprs)?
7815 } else {
7816 GPR::XzrSp
7817 };
7818 if signed {
7819 self.assembler
7820 .emit_fcvtzs(Size::S32, src, Size::S64, dest)?;
7821 } else {
7822 self.assembler
7823 .emit_fcvtzu(Size::S32, src, Size::S64, dest)?;
7824 }
7825 if !sat {
7826 self.trap_float_convertion_errors(old_fpcr, Size::S32, src, &mut gprs)?;
7827 }
7828 if ret != dest {
7829 self.move_location(Size::S64, dest, ret)?;
7830 }
7831 for r in gprs {
7832 self.release_gpr(r);
7833 }
7834 for r in neons {
7835 self.release_simd(r);
7836 }
7837 Ok(())
7838 }
7839 fn convert_i32_f32(
7840 &mut self,
7841 loc: Location,
7842 ret: Location,
7843 signed: bool,
7844 sat: bool,
7845 ) -> Result<(), CompileError> {
7846 let mut gprs = vec![];
7847 let mut neons = vec![];
7848 let src = self.location_to_neon(Size::S32, loc, &mut neons, ImmType::None, true)?;
7849 let dest = self.location_to_reg(Size::S32, ret, &mut gprs, ImmType::None, false, None)?;
7850 let old_fpcr = if !sat {
7851 self.reset_exception_fpsr()?;
7852 self.set_trap_enabled(&mut gprs)?
7853 } else {
7854 GPR::XzrSp
7855 };
7856 if signed {
7857 self.assembler
7858 .emit_fcvtzs(Size::S32, src, Size::S32, dest)?;
7859 } else {
7860 self.assembler
7861 .emit_fcvtzu(Size::S32, src, Size::S32, dest)?;
7862 }
7863 if !sat {
7864 self.trap_float_convertion_errors(old_fpcr, Size::S32, src, &mut gprs)?;
7865 }
7866 if ret != dest {
7867 self.move_location(Size::S32, dest, ret)?;
7868 }
7869 for r in gprs {
7870 self.release_gpr(r);
7871 }
7872 for r in neons {
7873 self.release_simd(r);
7874 }
7875 Ok(())
7876 }
7877 fn convert_f64_f32(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7878 self.emit_relaxed_binop_neon(Assembler::emit_fcvt, Size::S32, loc, ret, true)
7879 }
7880 fn convert_f32_f64(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7881 self.emit_relaxed_binop_neon(Assembler::emit_fcvt, Size::S64, loc, ret, true)
7882 }
7883 fn f64_neg(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7884 self.emit_relaxed_binop_neon(Assembler::emit_fneg, Size::S64, loc, ret, true)
7885 }
7886 fn f64_abs(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7887 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
7888 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7889 })?;
7890
7891 self.move_location(Size::S64, loc, Location::GPR(tmp))?;
7892 self.assembler.emit_and(
7893 Size::S64,
7894 Location::GPR(tmp),
7895 Location::Imm64(0x7fffffffffffffffu64),
7896 Location::GPR(tmp),
7897 )?;
7898 self.move_location(Size::S64, Location::GPR(tmp), ret)?;
7899
7900 self.release_gpr(tmp);
7901 Ok(())
7902 }
7903 fn emit_i64_copysign(&mut self, tmp1: GPR, tmp2: GPR) -> Result<(), CompileError> {
7904 self.assembler.emit_and(
7905 Size::S64,
7906 Location::GPR(tmp1),
7907 Location::Imm64(0x7fffffffffffffffu64),
7908 Location::GPR(tmp1),
7909 )?;
7910
7911 self.assembler.emit_and(
7912 Size::S64,
7913 Location::GPR(tmp2),
7914 Location::Imm64(0x8000000000000000u64),
7915 Location::GPR(tmp2),
7916 )?;
7917
7918 self.assembler.emit_or(
7919 Size::S64,
7920 Location::GPR(tmp1),
7921 Location::GPR(tmp2),
7922 Location::GPR(tmp1),
7923 )
7924 }
7925 fn f64_sqrt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7926 self.emit_relaxed_binop_neon(Assembler::emit_fsqrt, Size::S64, loc, ret, true)
7927 }
7928 fn f64_trunc(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7929 self.emit_relaxed_binop_neon(Assembler::emit_frintz, Size::S64, loc, ret, true)
7930 }
7931 fn f64_ceil(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7932 self.emit_relaxed_binop_neon(Assembler::emit_frintp, Size::S64, loc, ret, true)
7933 }
7934 fn f64_floor(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7935 self.emit_relaxed_binop_neon(Assembler::emit_frintm, Size::S64, loc, ret, true)
7936 }
7937 fn f64_nearest(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7938 self.emit_relaxed_binop_neon(Assembler::emit_frintn, Size::S64, loc, ret, true)
7939 }
7940 fn f64_cmp_ge(
7941 &mut self,
7942 loc_a: Location,
7943 loc_b: Location,
7944 ret: Location,
7945 ) -> Result<(), CompileError> {
7946 let mut temps = vec![];
7947 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7948 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S64, loc_b, loc_a, false)?;
7949 self.assembler.emit_cset(Size::S32, dest, Condition::Ls)?;
7950 if ret != dest {
7951 self.move_location(Size::S32, dest, ret)?;
7952 }
7953 for r in temps {
7954 self.release_gpr(r);
7955 }
7956 Ok(())
7957 }
7958 fn f64_cmp_gt(
7959 &mut self,
7960 loc_a: Location,
7961 loc_b: Location,
7962 ret: Location,
7963 ) -> Result<(), CompileError> {
7964 let mut temps = vec![];
7965 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7966 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S64, loc_b, loc_a, false)?;
7967 self.assembler.emit_cset(Size::S32, dest, Condition::Cc)?;
7968 if ret != dest {
7969 self.move_location(Size::S32, dest, ret)?;
7970 }
7971 for r in temps {
7972 self.release_gpr(r);
7973 }
7974 Ok(())
7975 }
7976 fn f64_cmp_le(
7977 &mut self,
7978 loc_a: Location,
7979 loc_b: Location,
7980 ret: Location,
7981 ) -> Result<(), CompileError> {
7982 let mut temps = vec![];
7983 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
7984 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S64, loc_a, loc_b, false)?;
7985 self.assembler.emit_cset(Size::S32, dest, Condition::Ls)?;
7986 if ret != dest {
7987 self.move_location(Size::S32, dest, ret)?;
7988 }
7989 for r in temps {
7990 self.release_gpr(r);
7991 }
7992 Ok(())
7993 }
7994 fn f64_cmp_lt(
7995 &mut self,
7996 loc_a: Location,
7997 loc_b: Location,
7998 ret: Location,
7999 ) -> Result<(), CompileError> {
8000 let mut temps = vec![];
8001 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
8002 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S64, loc_a, loc_b, false)?;
8003 self.assembler.emit_cset(Size::S32, dest, Condition::Cc)?;
8004 if ret != dest {
8005 self.move_location(Size::S32, dest, ret)?;
8006 }
8007 for r in temps {
8008 self.release_gpr(r);
8009 }
8010 Ok(())
8011 }
8012 fn f64_cmp_ne(
8013 &mut self,
8014 loc_a: Location,
8015 loc_b: Location,
8016 ret: Location,
8017 ) -> Result<(), CompileError> {
8018 let mut temps = vec![];
8019 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
8020 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S64, loc_a, loc_b, false)?;
8021 self.assembler.emit_cset(Size::S32, dest, Condition::Ne)?;
8022 if ret != dest {
8023 self.move_location(Size::S32, dest, ret)?;
8024 }
8025 for r in temps {
8026 self.release_gpr(r);
8027 }
8028 Ok(())
8029 }
8030 fn f64_cmp_eq(
8031 &mut self,
8032 loc_a: Location,
8033 loc_b: Location,
8034 ret: Location,
8035 ) -> Result<(), CompileError> {
8036 let mut temps = vec![];
8037 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
8038 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S64, loc_a, loc_b, false)?;
8039 self.assembler.emit_cset(Size::S32, dest, Condition::Eq)?;
8040 if ret != dest {
8041 self.move_location(Size::S32, dest, ret)?;
8042 }
8043 for r in temps {
8044 self.release_gpr(r);
8045 }
8046 Ok(())
8047 }
8048 fn f64_min(
8049 &mut self,
8050 loc_a: Location,
8051 loc_b: Location,
8052 ret: Location,
8053 ) -> Result<(), CompileError> {
8054 let mut temps = vec![];
8055 let old_fpcr = self.set_default_nan(&mut temps)?;
8056 self.emit_relaxed_binop3_neon(
8057 Assembler::emit_fmin,
8058 Size::S64,
8059 loc_a,
8060 loc_b,
8061 ret,
8062 ImmType::None,
8063 )?;
8064 self.restore_fpcr(old_fpcr)?;
8065 for r in temps {
8066 self.release_gpr(r);
8067 }
8068 Ok(())
8069 }
8070 fn f64_max(
8071 &mut self,
8072 loc_a: Location,
8073 loc_b: Location,
8074 ret: Location,
8075 ) -> Result<(), CompileError> {
8076 let mut temps = vec![];
8077 let old_fpcr = self.set_default_nan(&mut temps)?;
8078 self.emit_relaxed_binop3_neon(
8079 Assembler::emit_fmax,
8080 Size::S64,
8081 loc_a,
8082 loc_b,
8083 ret,
8084 ImmType::None,
8085 )?;
8086 self.restore_fpcr(old_fpcr)?;
8087 for r in temps {
8088 self.release_gpr(r);
8089 }
8090 Ok(())
8091 }
8092 fn f64_add(
8093 &mut self,
8094 loc_a: Location,
8095 loc_b: Location,
8096 ret: Location,
8097 ) -> Result<(), CompileError> {
8098 self.emit_relaxed_binop3_neon(
8099 Assembler::emit_fadd,
8100 Size::S64,
8101 loc_a,
8102 loc_b,
8103 ret,
8104 ImmType::None,
8105 )
8106 }
8107 fn f64_sub(
8108 &mut self,
8109 loc_a: Location,
8110 loc_b: Location,
8111 ret: Location,
8112 ) -> Result<(), CompileError> {
8113 self.emit_relaxed_binop3_neon(
8114 Assembler::emit_fsub,
8115 Size::S64,
8116 loc_a,
8117 loc_b,
8118 ret,
8119 ImmType::None,
8120 )
8121 }
8122 fn f64_mul(
8123 &mut self,
8124 loc_a: Location,
8125 loc_b: Location,
8126 ret: Location,
8127 ) -> Result<(), CompileError> {
8128 self.emit_relaxed_binop3_neon(
8129 Assembler::emit_fmul,
8130 Size::S64,
8131 loc_a,
8132 loc_b,
8133 ret,
8134 ImmType::None,
8135 )
8136 }
8137 fn f64_div(
8138 &mut self,
8139 loc_a: Location,
8140 loc_b: Location,
8141 ret: Location,
8142 ) -> Result<(), CompileError> {
8143 self.emit_relaxed_binop3_neon(
8144 Assembler::emit_fdiv,
8145 Size::S64,
8146 loc_a,
8147 loc_b,
8148 ret,
8149 ImmType::None,
8150 )
8151 }
8152 fn f32_neg(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
8153 self.emit_relaxed_binop_neon(Assembler::emit_fneg, Size::S32, loc, ret, true)
8154 }
8155 fn f32_abs(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
8156 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
8157 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
8158 })?;
8159 self.move_location(Size::S32, loc, Location::GPR(tmp))?;
8160 self.assembler.emit_and(
8161 Size::S32,
8162 Location::GPR(tmp),
8163 Location::Imm32(0x7fffffffu32),
8164 Location::GPR(tmp),
8165 )?;
8166 self.move_location(Size::S32, Location::GPR(tmp), ret)?;
8167 self.release_gpr(tmp);
8168 Ok(())
8169 }
8170 fn emit_i32_copysign(&mut self, tmp1: GPR, tmp2: GPR) -> Result<(), CompileError> {
8171 self.assembler.emit_and(
8172 Size::S32,
8173 Location::GPR(tmp1),
8174 Location::Imm32(0x7fffffffu32),
8175 Location::GPR(tmp1),
8176 )?;
8177 self.assembler.emit_and(
8178 Size::S32,
8179 Location::GPR(tmp2),
8180 Location::Imm32(0x80000000u32),
8181 Location::GPR(tmp2),
8182 )?;
8183 self.assembler.emit_or(
8184 Size::S32,
8185 Location::GPR(tmp1),
8186 Location::GPR(tmp2),
8187 Location::GPR(tmp1),
8188 )
8189 }
8190 fn f32_sqrt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
8191 self.emit_relaxed_binop_neon(Assembler::emit_fsqrt, Size::S32, loc, ret, true)
8192 }
8193 fn f32_trunc(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
8194 self.emit_relaxed_binop_neon(Assembler::emit_frintz, Size::S32, loc, ret, true)
8195 }
8196 fn f32_ceil(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
8197 self.emit_relaxed_binop_neon(Assembler::emit_frintp, Size::S32, loc, ret, true)
8198 }
8199 fn f32_floor(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
8200 self.emit_relaxed_binop_neon(Assembler::emit_frintm, Size::S32, loc, ret, true)
8201 }
8202 fn f32_nearest(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
8203 self.emit_relaxed_binop_neon(Assembler::emit_frintn, Size::S32, loc, ret, true)
8204 }
8205 fn f32_cmp_ge(
8206 &mut self,
8207 loc_a: Location,
8208 loc_b: Location,
8209 ret: Location,
8210 ) -> Result<(), CompileError> {
8211 let mut temps = vec![];
8212 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
8213 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S32, loc_b, loc_a, false)?;
8214 self.assembler.emit_cset(Size::S32, dest, Condition::Ls)?;
8215 if ret != dest {
8216 self.move_location(Size::S32, dest, ret)?;
8217 }
8218 for r in temps {
8219 self.release_gpr(r);
8220 }
8221 Ok(())
8222 }
8223 fn f32_cmp_gt(
8224 &mut self,
8225 loc_a: Location,
8226 loc_b: Location,
8227 ret: Location,
8228 ) -> Result<(), CompileError> {
8229 let mut temps = vec![];
8230 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
8231 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S32, loc_b, loc_a, false)?;
8232 self.assembler.emit_cset(Size::S32, dest, Condition::Cc)?;
8233 if ret != dest {
8234 self.move_location(Size::S32, dest, ret)?;
8235 }
8236 for r in temps {
8237 self.release_gpr(r);
8238 }
8239 Ok(())
8240 }
8241 fn f32_cmp_le(
8242 &mut self,
8243 loc_a: Location,
8244 loc_b: Location,
8245 ret: Location,
8246 ) -> Result<(), CompileError> {
8247 let mut temps = vec![];
8248 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
8249 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S32, loc_a, loc_b, false)?;
8250 self.assembler.emit_cset(Size::S32, dest, Condition::Ls)?;
8251 if ret != dest {
8252 self.move_location(Size::S32, dest, ret)?;
8253 }
8254 for r in temps {
8255 self.release_gpr(r);
8256 }
8257 Ok(())
8258 }
8259 fn f32_cmp_lt(
8260 &mut self,
8261 loc_a: Location,
8262 loc_b: Location,
8263 ret: Location,
8264 ) -> Result<(), CompileError> {
8265 let mut temps = vec![];
8266 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
8267 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S32, loc_a, loc_b, false)?;
8268 self.assembler.emit_cset(Size::S32, dest, Condition::Cc)?;
8269 if ret != dest {
8270 self.move_location(Size::S32, dest, ret)?;
8271 }
8272 for r in temps {
8273 self.release_gpr(r);
8274 }
8275 Ok(())
8276 }
8277 fn f32_cmp_ne(
8278 &mut self,
8279 loc_a: Location,
8280 loc_b: Location,
8281 ret: Location,
8282 ) -> Result<(), CompileError> {
8283 let mut temps = vec![];
8284 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
8285 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S32, loc_a, loc_b, false)?;
8286 self.assembler.emit_cset(Size::S32, dest, Condition::Ne)?;
8287 if ret != dest {
8288 self.move_location(Size::S32, dest, ret)?;
8289 }
8290 for r in temps {
8291 self.release_gpr(r);
8292 }
8293 Ok(())
8294 }
8295 fn f32_cmp_eq(
8296 &mut self,
8297 loc_a: Location,
8298 loc_b: Location,
8299 ret: Location,
8300 ) -> Result<(), CompileError> {
8301 let mut temps = vec![];
8302 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
8303 self.emit_relaxed_binop_neon(Assembler::emit_fcmp, Size::S32, loc_a, loc_b, false)?;
8304 self.assembler.emit_cset(Size::S32, dest, Condition::Eq)?;
8305 if ret != dest {
8306 self.move_location(Size::S32, dest, ret)?;
8307 }
8308 for r in temps {
8309 self.release_gpr(r);
8310 }
8311 Ok(())
8312 }
8313 fn f32_min(
8314 &mut self,
8315 loc_a: Location,
8316 loc_b: Location,
8317 ret: Location,
8318 ) -> Result<(), CompileError> {
8319 let mut temps = vec![];
8320 let old_fpcr = self.set_default_nan(&mut temps)?;
8321 self.emit_relaxed_binop3_neon(
8322 Assembler::emit_fmin,
8323 Size::S32,
8324 loc_a,
8325 loc_b,
8326 ret,
8327 ImmType::None,
8328 )?;
8329 self.restore_fpcr(old_fpcr)?;
8330 for r in temps {
8331 self.release_gpr(r);
8332 }
8333 Ok(())
8334 }
8335 fn f32_max(
8336 &mut self,
8337 loc_a: Location,
8338 loc_b: Location,
8339 ret: Location,
8340 ) -> Result<(), CompileError> {
8341 let mut temps = vec![];
8342 let old_fpcr = self.set_default_nan(&mut temps)?;
8343 self.emit_relaxed_binop3_neon(
8344 Assembler::emit_fmax,
8345 Size::S32,
8346 loc_a,
8347 loc_b,
8348 ret,
8349 ImmType::None,
8350 )?;
8351 self.restore_fpcr(old_fpcr)?;
8352 for r in temps {
8353 self.release_gpr(r);
8354 }
8355 Ok(())
8356 }
8357 fn f32_add(
8358 &mut self,
8359 loc_a: Location,
8360 loc_b: Location,
8361 ret: Location,
8362 ) -> Result<(), CompileError> {
8363 self.emit_relaxed_binop3_neon(
8364 Assembler::emit_fadd,
8365 Size::S32,
8366 loc_a,
8367 loc_b,
8368 ret,
8369 ImmType::None,
8370 )
8371 }
8372 fn f32_sub(
8373 &mut self,
8374 loc_a: Location,
8375 loc_b: Location,
8376 ret: Location,
8377 ) -> Result<(), CompileError> {
8378 self.emit_relaxed_binop3_neon(
8379 Assembler::emit_fsub,
8380 Size::S32,
8381 loc_a,
8382 loc_b,
8383 ret,
8384 ImmType::None,
8385 )
8386 }
8387 fn f32_mul(
8388 &mut self,
8389 loc_a: Location,
8390 loc_b: Location,
8391 ret: Location,
8392 ) -> Result<(), CompileError> {
8393 self.emit_relaxed_binop3_neon(
8394 Assembler::emit_fmul,
8395 Size::S32,
8396 loc_a,
8397 loc_b,
8398 ret,
8399 ImmType::None,
8400 )
8401 }
8402 fn f32_div(
8403 &mut self,
8404 loc_a: Location,
8405 loc_b: Location,
8406 ret: Location,
8407 ) -> Result<(), CompileError> {
8408 self.emit_relaxed_binop3_neon(
8409 Assembler::emit_fdiv,
8410 Size::S32,
8411 loc_a,
8412 loc_b,
8413 ret,
8414 ImmType::None,
8415 )
8416 }
8417
8418 fn gen_std_trampoline(
8419 &self,
8420 sig: &FunctionType,
8421 calling_convention: CallingConvention,
8422 ) -> Result<FunctionBody, CompileError> {
8423 gen_std_trampoline_arm64(sig, calling_convention)
8424 }
8425 fn gen_std_dynamic_import_trampoline(
8427 &self,
8428 vmoffsets: &VMOffsets,
8429 sig: &FunctionType,
8430 calling_convention: CallingConvention,
8431 ) -> Result<FunctionBody, CompileError> {
8432 gen_std_dynamic_import_trampoline_arm64(vmoffsets, sig, calling_convention)
8433 }
8434 fn gen_import_call_trampoline(
8436 &self,
8437 vmoffsets: &VMOffsets,
8438 index: FunctionIndex,
8439 sig: &FunctionType,
8440 calling_convention: CallingConvention,
8441 ) -> Result<CustomSection, CompileError> {
8442 gen_import_call_trampoline_arm64(vmoffsets, index, sig, calling_convention)
8443 }
8444 #[cfg(feature = "unwind")]
8445 fn gen_dwarf_unwind_info(&mut self, code_len: usize) -> Option<UnwindInstructions> {
8446 let mut instructions = vec![];
8447 for &(instruction_offset, ref inst) in &self.unwind_ops {
8448 let instruction_offset = instruction_offset as u32;
8449 match *inst {
8450 UnwindOps::PushFP { up_to_sp } => {
8451 instructions.push((
8452 instruction_offset,
8453 CallFrameInstruction::CfaOffset(up_to_sp as i32),
8454 ));
8455 instructions.push((
8456 instruction_offset,
8457 CallFrameInstruction::Offset(AArch64::X29, -(up_to_sp as i32)),
8458 ));
8459 }
8460 UnwindOps::Push2Regs {
8461 reg1,
8462 reg2,
8463 up_to_sp,
8464 } => {
8465 instructions.push((
8466 instruction_offset,
8467 CallFrameInstruction::CfaOffset(up_to_sp as i32),
8468 ));
8469 instructions.push((
8470 instruction_offset,
8471 CallFrameInstruction::Offset(reg2.dwarf_index(), -(up_to_sp as i32) + 8),
8472 ));
8473 instructions.push((
8474 instruction_offset,
8475 CallFrameInstruction::Offset(reg1.dwarf_index(), -(up_to_sp as i32)),
8476 ));
8477 }
8478 UnwindOps::DefineNewFrame => {
8479 instructions.push((
8480 instruction_offset,
8481 CallFrameInstruction::CfaRegister(AArch64::X29),
8482 ));
8483 }
8484 UnwindOps::SaveRegister { reg, bp_neg_offset } => instructions.push((
8485 instruction_offset,
8486 CallFrameInstruction::Offset(reg.dwarf_index(), -bp_neg_offset),
8487 )),
8488 }
8489 }
8490 Some(UnwindInstructions {
8491 instructions,
8492 len: code_len as u32,
8493 })
8494 }
8495 #[cfg(not(feature = "unwind"))]
8496 fn gen_dwarf_unwind_info(&mut self, _code_len: usize) -> Option<UnwindInstructions> {
8497 None
8498 }
8499
8500 fn gen_windows_unwind_info(&mut self, _code_len: usize) -> Option<Vec<u8>> {
8501 None
8502 }
8503}
8504
8505#[cfg(test)]
8506mod test {
8507 use super::*;
8508
8509 fn test_move_location(machine: &mut MachineARM64, size: Size) -> Result<(), CompileError> {
8510 machine.move_location(size, Location::GPR(GPR::X1), Location::GPR(GPR::X2))?;
8511 machine.move_location(size, Location::GPR(GPR::X1), Location::Memory(GPR::X2, 10))?;
8512 machine.move_location(size, Location::GPR(GPR::X1), Location::Memory(GPR::X2, -10))?;
8513 machine.move_location(
8514 size,
8515 Location::GPR(GPR::X1),
8516 Location::Memory(GPR::X2, 1024),
8517 )?;
8518 machine.move_location(
8519 size,
8520 Location::GPR(GPR::X1),
8521 Location::Memory(GPR::X2, -1024),
8522 )?;
8523 machine.move_location(size, Location::Memory(GPR::X2, 10), Location::GPR(GPR::X1))?;
8524 machine.move_location(size, Location::Memory(GPR::X2, -10), Location::GPR(GPR::X1))?;
8525 machine.move_location(
8526 size,
8527 Location::Memory(GPR::X2, 1024),
8528 Location::GPR(GPR::X1),
8529 )?;
8530 machine.move_location(
8531 size,
8532 Location::Memory(GPR::X2, -1024),
8533 Location::GPR(GPR::X1),
8534 )?;
8535 machine.move_location(size, Location::GPR(GPR::X1), Location::SIMD(NEON::V0))?;
8536 machine.move_location(size, Location::SIMD(NEON::V0), Location::GPR(GPR::X1))?;
8537 machine.move_location(
8538 size,
8539 Location::SIMD(NEON::V0),
8540 Location::Memory(GPR::X2, 10),
8541 )?;
8542 machine.move_location(
8543 size,
8544 Location::SIMD(NEON::V0),
8545 Location::Memory(GPR::X2, -10),
8546 )?;
8547 machine.move_location(
8548 size,
8549 Location::SIMD(NEON::V0),
8550 Location::Memory(GPR::X2, 1024),
8551 )?;
8552 machine.move_location(
8553 size,
8554 Location::SIMD(NEON::V0),
8555 Location::Memory(GPR::X2, -1024),
8556 )?;
8557 machine.move_location(
8558 size,
8559 Location::Memory(GPR::X2, 10),
8560 Location::SIMD(NEON::V0),
8561 )?;
8562 machine.move_location(
8563 size,
8564 Location::Memory(GPR::X2, -10),
8565 Location::SIMD(NEON::V0),
8566 )?;
8567 machine.move_location(
8568 size,
8569 Location::Memory(GPR::X2, 1024),
8570 Location::SIMD(NEON::V0),
8571 )?;
8572 machine.move_location(
8573 size,
8574 Location::Memory(GPR::X2, -1024),
8575 Location::SIMD(NEON::V0),
8576 )?;
8577
8578 Ok(())
8579 }
8580
8581 fn test_move_location_extended(
8582 machine: &mut MachineARM64,
8583 signed: bool,
8584 sized: Size,
8585 ) -> Result<(), CompileError> {
8586 machine.move_location_extend(
8587 sized,
8588 signed,
8589 Location::GPR(GPR::X0),
8590 Size::S64,
8591 Location::GPR(GPR::X1),
8592 )?;
8593 machine.move_location_extend(
8594 sized,
8595 signed,
8596 Location::GPR(GPR::X0),
8597 Size::S64,
8598 Location::Memory(GPR::X1, 10),
8599 )?;
8600 machine.move_location_extend(
8601 sized,
8602 signed,
8603 Location::GPR(GPR::X0),
8604 Size::S64,
8605 Location::Memory(GPR::X1, 16),
8606 )?;
8607 machine.move_location_extend(
8608 sized,
8609 signed,
8610 Location::GPR(GPR::X0),
8611 Size::S64,
8612 Location::Memory(GPR::X1, -16),
8613 )?;
8614 machine.move_location_extend(
8615 sized,
8616 signed,
8617 Location::GPR(GPR::X0),
8618 Size::S64,
8619 Location::Memory(GPR::X1, 1024),
8620 )?;
8621 machine.move_location_extend(
8622 sized,
8623 signed,
8624 Location::GPR(GPR::X0),
8625 Size::S64,
8626 Location::Memory(GPR::X1, -1024),
8627 )?;
8628 machine.move_location_extend(
8629 sized,
8630 signed,
8631 Location::Memory(GPR::X0, 10),
8632 Size::S64,
8633 Location::GPR(GPR::X1),
8634 )?;
8635
8636 Ok(())
8637 }
8638
8639 fn test_binop_op(
8640 machine: &mut MachineARM64,
8641 op: fn(&mut MachineARM64, Location, Location, Location) -> Result<(), CompileError>,
8642 ) -> Result<(), CompileError> {
8643 op(
8644 machine,
8645 Location::GPR(GPR::X2),
8646 Location::GPR(GPR::X2),
8647 Location::GPR(GPR::X0),
8648 )?;
8649 op(
8650 machine,
8651 Location::GPR(GPR::X2),
8652 Location::Imm32(10),
8653 Location::GPR(GPR::X0),
8654 )?;
8655 op(
8656 machine,
8657 Location::GPR(GPR::X0),
8658 Location::GPR(GPR::X0),
8659 Location::GPR(GPR::X0),
8660 )?;
8661 op(
8662 machine,
8663 Location::Imm32(10),
8664 Location::GPR(GPR::X2),
8665 Location::GPR(GPR::X0),
8666 )?;
8667 op(
8668 machine,
8669 Location::GPR(GPR::X0),
8670 Location::GPR(GPR::X2),
8671 Location::Memory(GPR::X0, 10),
8672 )?;
8673 op(
8674 machine,
8675 Location::GPR(GPR::X0),
8676 Location::Memory(GPR::X2, 16),
8677 Location::Memory(GPR::X0, 10),
8678 )?;
8679 op(
8680 machine,
8681 Location::Memory(GPR::X0, 0),
8682 Location::Memory(GPR::X2, 16),
8683 Location::Memory(GPR::X0, 10),
8684 )?;
8685
8686 Ok(())
8687 }
8688
8689 fn test_float_binop_op(
8690 machine: &mut MachineARM64,
8691 op: fn(&mut MachineARM64, Location, Location, Location) -> Result<(), CompileError>,
8692 ) -> Result<(), CompileError> {
8693 op(
8694 machine,
8695 Location::SIMD(NEON::V3),
8696 Location::SIMD(NEON::V2),
8697 Location::SIMD(NEON::V0),
8698 )?;
8699 op(
8700 machine,
8701 Location::SIMD(NEON::V0),
8702 Location::SIMD(NEON::V2),
8703 Location::SIMD(NEON::V0),
8704 )?;
8705 op(
8706 machine,
8707 Location::SIMD(NEON::V0),
8708 Location::SIMD(NEON::V0),
8709 Location::SIMD(NEON::V0),
8710 )?;
8711 op(
8712 machine,
8713 Location::Memory(GPR::X0, 0),
8714 Location::SIMD(NEON::V2),
8715 Location::SIMD(NEON::V0),
8716 )?;
8717 op(
8718 machine,
8719 Location::Memory(GPR::X0, 0),
8720 Location::Memory(GPR::X1, 10),
8721 Location::SIMD(NEON::V0),
8722 )?;
8723 op(
8724 machine,
8725 Location::Memory(GPR::X0, 0),
8726 Location::Memory(GPR::X1, 16),
8727 Location::Memory(GPR::X2, 32),
8728 )?;
8729 op(
8730 machine,
8731 Location::SIMD(NEON::V0),
8732 Location::Memory(GPR::X1, 16),
8733 Location::Memory(GPR::X2, 32),
8734 )?;
8735 op(
8736 machine,
8737 Location::SIMD(NEON::V0),
8738 Location::SIMD(NEON::V1),
8739 Location::Memory(GPR::X2, 32),
8740 )?;
8741
8742 Ok(())
8743 }
8744
8745 fn test_float_cmp_op(
8746 machine: &mut MachineARM64,
8747 op: fn(&mut MachineARM64, Location, Location, Location) -> Result<(), CompileError>,
8748 ) -> Result<(), CompileError> {
8749 op(
8750 machine,
8751 Location::SIMD(NEON::V3),
8752 Location::SIMD(NEON::V2),
8753 Location::GPR(GPR::X0),
8754 )?;
8755 op(
8756 machine,
8757 Location::SIMD(NEON::V0),
8758 Location::SIMD(NEON::V0),
8759 Location::GPR(GPR::X0),
8760 )?;
8761 op(
8762 machine,
8763 Location::Memory(GPR::X1, 0),
8764 Location::SIMD(NEON::V2),
8765 Location::GPR(GPR::X0),
8766 )?;
8767 op(
8768 machine,
8769 Location::Memory(GPR::X1, 0),
8770 Location::Memory(GPR::X2, 10),
8771 Location::GPR(GPR::X0),
8772 )?;
8773 op(
8774 machine,
8775 Location::Memory(GPR::X1, 0),
8776 Location::Memory(GPR::X2, 16),
8777 Location::Memory(GPR::X0, 32),
8778 )?;
8779 op(
8780 machine,
8781 Location::SIMD(NEON::V0),
8782 Location::Memory(GPR::X2, 16),
8783 Location::Memory(GPR::X0, 32),
8784 )?;
8785 op(
8786 machine,
8787 Location::SIMD(NEON::V0),
8788 Location::SIMD(NEON::V1),
8789 Location::Memory(GPR::X0, 32),
8790 )?;
8791
8792 Ok(())
8793 }
8794
8795 #[test]
8796 fn tests_arm64() -> Result<(), CompileError> {
8797 let mut machine = MachineARM64::new(None);
8798
8799 test_move_location(&mut machine, Size::S32)?;
8800 test_move_location(&mut machine, Size::S64)?;
8801 test_move_location_extended(&mut machine, false, Size::S8)?;
8802 test_move_location_extended(&mut machine, false, Size::S16)?;
8803 test_move_location_extended(&mut machine, false, Size::S32)?;
8804 test_move_location_extended(&mut machine, true, Size::S8)?;
8805 test_move_location_extended(&mut machine, true, Size::S16)?;
8806 test_move_location_extended(&mut machine, true, Size::S32)?;
8807 test_binop_op(&mut machine, MachineARM64::emit_binop_add32)?;
8808 test_binop_op(&mut machine, MachineARM64::emit_binop_add64)?;
8809 test_binop_op(&mut machine, MachineARM64::emit_binop_sub32)?;
8810 test_binop_op(&mut machine, MachineARM64::emit_binop_sub64)?;
8811 test_binop_op(&mut machine, MachineARM64::emit_binop_and32)?;
8812 test_binop_op(&mut machine, MachineARM64::emit_binop_and64)?;
8813 test_binop_op(&mut machine, MachineARM64::emit_binop_xor32)?;
8814 test_binop_op(&mut machine, MachineARM64::emit_binop_xor64)?;
8815 test_binop_op(&mut machine, MachineARM64::emit_binop_or32)?;
8816 test_binop_op(&mut machine, MachineARM64::emit_binop_or64)?;
8817 test_binop_op(&mut machine, MachineARM64::emit_binop_mul32)?;
8818 test_binop_op(&mut machine, MachineARM64::emit_binop_mul64)?;
8819 test_float_binop_op(&mut machine, MachineARM64::f32_add)?;
8820 test_float_binop_op(&mut machine, MachineARM64::f32_sub)?;
8821 test_float_binop_op(&mut machine, MachineARM64::f32_mul)?;
8822 test_float_binop_op(&mut machine, MachineARM64::f32_div)?;
8823 test_float_cmp_op(&mut machine, MachineARM64::f32_cmp_eq)?;
8824 test_float_cmp_op(&mut machine, MachineARM64::f32_cmp_lt)?;
8825 test_float_cmp_op(&mut machine, MachineARM64::f32_cmp_le)?;
8826
8827 Ok(())
8828 }
8829}