1use dynasmrt::{DynasmError, VecAssembler, riscv::RiscvRelocation};
4use fixedbitset::FixedBitSet;
5#[cfg(feature = "unwind")]
6use gimli::{RiscV, write::CallFrameInstruction};
7
8use wasmer_compiler::{
9 CANONICAL_NAN_F32, CANONICAL_NAN_F64,
10 types::{
11 address_map::InstructionAddressMap,
12 function::FunctionBody,
13 relocation::{Relocation, RelocationKind, RelocationTarget},
14 section::CustomSection,
15 },
16 wasmparser::MemArg,
17};
18use wasmer_types::{
19 CompileError, FunctionIndex, FunctionType, SourceLoc, TrapCode, TrapInformation, VMOffsets,
20 target::{CallingConvention, Target},
21};
22
23use crate::{
24 codegen_error,
25 common_decl::*,
26 emitter_riscv::*,
27 location::{Location as AbstractLocation, Reg},
28 machine::*,
29 riscv_decl::{FPR, GPR},
30 unwind::{UnwindInstructions, UnwindOps, UnwindRegister},
31};
32
33type Assembler = VecAssembler<RiscvRelocation>;
34type Location = AbstractLocation<GPR, FPR>;
35
36use std::{
37 collections::HashMap,
38 ops::{Deref, DerefMut},
39};
40pub struct AssemblerRiscv {
42 pub inner: Assembler,
44}
45
46impl AssemblerRiscv {
47 pub fn new(base_addr: usize, _target: Option<Target>) -> Result<Self, CompileError> {
49 Ok(Self {
51 inner: Assembler::new(base_addr),
52 })
53 }
54
55 pub fn finalize(self) -> Result<Vec<u8>, DynasmError> {
57 self.inner.finalize()
58 }
59}
60
61impl Deref for AssemblerRiscv {
62 type Target = Assembler;
63 fn deref(&self) -> &Self::Target {
64 &self.inner
65 }
66}
67
68impl DerefMut for AssemblerRiscv {
69 fn deref_mut(&mut self) -> &mut Self::Target {
70 &mut self.inner
71 }
72}
73
74pub struct MachineRiscv {
76 assembler: AssemblerRiscv,
77 used_gprs: FixedBitSet,
78 used_fprs: FixedBitSet,
79 trap_table: TrapTable,
80 instructions_address_map: Vec<InstructionAddressMap>,
83 src_loc: u32,
85 unwind_ops: Vec<(usize, UnwindOps<GPR, FPR>)>,
87}
88
89const SCRATCH_REG: GPR = GPR::X28;
90
91impl MachineRiscv {
92 pub fn new(target: Option<Target>) -> Result<Self, CompileError> {
94 Ok(MachineRiscv {
96 assembler: AssemblerRiscv::new(0, target)?,
97 used_gprs: FixedBitSet::with_capacity(32),
98 used_fprs: FixedBitSet::with_capacity(32),
99 trap_table: TrapTable::default(),
100 instructions_address_map: vec![],
101 src_loc: 0,
102 unwind_ops: vec![],
103 })
104 }
105
106 fn used_gprs_contains(&self, r: &GPR) -> bool {
107 self.used_gprs.contains(r.into_index())
108 }
109 fn used_gprs_insert(&mut self, r: GPR) {
110 self.used_gprs.insert(r.into_index());
111 }
112 fn used_gprs_remove(&mut self, r: &GPR) -> bool {
113 let ret = self.used_gprs_contains(r);
114 self.used_gprs.set(r.into_index(), false);
115 ret
116 }
117
118 fn used_fp_contains(&self, r: &FPR) -> bool {
119 self.used_fprs.contains(r.into_index())
120 }
121 fn used_fprs_insert(&mut self, r: FPR) {
122 self.used_fprs.insert(r.into_index());
123 }
124 fn used_fprs_remove(&mut self, r: &FPR) -> bool {
125 let ret = self.used_fp_contains(r);
126 self.used_fprs.set(r.into_index(), false);
127 ret
128 }
129
130 fn location_to_reg(
131 &mut self,
132 sz: Size,
133 src: Location,
134 temps: &mut Vec<GPR>,
135 allow_imm: ImmType,
136 read_val: bool,
137 wanted: Option<GPR>,
138 ) -> Result<Location, CompileError> {
139 match src {
140 Location::GPR(_) | Location::SIMD(_) => Ok(src),
141 Location::Memory(reg, val) => {
142 let tmp = if let Some(wanted) = wanted {
143 wanted
144 } else {
145 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
146 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
147 })?;
148 temps.push(tmp);
149 tmp
150 };
151 if read_val {
152 if ImmType::Bits12.compatible_imm(val as _) {
153 self.assembler.emit_ld(sz, false, Location::GPR(tmp), src)?;
154 } else {
155 if reg == tmp {
156 codegen_error!("singlepass reg == tmp unreachable");
157 }
158 self.assembler.emit_mov_imm(Location::GPR(tmp), val as _)?;
159 self.assembler.emit_add(
160 Size::S64,
161 Location::GPR(reg),
162 Location::GPR(tmp),
163 Location::GPR(tmp),
164 )?;
165 self.assembler.emit_ld(
166 sz,
167 false,
168 Location::GPR(tmp),
169 Location::Memory(tmp, 0),
170 )?;
171 }
172 }
173 Ok(Location::GPR(tmp))
174 }
175 _ if src.is_imm() => {
176 let imm = src.imm_value_scalar().unwrap();
177 if imm == 0 {
178 Ok(Location::GPR(GPR::XZero))
179 } else if allow_imm.compatible_imm(imm) {
180 Ok(src)
181 } else {
182 let tmp = if let Some(wanted) = wanted {
183 wanted
184 } else {
185 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
186 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
187 })?;
188 temps.push(tmp);
189 tmp
190 };
191 self.assembler.emit_mov_imm(Location::GPR(tmp), imm as _)?;
192 Ok(Location::GPR(tmp))
193 }
194 }
195 _ => todo!("unsupported location"),
196 }
197 }
198
199 fn location_to_fpr(
200 &mut self,
201 sz: Size,
202 src: Location,
203 temps: &mut Vec<FPR>,
204 allow_imm: ImmType,
205 read_val: bool,
206 ) -> Result<Location, CompileError> {
207 match src {
208 Location::SIMD(_) => Ok(src),
209 Location::GPR(_) => {
210 let tmp = self.acquire_temp_simd().ok_or_else(|| {
211 CompileError::Codegen("singlepass cannot acquire temp fpr".to_owned())
212 })?;
213 temps.push(tmp);
214 if read_val {
215 self.assembler.emit_mov(sz, src, Location::SIMD(tmp))?;
216 }
217 Ok(Location::SIMD(tmp))
218 }
219 Location::Memory(_, _) => {
220 let tmp = self.acquire_temp_simd().ok_or_else(|| {
221 CompileError::Codegen("singlepass cannot acquire temp fpr".to_owned())
222 })?;
223 temps.push(tmp);
224 if read_val {
225 self.assembler
226 .emit_ld(sz, false, Location::SIMD(tmp), src)?;
227 }
228 Ok(Location::SIMD(tmp))
229 }
230 _ if src.is_imm() => {
231 let tmp = self.acquire_temp_simd().ok_or_else(|| {
232 CompileError::Codegen("singlepass cannot acquire temp fpr".to_owned())
233 })?;
234 temps.push(tmp);
235
236 let mut gpr_temps = vec![];
237 let dst =
238 self.location_to_reg(sz, src, &mut gpr_temps, allow_imm, read_val, None)?;
239 self.assembler.emit_mov(sz, dst, Location::SIMD(tmp))?;
240 for r in gpr_temps {
241 self.release_gpr(r);
242 }
243
244 Ok(Location::SIMD(tmp))
245 }
246 _ => todo!("unsupported location"),
247 }
248 }
249
250 fn emit_relaxed_binop(
251 &mut self,
252 op: fn(&mut Assembler, Size, Location, Location) -> Result<(), CompileError>,
253 sz: Size,
254 src: Location,
255 dst: Location,
256 ) -> Result<(), CompileError> {
257 let mut temps = vec![];
258 let src = self.location_to_reg(sz, src, &mut temps, ImmType::None, true, None)?;
259 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
260 op(&mut self.assembler, sz, src, dest)?;
261 if dst != dest {
262 self.move_location(sz, dest, dst)?;
263 }
264 for r in temps {
265 self.release_gpr(r);
266 }
267 Ok(())
268 }
269
270 fn emit_relaxed_binop_fp(
271 &mut self,
272 op: fn(&mut Assembler, Size, Location, Location) -> Result<(), CompileError>,
273 sz: Size,
274 src: Location,
275 dst: Location,
276 putback: bool,
277 ) -> Result<(), CompileError> {
278 let mut temps = vec![];
279 let src = self.location_to_fpr(sz, src, &mut temps, ImmType::None, true)?;
280 let dest = self.location_to_fpr(sz, dst, &mut temps, ImmType::None, !putback)?;
281 op(&mut self.assembler, sz, src, dest)?;
282 if dst != dest && putback {
283 self.move_location(sz, dest, dst)?;
284 }
285 for r in temps {
286 self.release_simd(r);
287 }
288 Ok(())
289 }
290
291 fn emit_relaxed_binop3(
292 &mut self,
293 op: fn(&mut Assembler, Size, Location, Location, Location) -> Result<(), CompileError>,
294 sz: Size,
295 src1: Location,
296 src2: Location,
297 dst: Location,
298 allow_imm: ImmType,
299 ) -> Result<(), CompileError> {
300 let mut temps = vec![];
301 let src1 = self.location_to_reg(sz, src1, &mut temps, ImmType::None, true, None)?;
302 let src2 = self.location_to_reg(sz, src2, &mut temps, allow_imm, true, None)?;
303 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
304 op(&mut self.assembler, sz, src1, src2, dest)?;
305 if dst != dest {
306 self.move_location(sz, dest, dst)?;
307 }
308 for r in temps {
309 self.release_gpr(r);
310 }
311 Ok(())
312 }
313
314 fn emit_relaxed_atomic_binop3(
315 &mut self,
316 op: AtomicBinaryOp,
317 sz: Size,
318 dst: Location,
319 addr: GPR,
320 src: Location,
321 ) -> Result<(), CompileError> {
322 let mut temps = vec![];
323 let source = self.location_to_reg(sz, src, &mut temps, ImmType::None, false, None)?;
324 let dest = self.location_to_reg(Size::S64, dst, &mut temps, ImmType::None, false, None)?;
325 let (Location::GPR(source), Location::GPR(dest)) = (source, dest) else {
326 panic!("emit_relaxed_atomic_binop3 expects locations in registers");
327 };
328
329 match sz {
332 Size::S32 | Size::S64 => {
333 if op == AtomicBinaryOp::Sub {
334 self.assembler.emit_neg(
335 Size::S64,
336 Location::GPR(source),
337 Location::GPR(source),
338 )?;
339 self.assembler.emit_atomic_binop(
340 AtomicBinaryOp::Add,
341 sz,
342 dest,
343 addr,
344 source,
345 )?;
346 } else {
347 self.assembler
348 .emit_atomic_binop(op, sz, dest, addr, source)?;
349 }
350 self.assembler.emit_rwfence()?;
351 }
352 Size::S8 | Size::S16 => {
353 let aligned_addr = self.acquire_temp_gpr().ok_or_else(|| {
354 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
355 })?;
356 temps.push(aligned_addr);
357 let bit_offset = self.acquire_temp_gpr().ok_or_else(|| {
358 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
359 })?;
360 temps.push(bit_offset);
361 let bit_mask = self.acquire_temp_gpr().ok_or_else(|| {
362 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
363 })?;
364 temps.push(bit_mask);
365
366 self.assembler.emit_and(
367 Size::S64,
368 Location::GPR(addr),
369 Location::Imm64(-4i64 as _),
370 Location::GPR(aligned_addr),
371 )?;
372 self.assembler.emit_and(
373 Size::S64,
374 Location::GPR(addr),
375 Location::Imm64(3),
376 Location::GPR(bit_offset),
377 )?;
378 self.assembler.emit_sll(
379 Size::S64,
380 Location::GPR(bit_offset),
381 Location::Imm64(3),
382 Location::GPR(bit_offset),
383 )?;
384 self.assembler.emit_mov_imm(
385 Location::GPR(bit_mask),
386 if sz == Size::S8 {
387 u8::MAX as _
388 } else {
389 u16::MAX as _
390 },
391 )?;
392 self.assembler.emit_and(
393 Size::S32,
394 Location::GPR(source),
395 Location::GPR(bit_mask),
396 Location::GPR(source),
397 )?;
398 self.assembler.emit_sll(
399 Size::S64,
400 Location::GPR(bit_mask),
401 Location::GPR(bit_offset),
402 Location::GPR(bit_mask),
403 )?;
404 self.assembler.emit_sll(
405 Size::S64,
406 Location::GPR(source),
407 Location::GPR(bit_offset),
408 Location::GPR(source),
409 )?;
410
411 match op {
412 AtomicBinaryOp::Add | AtomicBinaryOp::Sub | AtomicBinaryOp::Exchange => {
413 let loaded_value = self.acquire_temp_gpr().ok_or_else(|| {
414 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
415 })?;
416 temps.push(loaded_value);
417 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
418 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
419 })?;
420 temps.push(tmp);
421
422 let label_retry = self.get_label();
424 self.emit_label(label_retry)?;
425
426 self.assembler
427 .emit_reserved_ld(Size::S32, loaded_value, aligned_addr)?;
428
429 match op {
430 AtomicBinaryOp::Add => self.assembler.emit_add(
431 Size::S64,
432 Location::GPR(loaded_value),
433 Location::GPR(source),
434 Location::GPR(tmp),
435 )?,
436 AtomicBinaryOp::Sub => self.assembler.emit_sub(
437 Size::S64,
438 Location::GPR(loaded_value),
439 Location::GPR(source),
440 Location::GPR(tmp),
441 )?,
442 AtomicBinaryOp::Exchange => self.assembler.emit_mov(
443 Size::S64,
444 Location::GPR(source),
445 Location::GPR(tmp),
446 )?,
447 _ => unreachable!(),
448 }
449
450 self.assembler.emit_xor(
451 Size::S64,
452 Location::GPR(tmp),
453 Location::GPR(loaded_value),
454 Location::GPR(tmp),
455 )?;
456 self.assembler.emit_and(
457 Size::S64,
458 Location::GPR(tmp),
459 Location::GPR(bit_mask),
460 Location::GPR(tmp),
461 )?;
462 self.assembler.emit_xor(
463 Size::S64,
464 Location::GPR(tmp),
465 Location::GPR(loaded_value),
466 Location::GPR(tmp),
467 )?;
468 self.assembler
469 .emit_reserved_sd(Size::S32, tmp, aligned_addr, tmp)?;
470 let tmp2 = self.acquire_temp_gpr().ok_or_else(|| {
471 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
472 })?;
473 temps.push(tmp2);
474 self.assembler
475 .emit_on_true_label(Location::GPR(tmp), label_retry, tmp2)?;
476
477 self.assembler.emit_rwfence()?;
478
479 self.assembler.emit_and(
481 Size::S32,
482 Location::GPR(loaded_value),
483 Location::GPR(bit_mask),
484 Location::GPR(dest),
485 )?;
486 self.assembler.emit_srl(
487 Size::S32,
488 Location::GPR(dest),
489 Location::GPR(bit_offset),
490 Location::GPR(dest),
491 )?;
492 }
493 AtomicBinaryOp::Or | AtomicBinaryOp::Xor => {
494 self.assembler.emit_atomic_binop(
495 op,
496 Size::S32,
497 dest,
498 aligned_addr,
499 source,
500 )?;
501 self.assembler.emit_rwfence()?;
502 self.assembler.emit_and(
503 Size::S32,
504 Location::GPR(dest),
505 Location::GPR(bit_mask),
506 Location::GPR(dest),
507 )?;
508 self.assembler.emit_srl(
509 Size::S32,
510 Location::GPR(dest),
511 Location::GPR(bit_offset),
512 Location::GPR(dest),
513 )?;
514 }
515 AtomicBinaryOp::And => {
516 self.assembler.emit_not(
517 Size::S64,
518 Location::GPR(bit_mask),
519 Location::GPR(bit_mask),
520 )?;
521 self.assembler.emit_or(
522 Size::S64,
523 Location::GPR(bit_mask),
524 Location::GPR(source),
525 Location::GPR(source),
526 )?;
527 self.assembler.emit_not(
528 Size::S64,
529 Location::GPR(bit_mask),
530 Location::GPR(bit_mask),
531 )?;
532 self.assembler.emit_atomic_binop(
533 op,
534 Size::S32,
535 dest,
536 aligned_addr,
537 source,
538 )?;
539 self.assembler.emit_rwfence()?;
540 self.assembler.emit_and(
541 Size::S32,
542 Location::GPR(dest),
543 Location::GPR(bit_mask),
544 Location::GPR(dest),
545 )?;
546 self.assembler.emit_srl(
547 Size::S32,
548 Location::GPR(dest),
549 Location::GPR(bit_offset),
550 Location::GPR(dest),
551 )?;
552 }
553 }
554 }
555 }
556
557 if dst != Location::GPR(dest) {
558 self.move_location(sz, Location::GPR(dest), dst)?;
559 }
560
561 for r in temps {
562 self.release_gpr(r);
563 }
564 Ok(())
565 }
566
567 fn emit_relaxed_atomic_cmpxchg(
568 &mut self,
569 size: Size,
570 dst: Location,
571 addr: GPR,
572 new: Location,
573 cmp: Location,
574 ) -> Result<(), CompileError> {
575 let mut temps = vec![];
576 let cmp = self.location_to_reg(size, cmp, &mut temps, ImmType::None, true, None)?;
577 let new = self.location_to_reg(size, new, &mut temps, ImmType::None, true, None)?;
578 let (Location::GPR(cmp), Location::GPR(new)) = (cmp, new) else {
579 panic!("emit_relaxed_atomic_cmpxchg expects locations in registers");
580 };
581
582 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
583 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
584 })?;
585 temps.push(jmp_tmp);
586
587 match size {
588 Size::S32 | Size::S64 => {
589 let value = self.acquire_temp_gpr().ok_or_else(|| {
590 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
591 })?;
592 temps.push(value);
593 let cond = self.acquire_temp_gpr().ok_or_else(|| {
594 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
595 })?;
596 temps.push(cond);
597
598 let label_retry = self.get_label();
600 let label_after_retry = self.get_label();
601 self.emit_label(label_retry)?;
602
603 self.assembler.emit_reserved_ld(size, value, addr)?;
604 self.assembler.emit_cmp(
605 Condition::Eq,
606 Location::GPR(value),
607 Location::GPR(cmp),
608 Location::GPR(cond),
609 )?;
610 self.assembler.emit_on_false_label(
611 Location::GPR(cond),
612 label_after_retry,
613 jmp_tmp,
614 )?;
615 self.assembler.emit_reserved_sd(size, cond, addr, new)?;
616 self.assembler
617 .emit_on_true_label(Location::GPR(cond), label_retry, jmp_tmp)?;
618
619 self.assembler.emit_rwfence()?;
621 self.emit_label(label_after_retry)?;
622
623 self.assembler.emit_mov(size, Location::GPR(value), dst)?;
624 }
625 Size::S8 | Size::S16 => {
626 let value = self.acquire_temp_gpr().ok_or_else(|| {
627 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
628 })?;
629 temps.push(value);
630 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
631 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
632 })?;
633 temps.push(tmp);
634 let bit_offset = self.acquire_temp_gpr().ok_or_else(|| {
635 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
636 })?;
637 temps.push(bit_offset);
638 let bit_mask = self.acquire_temp_gpr().ok_or_else(|| {
639 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
640 })?;
641 temps.push(bit_mask);
642 let cond = self.acquire_temp_gpr().ok_or_else(|| {
643 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
644 })?;
645 temps.push(cond);
646
647 self.assembler.emit_and(
649 Size::S64,
650 Location::GPR(addr),
651 Location::Imm64(3),
652 Location::GPR(bit_offset),
653 )?;
654 self.assembler.emit_and(
655 Size::S64,
656 Location::GPR(addr),
657 Location::Imm64(-4i64 as _),
658 Location::GPR(addr),
659 )?;
660 self.assembler.emit_sll(
661 Size::S64,
662 Location::GPR(bit_offset),
663 Location::Imm64(3),
664 Location::GPR(bit_offset),
665 )?;
666 self.assembler.emit_mov_imm(
667 Location::GPR(bit_mask),
668 if size == Size::S8 {
669 u8::MAX as _
670 } else {
671 u16::MAX as _
672 },
673 )?;
674 self.assembler.emit_and(
675 Size::S32,
676 Location::GPR(new),
677 Location::GPR(bit_mask),
678 Location::GPR(new),
679 )?;
680 self.assembler.emit_sll(
681 Size::S64,
682 Location::GPR(bit_mask),
683 Location::GPR(bit_offset),
684 Location::GPR(bit_mask),
685 )?;
686 self.assembler.emit_sll(
687 Size::S64,
688 Location::GPR(new),
689 Location::GPR(bit_offset),
690 Location::GPR(new),
691 )?;
692 self.assembler.emit_sll(
693 Size::S64,
694 Location::GPR(cmp),
695 Location::GPR(bit_offset),
696 Location::GPR(cmp),
697 )?;
698
699 let label_retry = self.get_label();
701 let label_after_retry = self.get_label();
702 self.emit_label(label_retry)?;
703
704 self.assembler.emit_reserved_ld(Size::S32, value, addr)?;
705 self.assembler.emit_and(
706 Size::S32,
707 Location::GPR(value),
708 Location::GPR(bit_mask),
709 Location::GPR(tmp),
710 )?;
711
712 self.assembler.emit_cmp(
713 Condition::Eq,
714 Location::GPR(tmp),
715 Location::GPR(cmp),
716 Location::GPR(cond),
717 )?;
718 self.assembler.emit_on_false_label(
719 Location::GPR(cond),
720 label_after_retry,
721 jmp_tmp,
722 )?;
723
724 self.assembler.emit_xor(
726 Size::S32,
727 Location::GPR(value),
728 Location::GPR(new),
729 Location::GPR(tmp),
730 )?;
731 self.assembler.emit_and(
732 Size::S32,
733 Location::GPR(tmp),
734 Location::GPR(bit_mask),
735 Location::GPR(tmp),
736 )?;
737 self.assembler.emit_xor(
738 Size::S32,
739 Location::GPR(tmp),
740 Location::GPR(value),
741 Location::GPR(tmp),
742 )?;
743 self.assembler
744 .emit_reserved_sd(Size::S32, cond, addr, tmp)?;
745 self.assembler
746 .emit_on_true_label(Location::GPR(cond), label_retry, jmp_tmp)?;
747
748 self.assembler.emit_rwfence()?;
750 self.emit_label(label_after_retry)?;
751
752 self.assembler.emit_and(
753 Size::S32,
754 Location::GPR(value),
755 Location::GPR(bit_mask),
756 Location::GPR(tmp),
757 )?;
758 self.assembler.emit_srl(
759 Size::S32,
760 Location::GPR(tmp),
761 Location::GPR(bit_offset),
762 Location::GPR(tmp),
763 )?;
764 self.assembler
765 .emit_mov(Size::S32, Location::GPR(tmp), dst)?;
766 }
767 }
768
769 for r in temps {
770 self.release_gpr(r);
771 }
772 Ok(())
773 }
774
775 #[allow(clippy::too_many_arguments)]
776 fn emit_relaxed_binop3_fp(
777 &mut self,
778 op: fn(&mut Assembler, Size, Location, Location, Location) -> Result<(), CompileError>,
779 sz: Size,
780 src1: Location,
781 src2: Location,
782 dst: Location,
783 allow_imm: ImmType,
784 return_nan_if_present: bool,
785 ) -> Result<(), CompileError> {
786 let mut temps = vec![];
787 let mut gprs = vec![];
788 let src1 = self.location_to_fpr(sz, src1, &mut temps, ImmType::None, true)?;
789 let src2 = self.location_to_fpr(sz, src2, &mut temps, allow_imm, true)?;
790 let dest = self.location_to_fpr(sz, dst, &mut temps, ImmType::None, false)?;
791
792 let label_after = self.get_label();
793 if return_nan_if_present {
794 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
795 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
796 })?;
797 gprs.push(tmp);
798 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
799 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
800 })?;
801 gprs.push(jmp_tmp);
802
803 let canonical_nan = match sz {
805 Size::S32 => CANONICAL_NAN_F32 as u64,
806 Size::S64 => CANONICAL_NAN_F64,
807 _ => unreachable!(),
808 };
809
810 self.assembler
811 .emit_mov_imm(Location::GPR(tmp), canonical_nan as _)?;
812 self.assembler.emit_mov(sz, Location::GPR(tmp), dest)?;
813
814 self.assembler
815 .emit_fcmp(Condition::Eq, sz, src1, src1, Location::GPR(tmp))?;
816 self.assembler
817 .emit_on_false_label(Location::GPR(tmp), label_after, jmp_tmp)?;
818
819 self.assembler
820 .emit_fcmp(Condition::Eq, sz, src2, src2, Location::GPR(tmp))?;
821 self.assembler
822 .emit_on_false_label(Location::GPR(tmp), label_after, jmp_tmp)?;
823 }
824
825 op(&mut self.assembler, sz, src1, src2, dest)?;
826 self.emit_label(label_after)?;
827
828 if dst != dest {
829 self.move_location(sz, dest, dst)?;
830 }
831 for r in temps {
832 self.release_simd(r);
833 }
834 for r in gprs {
835 self.release_gpr(r);
836 }
837 Ok(())
838 }
839
840 fn emit_relaxed_cmp(
841 &mut self,
842 c: Condition,
843 loc_a: Location,
844 loc_b: Location,
845 ret: Location,
846 sz: Size,
847 signed: bool,
848 ) -> Result<(), CompileError> {
849 let mut temps = vec![];
851 let loc_a = self.location_to_reg(sz, loc_a, &mut temps, ImmType::None, true, None)?;
852 let loc_b = self.location_to_reg(sz, loc_b, &mut temps, ImmType::None, true, None)?;
853
854 if sz != Size::S64 {
855 self.assembler.emit_extend(sz, signed, loc_a, loc_a)?;
856 self.assembler.emit_extend(sz, signed, loc_b, loc_b)?;
857 }
858
859 let dest = self.location_to_reg(sz, ret, &mut temps, ImmType::None, false, None)?;
860 self.assembler.emit_cmp(c, loc_a, loc_b, dest)?;
861 if ret != dest {
862 self.move_location(sz, dest, ret)?;
863 }
864 for r in temps {
865 self.release_gpr(r);
866 }
867 Ok(())
868 }
869
870 fn emit_cmpop_i32_dynamic_b(
872 &mut self,
873 c: Condition,
874 loc_a: Location,
875 loc_b: Location,
876 ret: Location,
877 signed: bool,
878 ) -> Result<(), CompileError> {
879 self.emit_relaxed_cmp(c, loc_a, loc_b, ret, Size::S32, signed)
880 }
881
882 fn emit_cmpop_i64_dynamic_b(
884 &mut self,
885 c: Condition,
886 loc_a: Location,
887 loc_b: Location,
888 ret: Location,
889 ) -> Result<(), CompileError> {
890 self.emit_relaxed_cmp(c, loc_a, loc_b, ret, Size::S64, false)
891 }
892
893 #[allow(clippy::too_many_arguments)]
897 fn memory_op<F: FnOnce(&mut Self, GPR) -> Result<(), CompileError>>(
898 &mut self,
899 addr: Location,
900 memarg: &MemArg,
901 check_alignment: bool,
902 value_size: usize,
903 imported_memories: bool,
904 offset: i32,
905 heap_access_oob: Label,
906 unaligned_atomic: Label,
907 cb: F,
908 ) -> Result<(), CompileError> {
909 let value_size = value_size as i64;
910 let tmp_addr = self.acquire_temp_gpr().ok_or_else(|| {
911 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
912 })?;
913
914 let (base_loc, bound_loc) = if imported_memories {
916 self.emit_relaxed_binop(
918 Assembler::emit_mov,
919 Size::S64,
920 Location::Memory(self.get_vmctx_reg(), offset),
921 Location::GPR(tmp_addr),
922 )?;
923 (Location::Memory(tmp_addr, 0), Location::Memory(tmp_addr, 8))
924 } else {
925 (
926 Location::Memory(self.get_vmctx_reg(), offset),
927 Location::Memory(self.get_vmctx_reg(), offset + 8),
928 )
929 };
930
931 let tmp_base = self.acquire_temp_gpr().ok_or_else(|| {
932 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
933 })?;
934 let tmp_bound = self.acquire_temp_gpr().ok_or_else(|| {
935 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
936 })?;
937
938 self.emit_relaxed_load(Size::S64, false, Location::GPR(tmp_base), base_loc)?;
940
941 self.emit_relaxed_load(Size::S64, false, Location::GPR(tmp_bound), bound_loc)?;
943
944 self.assembler.emit_add(
948 Size::S64,
949 Location::GPR(tmp_bound),
950 Location::GPR(tmp_base),
951 Location::GPR(tmp_bound),
952 )?;
953 self.assembler.emit_sub(
954 Size::S64,
955 Location::GPR(tmp_bound),
956 Location::Imm64(value_size as _),
957 Location::GPR(tmp_bound),
958 )?;
959
960 self.move_location(Size::S32, addr, Location::GPR(tmp_addr))?;
964
965 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
966 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
967 })?;
968
969 if memarg.offset != 0 {
971 if ImmType::Bits12.compatible_imm(memarg.offset as _) {
972 self.assembler.emit_add(
973 Size::S64,
974 Location::Imm64(memarg.offset),
975 Location::GPR(tmp_addr),
976 Location::GPR(tmp_addr),
977 )?;
978 } else {
979 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
980 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
981 })?;
982 self.assembler
983 .emit_mov_imm(Location::GPR(tmp), memarg.offset as _)?;
984 self.assembler.emit_add(
985 Size::S64,
986 Location::GPR(tmp_addr),
987 Location::GPR(tmp),
988 Location::GPR(tmp_addr),
989 )?;
990 self.release_gpr(tmp);
991 }
992
993 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
996 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
997 })?;
998 self.assembler.emit_srl(
999 Size::S64,
1000 Location::GPR(tmp_addr),
1001 Location::Imm64(32),
1002 Location::GPR(tmp),
1003 )?;
1004 self.assembler
1005 .emit_on_true_label_far(Location::GPR(tmp), heap_access_oob, jmp_tmp)?;
1006 self.release_gpr(tmp);
1007 }
1008
1009 self.assembler.emit_add(
1011 Size::S64,
1012 Location::GPR(tmp_base),
1013 Location::GPR(tmp_addr),
1014 Location::GPR(tmp_addr),
1015 )?;
1016
1017 let cond = tmp_base;
1019
1020 self.assembler.emit_cmp(
1022 Condition::Le,
1023 Location::GPR(tmp_addr),
1024 Location::GPR(tmp_bound),
1025 Location::GPR(cond),
1026 )?;
1027
1028 self.assembler
1030 .emit_on_false_label_far(Location::GPR(cond), heap_access_oob, jmp_tmp)?;
1031
1032 self.release_gpr(tmp_bound);
1033 self.release_gpr(cond);
1034
1035 let align = value_size as u32;
1036 if check_alignment && align != 1 {
1037 self.assembler.emit_and(
1038 Size::S64,
1039 Location::GPR(tmp_addr),
1040 Location::Imm64((align - 1) as u64),
1041 Location::GPR(cond),
1042 )?;
1043 self.assembler.emit_on_true_label_far(
1044 Location::GPR(cond),
1045 unaligned_atomic,
1046 jmp_tmp,
1047 )?;
1048 }
1049 let begin = self.assembler.get_offset().0;
1050 cb(self, tmp_addr)?;
1051 let end = self.assembler.get_offset().0;
1052 self.mark_address_range_with_trap_code(TrapCode::HeapAccessOutOfBounds, begin, end);
1053
1054 self.release_gpr(jmp_tmp);
1055 self.release_gpr(tmp_addr);
1056 Ok(())
1057 }
1058
1059 fn emit_relaxed_load(
1060 &mut self,
1061 sz: Size,
1062 signed: bool,
1063 dst: Location,
1064 src: Location,
1065 ) -> Result<(), CompileError> {
1066 let mut temps = vec![];
1067 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
1068 match src {
1069 Location::Memory(addr, offset) => {
1070 if ImmType::Bits12.compatible_imm(offset as i64) {
1071 self.assembler.emit_ld(sz, signed, dest, src)?;
1072 } else {
1073 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
1074 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1075 })?;
1076 self.assembler
1077 .emit_mov_imm(Location::GPR(tmp), offset as i64)?;
1078 self.assembler.emit_add(
1079 Size::S64,
1080 Location::GPR(addr),
1081 Location::GPR(tmp),
1082 Location::GPR(tmp),
1083 )?;
1084 self.assembler
1085 .emit_ld(sz, signed, dest, Location::Memory(tmp, 0))?;
1086 temps.push(tmp);
1087 }
1088 }
1089 _ => codegen_error!("singlepass emit_relaxed_load unreachable"),
1090 }
1091 if dst != dest {
1092 self.move_location(Size::S64, dest, dst)?;
1094 }
1095 for r in temps {
1096 self.release_gpr(r);
1097 }
1098 Ok(())
1099 }
1100
1101 fn emit_maybe_unaligned_load(
1102 &mut self,
1103 sz: Size,
1104 signed: bool,
1105 dst: Location,
1106 src: GPR,
1107 ) -> Result<(), CompileError> {
1108 if let Size::S8 = sz {
1109 return self.emit_relaxed_load(sz, signed, dst, Location::Memory(src, 0));
1110 }
1111
1112 let label_unaligned = self.get_label();
1113 let label_completed = self.get_label();
1114 let mut temps = vec![];
1115 let tmp_cond = self.acquire_temp_gpr().ok_or_else(|| {
1116 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1117 })?;
1118 temps.push(tmp_cond);
1119 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
1120 self.assembler.emit_and(
1121 Size::S64,
1122 Location::GPR(src),
1123 Location::Imm64((sz.bytes() - 1) as u64),
1124 Location::GPR(tmp_cond),
1125 )?;
1126 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
1127 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1128 })?;
1129 temps.push(jmp_tmp);
1130 self.assembler
1131 .emit_on_true_label(Location::GPR(tmp_cond), label_unaligned, jmp_tmp)?;
1132 self.assembler
1134 .emit_ld(sz, signed, dest, Location::Memory(src, 0))?;
1135 self.jmp_unconditional(label_completed)?;
1136 self.emit_label(label_unaligned)?;
1137
1138 let tmp_value = tmp_cond;
1140
1141 self.assembler
1143 .emit_ld(Size::S8, false, dest, Location::Memory(src, 0))?;
1144 for i in 1..sz.bytes() {
1145 self.assembler.emit_ld(
1146 Size::S8,
1147 if i == sz.bytes() - 1 { signed } else { false },
1148 Location::GPR(tmp_value),
1149 Location::Memory(src, i as _),
1150 )?;
1151 self.assembler.emit_sll(
1152 Size::S64,
1153 Location::GPR(tmp_value),
1154 Location::Imm64(8 * i as u64),
1155 Location::GPR(tmp_value),
1156 )?;
1157 self.assembler
1158 .emit_or(Size::S64, dest, Location::GPR(tmp_value), dest)?;
1159 }
1160
1161 self.emit_label(label_completed)?;
1163 if dst != dest {
1164 self.move_location(Size::S64, dest, dst)?;
1166 }
1167
1168 for tmp in temps {
1169 self.release_gpr(tmp);
1170 }
1171 Ok(())
1172 }
1173
1174 fn emit_relaxed_store(
1175 &mut self,
1176 sz: Size,
1177 dst: Location,
1178 src: Location,
1179 ) -> Result<(), CompileError> {
1180 let mut temps = vec![];
1181 let dest = self.location_to_reg(Size::S64, dst, &mut temps, ImmType::None, true, None)?;
1182 match src {
1183 Location::Memory(addr, offset) => {
1184 if ImmType::Bits12.compatible_imm(offset as i64) {
1185 self.assembler.emit_sd(sz, dest, src)?;
1186 } else {
1187 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
1188 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1189 })?;
1190 self.assembler
1191 .emit_mov_imm(Location::GPR(tmp), offset as i64)?;
1192 self.assembler.emit_add(
1193 Size::S64,
1194 Location::GPR(addr),
1195 Location::GPR(tmp),
1196 Location::GPR(tmp),
1197 )?;
1198 self.assembler.emit_sd(sz, dest, Location::Memory(tmp, 0))?;
1199 temps.push(tmp);
1200 }
1201 }
1202 _ => codegen_error!("singlepass emit_relaxed_store unreachable"),
1203 }
1204 for r in temps {
1205 self.release_gpr(r);
1206 }
1207 Ok(())
1208 }
1209
1210 fn emit_maybe_unaligned_store(
1211 &mut self,
1212 sz: Size,
1213 src: Location,
1214 dst: GPR,
1215 ) -> Result<(), CompileError> {
1216 if let Size::S8 = sz {
1217 return self.emit_relaxed_store(sz, src, Location::Memory(dst, 0));
1222 }
1223
1224 let label_unaligned = self.get_label();
1225 let label_completed = self.get_label();
1226 let mut temps = vec![];
1227 let tmp_cond = self.acquire_temp_gpr().ok_or_else(|| {
1228 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1229 })?;
1230 temps.push(tmp_cond);
1231 let src_value =
1232 self.location_to_reg(Size::S64, src, &mut temps, ImmType::None, true, None)?;
1233 self.assembler.emit_and(
1234 Size::S64,
1235 Location::GPR(dst),
1236 Location::Imm64((sz.bytes() - 1) as u64),
1237 Location::GPR(tmp_cond),
1238 )?;
1239 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
1240 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1241 })?;
1242 temps.push(jmp_tmp);
1243 self.assembler
1244 .emit_on_true_label(Location::GPR(tmp_cond), label_unaligned, jmp_tmp)?;
1245 self.assembler
1247 .emit_sd(sz, src_value, Location::Memory(dst, 0))?;
1248 self.jmp_unconditional(label_completed)?;
1249 self.emit_label(label_unaligned)?;
1250
1251 let size_bytes = sz.bytes();
1253 for i in 0..size_bytes {
1255 self.assembler
1256 .emit_sd(Size::S8, src_value, Location::Memory(dst, i as _))?;
1257 if i != size_bytes - 1 {
1258 self.assembler
1259 .emit_srl(Size::S64, src_value, Location::Imm64(8), src_value)?;
1260 }
1261 }
1262
1263 self.emit_label(label_completed)?;
1265
1266 for tmp in temps {
1267 self.release_gpr(tmp);
1268 }
1269 Ok(())
1270 }
1271
1272 fn emit_rol(
1273 &mut self,
1274 sz: Size,
1275 loc_a: Location,
1276 loc_b: Location,
1277 ret: Location,
1278 allow_imm: ImmType,
1279 ) -> Result<(), CompileError> {
1280 let mut temps = vec![];
1281 let size_bits = sz.bits();
1282
1283 let src2 = if let Some(imm) = loc_b.imm_value_scalar() {
1284 Location::Imm32(size_bits - (imm as u32) % size_bits)
1285 } else {
1286 let tmp1 = self.location_to_reg(
1287 sz,
1288 Location::Imm32(size_bits),
1289 &mut temps,
1290 ImmType::None,
1291 true,
1292 None,
1293 )?;
1294 let tmp2 = self.location_to_reg(sz, loc_b, &mut temps, ImmType::None, true, None)?;
1295 self.assembler.emit_sub(sz, tmp1, tmp2, tmp1)?;
1296 tmp1
1297 };
1298
1299 self.emit_ror(sz, loc_a, src2, ret, allow_imm)?;
1300
1301 for r in temps {
1302 self.release_gpr(r);
1303 }
1304 Ok(())
1305 }
1306
1307 fn emit_ror(
1308 &mut self,
1309 sz: Size,
1310 loc_a: Location,
1311 loc_b: Location,
1312 ret: Location,
1313 allow_imm: ImmType,
1314 ) -> Result<(), CompileError> {
1315 let mut temps = vec![];
1316
1317 let imm = match sz {
1318 Size::S32 | Size::S64 => Location::Imm32(sz.bits() as u32),
1319 _ => codegen_error!("singlepass emit_ror unreachable"),
1320 };
1321 let imm = self.location_to_reg(sz, imm, &mut temps, ImmType::None, false, None)?;
1322
1323 let tmp1 = self.acquire_temp_gpr().ok_or_else(|| {
1324 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1325 })?;
1326 self.emit_relaxed_binop3(
1327 Assembler::emit_srl,
1328 sz,
1329 loc_a,
1330 loc_b,
1331 Location::GPR(tmp1),
1332 allow_imm,
1333 )?;
1334
1335 let tmp2 = self.acquire_temp_gpr().ok_or_else(|| {
1336 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1337 })?;
1338 self.emit_relaxed_binop3(
1339 Assembler::emit_sub,
1340 sz,
1341 imm,
1342 loc_b,
1343 Location::GPR(tmp2),
1344 allow_imm,
1345 )?;
1346 self.emit_relaxed_binop3(
1347 Assembler::emit_sll,
1348 sz,
1349 loc_a,
1350 Location::GPR(tmp2),
1351 Location::GPR(tmp2),
1352 ImmType::Bits12,
1353 )?;
1354 self.assembler.emit_or(
1355 sz,
1356 Location::GPR(tmp1),
1357 Location::GPR(tmp2),
1358 Location::GPR(tmp1),
1359 )?;
1360
1361 self.move_location(sz, Location::GPR(tmp1), ret)?;
1362 self.release_gpr(tmp1);
1363 self.release_gpr(tmp2);
1364 for r in temps {
1365 self.release_gpr(r);
1366 }
1367 Ok(())
1368 }
1369
1370 fn emit_popcnt(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1371 let arg = self.acquire_temp_gpr().ok_or_else(|| {
1372 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1373 })?;
1374 let cnt = self.acquire_temp_gpr().ok_or_else(|| {
1375 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1376 })?;
1377 let temp = self.acquire_temp_gpr().ok_or_else(|| {
1378 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1379 })?;
1380 self.move_location(sz, src, Location::GPR(arg))?;
1381
1382 self.move_location(sz, Location::Imm32(0), Location::GPR(cnt))?;
1383 let one_imm = match sz {
1384 Size::S32 => Location::Imm32(1),
1385 Size::S64 => Location::Imm64(1),
1386 _ => codegen_error!("singlepass emit_popcnt unreachable"),
1387 };
1388
1389 let label_loop = self.assembler.get_label();
1390 let label_exit = self.assembler.get_label();
1391
1392 self.assembler.emit_label(label_loop)?; self.assembler
1394 .emit_and(sz, Location::GPR(arg), one_imm, Location::GPR(temp))?;
1395 self.assembler.emit_add(
1396 sz,
1397 Location::GPR(cnt),
1398 Location::GPR(temp),
1399 Location::GPR(cnt),
1400 )?;
1401 self.assembler
1402 .emit_srl(sz, Location::GPR(arg), one_imm, Location::GPR(arg))?;
1403 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
1404 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1405 })?;
1406 self.assembler
1407 .emit_on_false_label(Location::GPR(arg), label_exit, jmp_tmp)?;
1408 self.release_gpr(jmp_tmp);
1409 self.jmp_unconditional(label_loop)?;
1410
1411 self.assembler.emit_label(label_exit)?; self.move_location(sz, Location::GPR(cnt), dst)?;
1414
1415 self.release_gpr(arg);
1416 self.release_gpr(cnt);
1417 self.release_gpr(temp);
1418 Ok(())
1419 }
1420
1421 fn emit_ctz(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1422 let size_bits = sz.bits();
1423 let arg = self.acquire_temp_gpr().ok_or_else(|| {
1424 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1425 })?;
1426 let cnt = self.acquire_temp_gpr().ok_or_else(|| {
1427 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1428 })?;
1429 let temp = self.acquire_temp_gpr().ok_or_else(|| {
1430 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1431 })?;
1432 self.move_location(sz, src, Location::GPR(arg))?;
1433
1434 let one_imm = match sz {
1435 Size::S32 => Location::Imm32(1),
1436 Size::S64 => Location::Imm64(1),
1437 _ => codegen_error!("singlepass emit_ctz unreachable"),
1438 };
1439
1440 let label_loop = self.assembler.get_label();
1441 let label_exit = self.assembler.get_label();
1442
1443 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
1444 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1445 })?;
1446
1447 self.move_location(sz, Location::Imm32(size_bits), Location::GPR(cnt))?;
1449 self.assembler
1450 .emit_on_false_label(Location::GPR(arg), label_exit, jmp_tmp)?;
1451
1452 self.move_location(sz, Location::Imm32(0), Location::GPR(cnt))?;
1453
1454 self.assembler.emit_label(label_loop)?; self.assembler
1456 .emit_and(sz, Location::GPR(arg), one_imm, Location::GPR(temp))?;
1457 self.assembler
1458 .emit_on_true_label(Location::GPR(temp), label_exit, jmp_tmp)?;
1459
1460 self.release_gpr(jmp_tmp);
1461
1462 self.assembler
1463 .emit_add(sz, Location::GPR(cnt), one_imm, Location::GPR(cnt))?;
1464 self.assembler
1465 .emit_srl(sz, Location::GPR(arg), one_imm, Location::GPR(arg))?;
1466 self.jmp_unconditional(label_loop)?;
1467
1468 self.assembler.emit_label(label_exit)?; self.move_location(sz, Location::GPR(cnt), dst)?;
1471
1472 self.release_gpr(arg);
1473 self.release_gpr(cnt);
1474 self.release_gpr(temp);
1475 Ok(())
1476 }
1477
1478 fn emit_clz(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1479 let size_bits = sz.bits();
1480 let arg = self.acquire_temp_gpr().ok_or_else(|| {
1481 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1482 })?;
1483 let cnt = self.acquire_temp_gpr().ok_or_else(|| {
1484 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1485 })?;
1486 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
1487 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1488 })?;
1489 self.move_location(sz, src, Location::GPR(arg))?;
1490
1491 let one_imm = match sz {
1492 Size::S32 => Location::Imm32(1),
1493 Size::S64 => Location::Imm64(1),
1494 _ => codegen_error!("singlepass emit_ctz unreachable"),
1495 };
1496
1497 let label_loop = self.assembler.get_label();
1498 let label_exit = self.assembler.get_label();
1499
1500 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
1501 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1502 })?;
1503
1504 self.move_location(sz, Location::Imm32(size_bits), Location::GPR(cnt))?;
1506 self.assembler
1507 .emit_on_false_label(Location::GPR(arg), label_exit, jmp_tmp)?;
1508
1509 self.move_location(sz, Location::Imm32(0), Location::GPR(cnt))?;
1510
1511 self.assembler.emit_label(label_loop)?;
1513 self.move_location(
1515 Size::S32,
1516 Location::Imm32(size_bits - 1),
1517 Location::GPR(tmp),
1518 )?;
1519 self.assembler.emit_sub(
1520 Size::S32,
1521 Location::GPR(tmp),
1522 Location::GPR(cnt),
1523 Location::GPR(tmp),
1524 )?;
1525 self.assembler.emit_srl(
1526 sz,
1527 Location::GPR(arg),
1528 Location::GPR(tmp),
1529 Location::GPR(tmp),
1530 )?;
1531 self.assembler
1532 .emit_on_true_label(Location::GPR(tmp), label_exit, jmp_tmp)?;
1533
1534 self.assembler
1535 .emit_add(sz, Location::GPR(cnt), one_imm, Location::GPR(cnt))?;
1536 self.release_gpr(jmp_tmp);
1537 self.jmp_unconditional(label_loop)?;
1538
1539 self.assembler.emit_label(label_exit)?; self.move_location(sz, Location::GPR(cnt), dst)?;
1542
1543 self.release_gpr(arg);
1544 self.release_gpr(cnt);
1545 self.release_gpr(tmp);
1546 Ok(())
1547 }
1548
1549 fn convert_float_to_int(
1550 &mut self,
1551 loc: Location,
1552 size_in: Size,
1553 ret: Location,
1554 size_out: Size,
1555 signed: bool,
1556 sat: bool,
1557 ) -> Result<(), CompileError> {
1558 let mut gprs = vec![];
1559 let mut fprs = vec![];
1560 let src = self.location_to_fpr(size_in, loc, &mut fprs, ImmType::None, true)?;
1561 let dest = self.location_to_reg(size_out, ret, &mut gprs, ImmType::None, false, None)?;
1562
1563 if sat {
1564 let end = self.assembler.get_label();
1567 let cond = self.acquire_temp_gpr().ok_or_else(|| {
1568 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1569 })?;
1570 self.zero_location(size_out, dest)?;
1571 self.assembler
1573 .emit_fcmp(Condition::Eq, size_in, src, src, Location::GPR(cond))?;
1574 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
1575 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1576 })?;
1577 self.assembler
1578 .emit_on_false_label(Location::GPR(cond), end, jmp_tmp)?;
1579 self.release_gpr(jmp_tmp);
1580 self.release_gpr(cond);
1581
1582 self.assembler
1583 .emit_fcvt(signed, size_in, src, size_out, dest)?;
1584 self.emit_label(end)?;
1585 } else {
1586 let old_fcsr = self.acquire_temp_gpr().ok_or_else(|| {
1587 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1588 })?;
1589 self.move_location(
1590 Size::S32,
1591 Location::GPR(GPR::XZero),
1592 Location::GPR(old_fcsr),
1593 )?;
1594 self.assembler.emit_swap_fscr(old_fcsr)?;
1595 self.assembler
1596 .emit_fcvt(signed, size_in, src, size_out, dest)?;
1597 self.trap_float_convertion_errors(size_in, src, old_fcsr, &mut gprs)?;
1598 self.release_gpr(old_fcsr);
1599 }
1600
1601 if ret != dest {
1602 self.move_location(size_out, dest, ret)?;
1603 }
1604 for r in gprs {
1605 self.release_gpr(r);
1606 }
1607 for r in fprs {
1608 self.release_simd(r);
1609 }
1610 Ok(())
1611 }
1612
1613 fn convert_int_to_float(
1614 &mut self,
1615 loc: Location,
1616 size_in: Size,
1617 ret: Location,
1618 size_out: Size,
1619 signed: bool,
1620 ) -> Result<(), CompileError> {
1621 let mut gprs = vec![];
1622 let mut fprs = vec![];
1623 let src = self.location_to_reg(size_in, loc, &mut gprs, ImmType::None, true, None)?;
1624 let dest = self.location_to_fpr(size_out, ret, &mut fprs, ImmType::None, false)?;
1625 self.assembler
1626 .emit_fcvt(signed, size_in, src, size_out, dest)?;
1627 if ret != dest {
1628 self.move_location(Size::S32, dest, ret)?;
1629 }
1630 for r in gprs {
1631 self.release_gpr(r);
1632 }
1633 for r in fprs {
1634 self.release_simd(r);
1635 }
1636 Ok(())
1637 }
1638
1639 fn convert_float_to_float(
1640 &mut self,
1641 loc: Location,
1642 size_in: Size,
1643 ret: Location,
1644 size_out: Size,
1645 ) -> Result<(), CompileError> {
1646 let mut temps = vec![];
1647 let src = self.location_to_fpr(size_in, loc, &mut temps, ImmType::None, true)?;
1648 let dest = self.location_to_fpr(size_out, ret, &mut temps, ImmType::None, false)?;
1649
1650 match (size_in, size_out) {
1651 (Size::S32, Size::S64) => self
1652 .assembler
1653 .emit_fcvt(false, size_in, src, size_out, dest)?,
1654 (Size::S64, Size::S32) => self
1655 .assembler
1656 .emit_fcvt(false, size_in, src, size_out, dest)?,
1657 _ => codegen_error!("singlepass convert_float_to_float unreachable"),
1658 }
1659
1660 if ret != dest {
1661 self.move_location(size_out, dest, ret)?;
1662 }
1663 for r in temps {
1664 self.release_simd(r);
1665 }
1666 Ok(())
1667 }
1668
1669 fn trap_float_convertion_errors(
1670 &mut self,
1671 sz: Size,
1672 f: Location,
1673 old_fcsr: GPR,
1674 temps: &mut Vec<GPR>,
1675 ) -> Result<(), CompileError> {
1676 let fscr = self.acquire_temp_gpr().ok_or_else(|| {
1677 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1678 })?;
1679 self.zero_location(Size::S64, Location::GPR(fscr))?;
1680 temps.push(fscr);
1681
1682 let trap_badconv = self.assembler.get_label();
1683 let end = self.assembler.get_label();
1684
1685 self.assembler.emit_swap_fscr(fscr)?;
1686
1687 self.assembler.emit_srl(
1690 Size::S32,
1691 Location::GPR(fscr),
1692 Location::Imm32(4),
1693 Location::GPR(fscr),
1694 )?;
1695
1696 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
1697 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1698 })?;
1699 temps.push(jmp_tmp);
1700
1701 self.assembler
1702 .emit_on_false_label(Location::GPR(fscr), end, jmp_tmp)?;
1703
1704 let cond = self.acquire_temp_gpr().ok_or_else(|| {
1706 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1707 })?;
1708 temps.push(cond);
1709
1710 self.assembler
1711 .emit_fcmp(Condition::Eq, sz, f, f, Location::GPR(cond))?;
1712 self.assembler
1714 .emit_on_false_label(Location::GPR(cond), trap_badconv, jmp_tmp)?;
1715 self.emit_illegal_op_internal(TrapCode::IntegerOverflow)?;
1716 self.emit_label(trap_badconv)?;
1717 self.emit_illegal_op_internal(TrapCode::BadConversionToInteger)?;
1718
1719 self.emit_label(end)?;
1720 self.assembler.emit_swap_fscr(old_fcsr)?;
1721
1722 Ok(())
1723 }
1724
1725 fn emit_illegal_op_internal(&mut self, trap: TrapCode) -> Result<(), CompileError> {
1726 self.assembler.emit_udf(trap as u8)
1727 }
1728
1729 fn emit_relaxed_fcmp(
1730 &mut self,
1731 c: Condition,
1732 size: Size,
1733 loc_a: Location,
1734 loc_b: Location,
1735 ret: Location,
1736 ) -> Result<(), CompileError> {
1737 let mut fprs = vec![];
1739 let mut gprs = vec![];
1740
1741 let loc_a = self.location_to_fpr(size, loc_a, &mut fprs, ImmType::None, true)?;
1742 let loc_b = self.location_to_fpr(size, loc_b, &mut fprs, ImmType::None, true)?;
1743 let dest = self.location_to_reg(size, ret, &mut gprs, ImmType::None, false, None)?;
1744
1745 self.assembler.emit_fcmp(c, size, loc_a, loc_b, dest)?;
1746 if ret != dest {
1747 self.move_location(size, dest, ret)?;
1748 }
1749 for r in fprs {
1750 self.release_simd(r);
1751 }
1752 for r in gprs {
1753 self.release_gpr(r);
1754 }
1755 Ok(())
1756 }
1757
1758 fn emit_relaxed_fcvt_with_rounding(
1759 &mut self,
1760 rounding: RoundingMode,
1761 size: Size,
1762 loc: Location,
1763 ret: Location,
1764 ) -> Result<(), CompileError> {
1765 let mut fprs = vec![];
1770 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
1771 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1772 })?;
1773
1774 let loc = self.location_to_fpr(size, loc, &mut fprs, ImmType::None, true)?;
1775 let dest = self.location_to_fpr(size, ret, &mut fprs, ImmType::None, false)?;
1776
1777 let cond = self.acquire_temp_gpr().ok_or_else(|| {
1778 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1779 })?;
1780 let tmp1 = self.acquire_temp_simd().ok_or_else(|| {
1781 CompileError::Codegen("singlepass cannot acquire temp fpr".to_owned())
1782 })?;
1783 let tmp2 = self.acquire_temp_simd().ok_or_else(|| {
1784 CompileError::Codegen("singlepass cannot acquire temp fpr".to_owned())
1785 })?;
1786
1787 let label_after = self.get_label();
1788
1789 let canonical_nan = match size {
1791 Size::S32 => CANONICAL_NAN_F32 as u64,
1792 Size::S64 => CANONICAL_NAN_F64,
1793 _ => unreachable!(),
1794 };
1795
1796 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
1797 CompileError::Codegen("singlepass cannot acquire temp fpr".to_owned())
1798 })?;
1799
1800 self.assembler
1801 .emit_mov_imm(Location::GPR(tmp), canonical_nan as _)?;
1802 self.assembler.emit_mov(size, Location::GPR(tmp), dest)?;
1803 self.assembler
1804 .emit_fcmp(Condition::Eq, size, loc, loc, Location::GPR(cond))?;
1805 self.assembler
1806 .emit_on_false_label(Location::GPR(cond), label_after, jmp_tmp)?;
1807
1808 if size == Size::S64 {
1810 self.assembler
1811 .emit_mov_imm(Location::GPR(cond), 0x4330000000000000)?;
1812 self.assembler
1813 .emit_mov(Size::S64, Location::GPR(cond), Location::SIMD(tmp1))?;
1814 self.f64_abs(loc, Location::SIMD(tmp2))?;
1815 } else {
1816 assert!(size == Size::S32);
1817 self.assembler
1818 .emit_mov_imm(Location::GPR(cond), 0x4b000000)?;
1819 self.assembler
1820 .emit_mov(Size::S32, Location::GPR(cond), Location::SIMD(tmp1))?;
1821 self.f32_abs(loc, Location::SIMD(tmp2))?;
1822 }
1823 self.emit_relaxed_fcmp(
1824 Condition::Lt,
1825 size,
1826 Location::SIMD(tmp2),
1827 Location::SIMD(tmp1),
1828 Location::GPR(cond),
1829 )?;
1830
1831 self.assembler.emit_mov(size, loc, dest)?;
1832 self.assembler
1833 .emit_on_false_label(Location::GPR(cond), label_after, jmp_tmp)?;
1834
1835 self.assembler
1837 .emit_fcvt_with_rounding(rounding, size, loc, dest, cond)?;
1838 self.emit_label(label_after)?;
1839
1840 if ret != dest {
1841 self.move_location(size, dest, ret)?;
1842 }
1843
1844 for r in fprs {
1845 self.release_simd(r);
1846 }
1847 self.release_gpr(jmp_tmp);
1848 self.release_gpr(tmp);
1849 self.release_gpr(cond);
1850 self.release_simd(tmp1);
1851 self.release_simd(tmp2);
1852 Ok(())
1853 }
1854
1855 fn emit_unwind_op(&mut self, op: UnwindOps<GPR, FPR>) {
1856 self.unwind_ops.push((self.get_offset().0, op));
1857 }
1858}
1859
1860pub(crate) const RISCV_RETURN_VALUE_REGISTERS: [GPR; 7] = [
1865 GPR::X10,
1866 GPR::X11,
1867 GPR::X12,
1868 GPR::X13,
1869 GPR::X14,
1870 GPR::X15,
1871 GPR::X16,
1872];
1873
1874#[allow(dead_code)]
1875#[derive(PartialEq, Copy, Clone)]
1876pub(crate) enum ImmType {
1877 None,
1878 Bits12,
1879 Bits12Subtraction,
1882 Shift32,
1883 Shift64,
1884}
1885
1886impl ImmType {
1887 pub(crate) fn compatible_imm(&self, imm: i64) -> bool {
1888 match self {
1889 ImmType::None => false,
1890 ImmType::Bits12 => (-0x800..0x800).contains(&imm),
1891 ImmType::Bits12Subtraction => (-0x801..0x801).contains(&imm),
1892 ImmType::Shift32 => (0..32).contains(&imm),
1893 ImmType::Shift64 => (0..64).contains(&imm),
1894 }
1895 }
1896}
1897
1898impl Machine for MachineRiscv {
1899 type GPR = GPR;
1900 type SIMD = FPR;
1901
1902 fn assembler_get_offset(&self) -> Offset {
1903 self.assembler.get_offset()
1904 }
1905
1906 fn get_vmctx_reg(&self) -> Self::GPR {
1907 GPR::X27
1909 }
1910
1911 fn pick_gpr(&self) -> Option<Self::GPR> {
1912 use GPR::*;
1913 static REGS: &[GPR] = &[X5, X6, X7, X29, X30, X31];
1915 for r in REGS {
1916 if !self.used_gprs_contains(r) {
1917 return Some(*r);
1918 }
1919 }
1920 None
1921 }
1922
1923 fn pick_temp_gpr(&self) -> Option<GPR> {
1924 use GPR::*;
1925 static REGS: &[GPR] = &[X17, X16, X15, X14, X13, X12, X11, X10];
1929 for r in REGS {
1930 if !self.used_gprs_contains(r) {
1931 return Some(*r);
1932 }
1933 }
1934 None
1935 }
1936
1937 fn get_used_gprs(&self) -> Vec<Self::GPR> {
1938 GPR::iterator()
1939 .filter(|x| self.used_gprs.contains(x.into_index()))
1940 .cloned()
1941 .collect()
1942 }
1943
1944 fn get_used_simd(&self) -> Vec<Self::SIMD> {
1945 FPR::iterator()
1946 .filter(|x| self.used_fprs.contains(x.into_index()))
1947 .cloned()
1948 .collect()
1949 }
1950
1951 fn acquire_temp_gpr(&mut self) -> Option<Self::GPR> {
1952 let gpr = self.pick_temp_gpr();
1953 if let Some(x) = gpr {
1954 self.used_gprs_insert(x);
1955 }
1956 gpr
1957 }
1958
1959 fn release_gpr(&mut self, gpr: Self::GPR) {
1960 assert!(self.used_gprs_remove(&gpr));
1961 }
1962
1963 fn reserve_unused_temp_gpr(&mut self, gpr: Self::GPR) -> Self::GPR {
1964 assert!(!self.used_gprs_contains(&gpr));
1965 self.used_gprs_insert(gpr);
1966 gpr
1967 }
1968
1969 fn reserve_gpr(&mut self, gpr: Self::GPR) {
1970 self.used_gprs_insert(gpr);
1971 }
1972
1973 fn push_used_gpr(&mut self, used_gprs: &[Self::GPR]) -> Result<usize, CompileError> {
1974 for r in used_gprs.iter() {
1975 self.assembler.emit_push(Size::S64, Location::GPR(*r))?;
1976 }
1977 Ok(used_gprs.len() * 16)
1978 }
1979
1980 fn pop_used_gpr(&mut self, used_gprs: &[Self::GPR]) -> Result<(), CompileError> {
1981 for r in used_gprs.iter().rev() {
1982 self.emit_pop(Size::S64, Location::GPR(*r))?;
1983 }
1984 Ok(())
1985 }
1986
1987 fn pick_simd(&self) -> Option<Self::SIMD> {
1988 use FPR::*;
1989 static REGS: &[FPR] = &[F0, F1, F2, F3, F4, F5, F6, F7];
1990 for r in REGS {
1991 if !self.used_fp_contains(r) {
1992 return Some(*r);
1993 }
1994 }
1995 None
1996 }
1997
1998 fn pick_temp_simd(&self) -> Option<FPR> {
1999 use FPR::*;
2000 static REGS: &[FPR] = &[F28, F29, F31];
2001 for r in REGS {
2002 if !self.used_fp_contains(r) {
2003 return Some(*r);
2004 }
2005 }
2006 None
2007 }
2008
2009 fn acquire_temp_simd(&mut self) -> Option<Self::SIMD> {
2010 let fpr = self.pick_temp_simd();
2011 if let Some(x) = fpr {
2012 self.used_fprs_insert(x);
2013 }
2014 fpr
2015 }
2016
2017 fn reserve_simd(&mut self, fpr: Self::SIMD) {
2018 self.used_fprs_insert(fpr);
2019 }
2020
2021 fn release_simd(&mut self, fpr: Self::SIMD) {
2022 assert!(self.used_fprs_remove(&fpr));
2023 }
2024
2025 fn push_used_simd(&mut self, used_neons: &[Self::SIMD]) -> Result<usize, CompileError> {
2026 let stack_adjust = (used_neons.len() * 8) as u32;
2027 self.extend_stack(stack_adjust)?;
2028
2029 for (i, r) in used_neons.iter().enumerate() {
2030 self.assembler.emit_sd(
2031 Size::S64,
2032 Location::SIMD(*r),
2033 Location::Memory(GPR::Sp, (i * 8) as i32),
2034 )?;
2035 }
2036 Ok(stack_adjust as usize)
2037 }
2038
2039 fn pop_used_simd(&mut self, used_neons: &[Self::SIMD]) -> Result<(), CompileError> {
2040 for (i, r) in used_neons.iter().enumerate() {
2041 self.assembler.emit_ld(
2042 Size::S64,
2043 false,
2044 Location::SIMD(*r),
2045 Location::Memory(GPR::Sp, (i * 8) as i32),
2046 )?;
2047 }
2048 let stack_adjust = (used_neons.len() * 8) as u32;
2049 self.assembler.emit_add(
2050 Size::S64,
2051 Location::GPR(GPR::Sp),
2052 Location::Imm64(stack_adjust as _),
2053 Location::GPR(GPR::Sp),
2054 )
2055 }
2056
2057 fn round_stack_adjust(&self, value: usize) -> usize {
2058 if value & 0xf != 0 {
2059 ((value >> 4) + 1) << 4
2060 } else {
2061 value
2062 }
2063 }
2064
2065 fn set_srcloc(&mut self, offset: u32) {
2066 self.src_loc = offset;
2067 }
2068
2069 fn mark_address_range_with_trap_code(&mut self, code: TrapCode, begin: usize, end: usize) {
2070 for i in begin..end {
2071 self.trap_table.offset_to_code.insert(i, code);
2072 }
2073 self.mark_instruction_address_end(begin);
2074 }
2075
2076 fn mark_address_with_trap_code(&mut self, code: TrapCode) {
2077 let offset = self.assembler.get_offset().0;
2078 self.trap_table.offset_to_code.insert(offset, code);
2079 self.mark_instruction_address_end(offset);
2080 }
2081
2082 fn mark_instruction_with_trap_code(&mut self, code: TrapCode) -> usize {
2083 let offset = self.assembler.get_offset().0;
2084 self.trap_table.offset_to_code.insert(offset, code);
2085 offset
2086 }
2087
2088 fn mark_instruction_address_end(&mut self, begin: usize) {
2089 self.instructions_address_map.push(InstructionAddressMap {
2090 srcloc: SourceLoc::new(self.src_loc),
2091 code_offset: begin,
2092 code_len: self.assembler.get_offset().0 - begin,
2093 });
2094 }
2095
2096 fn insert_stackoverflow(&mut self) {
2097 let offset = 0;
2098 self.trap_table
2099 .offset_to_code
2100 .insert(offset, TrapCode::StackOverflow);
2101 self.mark_instruction_address_end(offset);
2102 }
2103
2104 fn collect_trap_information(&self) -> Vec<TrapInformation> {
2105 self.trap_table
2106 .offset_to_code
2107 .clone()
2108 .into_iter()
2109 .map(|(offset, code)| TrapInformation {
2110 code_offset: offset as u32,
2111 trap_code: code,
2112 })
2113 .collect()
2114 }
2115
2116 fn instructions_address_map(&self) -> Vec<InstructionAddressMap> {
2117 self.instructions_address_map.clone()
2118 }
2119
2120 fn local_on_stack(&mut self, stack_offset: i32) -> Location {
2121 Location::Memory(GPR::Fp, -stack_offset)
2122 }
2123
2124 fn extend_stack(&mut self, delta_stack_offset: u32) -> Result<(), CompileError> {
2125 let delta = if ImmType::Bits12Subtraction.compatible_imm(delta_stack_offset as _) {
2126 Location::Imm64(delta_stack_offset as _)
2127 } else {
2128 self.assembler
2129 .emit_mov_imm(Location::GPR(SCRATCH_REG), delta_stack_offset as _)?;
2130 Location::GPR(SCRATCH_REG)
2131 };
2132 self.assembler.emit_sub(
2133 Size::S64,
2134 Location::GPR(GPR::Sp),
2135 delta,
2136 Location::GPR(GPR::Sp),
2137 )
2138 }
2139
2140 fn truncate_stack(&mut self, delta_stack_offset: u32) -> Result<(), CompileError> {
2141 let delta = if ImmType::Bits12.compatible_imm(delta_stack_offset as _) {
2142 Location::Imm64(delta_stack_offset as _)
2143 } else {
2144 self.assembler
2145 .emit_mov_imm(Location::GPR(SCRATCH_REG), delta_stack_offset as _)?;
2146 Location::GPR(SCRATCH_REG)
2147 };
2148 self.assembler.emit_add(
2149 Size::S64,
2150 Location::GPR(GPR::Sp),
2151 delta,
2152 Location::GPR(GPR::Sp),
2153 )
2154 }
2155
2156 fn zero_location(&mut self, size: Size, location: Location) -> Result<(), CompileError> {
2157 self.move_location(size, Location::GPR(GPR::XZero), location)
2158 }
2159
2160 fn local_pointer(&self) -> Self::GPR {
2161 GPR::Fp
2162 }
2163
2164 fn move_location_for_native(
2165 &mut self,
2166 _size: Size,
2167 loc: Location,
2168 dest: Location,
2169 ) -> Result<(), CompileError> {
2170 match loc {
2171 Location::Imm64(_)
2172 | Location::Imm32(_)
2173 | Location::Imm8(_)
2174 | Location::Memory(_, _) => {
2175 self.move_location(Size::S64, loc, Location::GPR(SCRATCH_REG))?;
2176 self.move_location(Size::S64, Location::GPR(SCRATCH_REG), dest)
2177 }
2178 _ => self.move_location(Size::S64, loc, dest),
2179 }
2180 }
2181
2182 fn is_local_on_stack(&self, idx: usize) -> bool {
2183 idx > 9
2184 }
2185
2186 fn get_local_location(&self, idx: usize, callee_saved_regs_size: usize) -> Location {
2187 match idx {
2189 0 => Location::GPR(GPR::X9),
2190 1 => Location::GPR(GPR::X18),
2191 2 => Location::GPR(GPR::X19),
2192 3 => Location::GPR(GPR::X20),
2193 4 => Location::GPR(GPR::X21),
2194 5 => Location::GPR(GPR::X22),
2195 6 => Location::GPR(GPR::X23),
2196 7 => Location::GPR(GPR::X24),
2197 8 => Location::GPR(GPR::X25),
2198 9 => Location::GPR(GPR::X26),
2199 _ => Location::Memory(GPR::Fp, -(((idx - 9) * 8 + callee_saved_regs_size) as i32)),
2200 }
2201 }
2202
2203 fn move_local(&mut self, stack_offset: i32, location: Location) -> Result<(), CompileError> {
2204 self.move_location(
2205 Size::S64,
2206 location,
2207 Location::Memory(GPR::Fp, -stack_offset),
2208 )?;
2209
2210 match location {
2211 Location::GPR(x) => self.emit_unwind_op(UnwindOps::SaveRegister {
2212 reg: UnwindRegister::GPR(x),
2213 bp_neg_offset: stack_offset,
2214 }),
2215 Location::SIMD(x) => self.emit_unwind_op(UnwindOps::SaveRegister {
2216 reg: UnwindRegister::FPR(x),
2217 bp_neg_offset: stack_offset,
2218 }),
2219 _ => (),
2220 }
2221 Ok(())
2222 }
2223
2224 fn list_to_save(&self, _calling_convention: CallingConvention) -> Vec<Location> {
2225 vec![]
2226 }
2227
2228 fn get_param_registers(&self, _calling_convention: CallingConvention) -> &'static [Self::GPR] {
2229 &[
2230 GPR::X10,
2231 GPR::X11,
2232 GPR::X12,
2233 GPR::X13,
2234 GPR::X14,
2235 GPR::X15,
2236 GPR::X16,
2237 GPR::X17,
2238 ]
2239 }
2240
2241 fn get_param_location(
2242 &self,
2243 idx: usize,
2244 _sz: Size,
2245 stack_args: &mut usize,
2246 calling_convention: CallingConvention,
2247 ) -> Location {
2248 let register_params: &[GPR] = self.get_param_registers(calling_convention);
2249 if let Some(reg) = register_params.get(idx) {
2250 Location::GPR(*reg)
2251 } else {
2252 let loc = Location::Memory(GPR::Sp, *stack_args as i32);
2253 *stack_args += 8;
2254 loc
2255 }
2256 }
2257
2258 fn get_call_param_location(
2259 &self,
2260 return_slots: usize,
2261 idx: usize,
2262 _sz: Size,
2263 stack_args: &mut usize,
2264 calling_convention: CallingConvention,
2265 ) -> Location {
2266 let return_values_memory_size =
2267 8 * return_slots.saturating_sub(RISCV_RETURN_VALUE_REGISTERS.len()) as i32;
2268 self.get_param_registers(calling_convention)
2269 .get(idx)
2270 .map_or_else(
2271 || {
2272 let loc = Location::Memory(
2273 GPR::Fp,
2274 16 + return_values_memory_size + *stack_args as i32,
2275 );
2276 *stack_args += 8;
2277 loc
2278 },
2279 |reg| Location::GPR(*reg),
2280 )
2281 }
2282
2283 fn get_simple_param_location(
2284 &self,
2285 idx: usize,
2286 calling_convention: CallingConvention,
2287 ) -> Self::GPR {
2288 self.get_param_registers(calling_convention)[idx]
2289 }
2290
2291 fn adjust_gpr_param_location(
2292 &mut self,
2293 register: Self::GPR,
2294 size: Size,
2295 ) -> Result<(), CompileError> {
2296 match size {
2301 Size::S64 => Ok(()),
2302 Size::S32 => self.assembler.emit_extend(
2303 Size::S32,
2304 true,
2305 Location::GPR(register),
2306 Location::GPR(register),
2307 ),
2308 Size::S8 | Size::S16 => {
2309 codegen_error!("singlepass adjust_gpr_param_location unreachable")
2310 }
2311 }
2312 }
2313
2314 fn get_return_value_location(
2315 &self,
2316 idx: usize,
2317 stack_location: &mut usize,
2318 _calling_convention: CallingConvention,
2319 ) -> AbstractLocation<Self::GPR, Self::SIMD> {
2320 RISCV_RETURN_VALUE_REGISTERS.get(idx).map_or_else(
2321 || {
2322 let loc = Location::Memory(GPR::Sp, *stack_location as i32);
2323 *stack_location += 8;
2324 loc
2325 },
2326 |reg| Location::GPR(*reg),
2327 )
2328 }
2329
2330 fn get_call_return_value_location(
2331 &self,
2332 idx: usize,
2333 _calling_convention: CallingConvention,
2334 ) -> AbstractLocation<Self::GPR, Self::SIMD> {
2335 RISCV_RETURN_VALUE_REGISTERS.get(idx).map_or_else(
2336 || {
2337 Location::Memory(
2338 GPR::Fp,
2339 (16 + (idx - RISCV_RETURN_VALUE_REGISTERS.len()) * 8) as i32,
2340 )
2341 },
2342 |reg| Location::GPR(*reg),
2343 )
2344 }
2345
2346 fn move_location(
2347 &mut self,
2348 size: Size,
2349 source: Location,
2350 dest: Location,
2351 ) -> Result<(), CompileError> {
2352 match (source, dest) {
2353 (Location::GPR(_), Location::GPR(_)) => self.assembler.emit_mov(size, source, dest),
2354 (Location::Imm32(_) | Location::Imm64(_), Location::GPR(_)) => self
2355 .assembler
2356 .emit_mov_imm(dest, source.imm_value_scalar().unwrap()),
2357 (Location::GPR(_), Location::Memory(addr, offset)) => {
2358 let addr = if ImmType::Bits12.compatible_imm(offset as _) {
2359 dest
2360 } else {
2361 self.assembler
2362 .emit_mov_imm(Location::GPR(SCRATCH_REG), offset as _)?;
2363 self.assembler.emit_add(
2364 Size::S64,
2365 Location::GPR(addr),
2366 Location::GPR(SCRATCH_REG),
2367 Location::GPR(SCRATCH_REG),
2368 )?;
2369 Location::Memory(SCRATCH_REG, 0)
2370 };
2371 self.assembler.emit_sd(size, source, addr)
2372 }
2373 (Location::SIMD(_), Location::Memory(..)) => {
2374 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2375 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2376 })?;
2377 self.move_location(size, source, Location::GPR(tmp))?;
2378 self.move_location(size, Location::GPR(tmp), dest)?;
2379 self.release_gpr(tmp);
2380 Ok(())
2381 }
2382 (Location::Memory(addr, offset), Location::GPR(_)) => {
2383 let addr = if ImmType::Bits12.compatible_imm(offset as _) {
2384 source
2385 } else {
2386 self.assembler
2387 .emit_mov_imm(Location::GPR(SCRATCH_REG), offset as _)?;
2388 self.assembler.emit_add(
2389 Size::S64,
2390 Location::GPR(addr),
2391 Location::GPR(SCRATCH_REG),
2392 Location::GPR(SCRATCH_REG),
2393 )?;
2394 Location::Memory(SCRATCH_REG, 0)
2395 };
2396 self.assembler.emit_ld(size, false, dest, addr)
2397 }
2398 (Location::GPR(_), Location::SIMD(_)) => self.assembler.emit_mov(size, source, dest),
2399 (Location::SIMD(_), Location::GPR(_)) => self.assembler.emit_mov(size, source, dest),
2400 (Location::SIMD(_), Location::SIMD(_)) => self.assembler.emit_mov(size, source, dest),
2401 _ => todo!("unsupported move: {size:?} {source:?} {dest:?}"),
2402 }
2403 }
2404
2405 fn move_location_extend(
2406 &mut self,
2407 size_val: Size,
2408 signed: bool,
2409 source: Location,
2410 size_op: Size,
2411 dest: Location,
2412 ) -> Result<(), CompileError> {
2413 if size_op != Size::S64 {
2414 codegen_error!("singlepass move_location_extend unreachable");
2415 }
2416 let mut temps = vec![];
2417 let dst = self.location_to_reg(size_op, dest, &mut temps, ImmType::None, false, None)?;
2418 let src = match (size_val, signed, source) {
2419 (Size::S64, _, _) => source,
2420 (_, _, Location::GPR(_)) => {
2421 self.assembler.emit_extend(size_val, signed, source, dst)?;
2422 dst
2423 }
2424 (_, _, Location::Memory(_, _)) => {
2425 self.assembler.emit_ld(size_val, signed, dst, source)?;
2426 dst
2427 }
2428 _ => codegen_error!(
2429 "singlepass can't emit move_location_extend {:?} {:?} {:?} => {:?} {:?}",
2430 size_val,
2431 signed,
2432 source,
2433 size_op,
2434 dest
2435 ),
2436 };
2437 if src != dst {
2438 self.move_location(size_op, src, dst)?;
2439 }
2440 if dst != dest {
2441 self.move_location(size_op, dst, dest)?;
2442 }
2443 for r in temps {
2444 self.release_gpr(r);
2445 }
2446 Ok(())
2447 }
2448
2449 fn init_stack_loc(
2450 &mut self,
2451 init_stack_loc_cnt: u64,
2452 last_stack_loc: Location,
2453 ) -> Result<(), CompileError> {
2454 let label = self.assembler.get_label();
2455 let mut temps = vec![];
2456 let dest = self.acquire_temp_gpr().ok_or_else(|| {
2457 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2458 })?;
2459 temps.push(dest);
2460 let cnt = self.location_to_reg(
2461 Size::S64,
2462 Location::Imm64(init_stack_loc_cnt),
2463 &mut temps,
2464 ImmType::None,
2465 true,
2466 None,
2467 )?;
2468 match last_stack_loc {
2469 Location::GPR(_) => codegen_error!("singlepass init_stack_loc unreachable"),
2470 Location::SIMD(_) => codegen_error!("singlepass init_stack_loc unreachable"),
2471 Location::Memory(reg, offset) => {
2472 if ImmType::Bits12.compatible_imm(offset as _) {
2473 self.assembler.emit_add(
2474 Size::S64,
2475 Location::GPR(reg),
2476 Location::Imm64(offset as _),
2477 Location::GPR(dest),
2478 )?;
2479 } else {
2480 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2481 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2482 })?;
2483 self.assembler
2484 .emit_mov_imm(Location::GPR(tmp), offset as _)?;
2485 self.assembler.emit_add(
2486 Size::S64,
2487 Location::GPR(reg),
2488 Location::GPR(tmp),
2489 Location::GPR(dest),
2490 )?;
2491 temps.push(tmp);
2492 }
2493 }
2494 _ => codegen_error!("singlepass can't emit init_stack_loc {:?}", last_stack_loc),
2495 };
2496 self.assembler.emit_label(label)?;
2497 self.assembler.emit_sd(
2498 Size::S64,
2499 Location::GPR(GPR::XZero),
2500 Location::Memory(dest, 0),
2501 )?;
2502 self.assembler
2503 .emit_sub(Size::S64, cnt, Location::Imm64(1), cnt)?;
2504 self.assembler.emit_add(
2505 Size::S64,
2506 Location::GPR(dest),
2507 Location::Imm64(8),
2508 Location::GPR(dest),
2509 )?;
2510 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
2511 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2512 })?;
2513 temps.push(jmp_tmp);
2514 self.assembler.emit_on_true_label(cnt, label, jmp_tmp)?;
2515 for r in temps {
2516 self.release_gpr(r);
2517 }
2518 Ok(())
2519 }
2520
2521 fn restore_saved_area(&mut self, saved_area_offset: i32) -> Result<(), CompileError> {
2522 self.assembler.emit_sub(
2523 Size::S64,
2524 Location::GPR(GPR::Fp),
2525 Location::Imm64(saved_area_offset as _),
2526 Location::GPR(GPR::Sp),
2527 )
2528 }
2529
2530 fn pop_location(&mut self, location: Location) -> Result<(), CompileError> {
2531 self.emit_pop(Size::S64, location)
2532 }
2533
2534 fn assembler_finalize(
2535 self,
2536 assembly_comments: HashMap<usize, AssemblyComment>,
2537 ) -> Result<FinalizedAssembly, CompileError> {
2538 Ok(FinalizedAssembly {
2539 body: self.assembler.finalize().map_err(|e| {
2540 CompileError::Codegen(format!("Assembler failed finalization with: {e:?}"))
2541 })?,
2542 assembly_comments,
2543 })
2544 }
2545
2546 fn get_offset(&self) -> Offset {
2547 self.assembler.get_offset()
2548 }
2549
2550 fn finalize_function(&mut self) -> Result<(), CompileError> {
2551 self.assembler.finalize_function()?;
2552 Ok(())
2553 }
2554
2555 fn emit_function_prolog(&mut self) -> Result<(), CompileError> {
2556 self.assembler.emit_sub(
2557 Size::S64,
2558 Location::GPR(GPR::Sp),
2559 Location::Imm64(16),
2560 Location::GPR(GPR::Sp),
2561 )?;
2562 self.emit_unwind_op(UnwindOps::SubtractFP { up_to_sp: 16 });
2563
2564 self.assembler.emit_sd(
2565 Size::S64,
2566 Location::GPR(GPR::X1), Location::Memory(GPR::Sp, 8),
2568 )?;
2569 self.assembler.emit_sd(
2570 Size::S64,
2571 Location::GPR(GPR::Fp),
2572 Location::Memory(GPR::Sp, 0),
2573 )?;
2574 self.emit_unwind_op(UnwindOps::SaveRegister {
2575 reg: UnwindRegister::GPR(GPR::X1),
2576 bp_neg_offset: 8,
2577 });
2578 self.emit_unwind_op(UnwindOps::SaveRegister {
2579 reg: UnwindRegister::GPR(GPR::Fp),
2580 bp_neg_offset: 16,
2581 });
2582
2583 self.assembler
2584 .emit_mov(Size::S64, Location::GPR(GPR::Sp), Location::GPR(GPR::Fp))?;
2585 self.emit_unwind_op(UnwindOps::DefineNewFrame);
2586 Ok(())
2587 }
2588
2589 fn emit_function_epilog(&mut self) -> Result<(), CompileError> {
2590 self.assembler
2591 .emit_mov(Size::S64, Location::GPR(GPR::Fp), Location::GPR(GPR::Sp))?;
2592 self.assembler.emit_ld(
2593 Size::S64,
2594 false,
2595 Location::GPR(GPR::X1), Location::Memory(GPR::Sp, 8),
2597 )?;
2598 self.assembler.emit_ld(
2599 Size::S64,
2600 false,
2601 Location::GPR(GPR::Fp),
2602 Location::Memory(GPR::Sp, 0),
2603 )?;
2604 self.assembler.emit_add(
2605 Size::S64,
2606 Location::GPR(GPR::Sp),
2607 Location::Imm64(16),
2608 Location::GPR(GPR::Sp),
2609 )?;
2610
2611 Ok(())
2612 }
2613
2614 fn emit_function_return_float(&mut self) -> Result<(), CompileError> {
2615 self.assembler
2616 .emit_mov(Size::S64, Location::GPR(GPR::X10), Location::SIMD(FPR::F10))
2617 }
2618
2619 fn canonicalize_nan(
2620 &mut self,
2621 sz: Size,
2622 input: Location,
2623 output: Location,
2624 ) -> Result<(), CompileError> {
2625 let mut temps = vec![];
2626 match (sz, input, output) {
2628 (Size::S32, Location::SIMD(_), Location::SIMD(_)) => {
2629 self.assembler.emit_fmax(sz, input, input, output)?;
2630 }
2631 (Size::S64, Location::SIMD(_), Location::SIMD(_)) => {
2632 self.assembler.emit_fmax(sz, input, input, output)?;
2633 }
2634 (Size::S32, Location::SIMD(_), _) | (Size::S64, Location::SIMD(_), _) => {
2635 let tmp = self.location_to_fpr(sz, output, &mut temps, ImmType::None, false)?;
2636 self.assembler.emit_fmax(sz, input, input, tmp)?;
2637 self.move_location(sz, tmp, output)?;
2638 }
2639 (Size::S32, Location::Memory(_, _), _) | (Size::S64, Location::Memory(_, _), _) => {
2640 let src = self.location_to_fpr(sz, input, &mut temps, ImmType::None, true)?;
2641 let tmp = self.location_to_fpr(sz, output, &mut temps, ImmType::None, false)?;
2642 self.assembler.emit_fmax(sz, src, src, tmp)?;
2643 if tmp != output {
2644 self.move_location(sz, tmp, output)?;
2645 }
2646 }
2647 _ => codegen_error!(
2648 "singlepass can't emit canonicalize_nan {:?} {:?} {:?}",
2649 sz,
2650 input,
2651 output
2652 ),
2653 }
2654
2655 for r in temps {
2656 self.release_simd(r);
2657 }
2658 Ok(())
2659 }
2660
2661 fn emit_illegal_op(&mut self, trap: TrapCode) -> Result<(), CompileError> {
2662 let offset = self.assembler.get_offset().0;
2663 self.assembler.emit_udf(trap as u8)?;
2664 self.mark_instruction_address_end(offset);
2665 Ok(())
2666 }
2667
2668 fn get_label(&mut self) -> Label {
2669 self.assembler.new_dynamic_label()
2670 }
2671
2672 fn emit_label(&mut self, label: Label) -> Result<(), CompileError> {
2673 self.assembler.emit_label(label)
2674 }
2675
2676 fn get_gpr_for_call(&self) -> Self::GPR {
2677 GPR::X1
2678 }
2679
2680 fn emit_call_register(&mut self, register: Self::GPR) -> Result<(), CompileError> {
2681 self.assembler.emit_call_register(register)
2682 }
2683
2684 fn emit_call_label(&mut self, label: Label) -> Result<(), CompileError> {
2685 self.assembler.emit_call_label(label)
2686 }
2687
2688 fn arch_emit_indirect_call_with_trampoline(
2689 &mut self,
2690 _location: Location,
2691 ) -> Result<(), CompileError> {
2692 codegen_error!("singlepass arch_emit_indirect_call_with_trampoline unimplemented")
2693 }
2694
2695 fn emit_call_location(&mut self, location: Location) -> Result<(), CompileError> {
2696 let mut temps = vec![];
2697 let loc = self.location_to_reg(
2698 Size::S64,
2699 location,
2700 &mut temps,
2701 ImmType::None,
2702 true,
2703 Some(self.get_gpr_for_call()),
2704 )?;
2705 match loc {
2706 Location::GPR(reg) => self.assembler.emit_call_register(reg),
2707 _ => codegen_error!("singlepass can't emit CALL Location"),
2708 }?;
2709 for r in temps {
2710 self.release_gpr(r);
2711 }
2712 Ok(())
2713 }
2714
2715 fn emit_debug_breakpoint(&mut self) -> Result<(), CompileError> {
2716 self.assembler.emit_brk()
2717 }
2718
2719 fn location_add(
2720 &mut self,
2721 size: Size,
2722 source: Location,
2723 dest: Location,
2724 _flags: bool,
2725 ) -> Result<(), CompileError> {
2726 let mut temps = vec![];
2727 let src = self.location_to_reg(size, source, &mut temps, ImmType::Bits12, true, None)?;
2728 let dst = self.location_to_reg(size, dest, &mut temps, ImmType::None, true, None)?;
2729 self.assembler.emit_add(size, dst, src, dst)?;
2730 if dst != dest {
2731 self.move_location(size, dst, dest)?;
2732 }
2733 for r in temps {
2734 self.release_gpr(r);
2735 }
2736 Ok(())
2737 }
2738
2739 fn location_cmp(
2740 &mut self,
2741 _size: Size,
2742 _source: Location,
2743 _dest: Location,
2744 ) -> Result<(), CompileError> {
2745 codegen_error!("singlepass location_cmp not implemented")
2746 }
2747
2748 fn jmp_unconditional(&mut self, label: Label) -> Result<(), CompileError> {
2749 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2750 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2751 })?;
2752 self.assembler.emit_j_label(label, Some(tmp))?;
2753 self.release_gpr(tmp);
2754 Ok(())
2755 }
2756
2757 fn jmp_on_condition(
2758 &mut self,
2759 cond: UnsignedCondition,
2760 size: Size,
2761 loc_a: AbstractLocation<Self::GPR, Self::SIMD>,
2762 loc_b: AbstractLocation<Self::GPR, Self::SIMD>,
2763 label: Label,
2764 ) -> Result<(), CompileError> {
2765 let c = match cond {
2766 UnsignedCondition::Equal => Condition::Eq,
2767 UnsignedCondition::NotEqual => Condition::Ne,
2768 UnsignedCondition::Above => Condition::Gtu,
2769 UnsignedCondition::AboveEqual => Condition::Geu,
2770 UnsignedCondition::Below => Condition::Ltu,
2771 UnsignedCondition::BelowEqual => Condition::Leu,
2772 };
2773
2774 let mut temps = vec![];
2775 let loc_a = self.location_to_reg(size, loc_a, &mut temps, ImmType::None, true, None)?;
2776 let loc_b = self.location_to_reg(size, loc_b, &mut temps, ImmType::None, true, None)?;
2777
2778 if size != Size::S64 {
2779 self.assembler.emit_extend(size, false, loc_a, loc_a)?;
2780 self.assembler.emit_extend(size, false, loc_b, loc_b)?;
2781 }
2782
2783 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2784 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2785 })?;
2786 temps.push(tmp);
2787 self.assembler
2788 .emit_jmp_on_condition(c, loc_a, loc_b, label, tmp)?;
2789
2790 for r in temps {
2791 self.release_gpr(r);
2792 }
2793 Ok(())
2794 }
2795
2796 fn emit_jmp_to_jumptable(&mut self, label: Label, cond: Location) -> Result<(), CompileError> {
2797 let tmp1 = self.acquire_temp_gpr().ok_or_else(|| {
2798 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2799 })?;
2800 let tmp2 = self.acquire_temp_gpr().ok_or_else(|| {
2801 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2802 })?;
2803
2804 self.assembler.emit_load_label(tmp1, label)?;
2805 self.move_location(Size::S32, cond, Location::GPR(tmp2))?;
2806
2807 self.assembler.emit_sll(
2810 Size::S32,
2811 Location::GPR(tmp2),
2812 Location::Imm32(3),
2813 Location::GPR(tmp2),
2814 )?;
2815 self.assembler.emit_add(
2816 Size::S64,
2817 Location::GPR(tmp1),
2818 Location::GPR(tmp2),
2819 Location::GPR(tmp2),
2820 )?;
2821 self.assembler.emit_j_register(tmp2)?;
2822 self.release_gpr(tmp2);
2823 self.release_gpr(tmp1);
2824 Ok(())
2825 }
2826
2827 fn align_for_loop(&mut self) -> Result<(), CompileError> {
2828 Ok(())
2830 }
2831
2832 fn emit_ret(&mut self) -> Result<(), CompileError> {
2833 self.assembler.emit_ret()
2834 }
2835
2836 fn emit_push(&mut self, size: Size, loc: Location) -> Result<(), CompileError> {
2837 self.assembler.emit_push(size, loc)
2838 }
2839
2840 fn emit_pop(&mut self, size: Size, loc: Location) -> Result<(), CompileError> {
2841 self.assembler.emit_pop(size, loc)
2842 }
2843
2844 fn emit_relaxed_mov(
2845 &mut self,
2846 sz: Size,
2847 src: Location,
2848 dst: Location,
2849 ) -> Result<(), CompileError> {
2850 self.emit_relaxed_binop(Assembler::emit_mov, sz, src, dst)
2851 }
2852
2853 fn emit_relaxed_cmp(
2854 &mut self,
2855 _sz: Size,
2856 _src: Location,
2857 _dst: Location,
2858 ) -> Result<(), CompileError> {
2859 todo!();
2860 }
2861
2862 fn emit_memory_fence(&mut self) -> Result<(), CompileError> {
2863 self.assembler.emit_rwfence()
2864 }
2865
2866 fn emit_relaxed_sign_extension(
2867 &mut self,
2868 sz_src: Size,
2869 src: Location,
2870 sz_dst: Size,
2871 dst: Location,
2872 ) -> Result<(), CompileError> {
2873 let mut temps = vec![];
2874
2875 match (src, dst) {
2876 (Location::Memory(reg, offset), Location::GPR(_)) => {
2877 let src = if ImmType::Bits12.compatible_imm(offset as _) {
2878 src
2879 } else {
2880 let tmp =
2881 self.location_to_reg(sz_src, src, &mut temps, ImmType::None, true, None)?;
2882 self.assembler.emit_mov_imm(tmp, offset as _)?;
2883 self.assembler
2884 .emit_add(Size::S64, Location::GPR(reg), tmp, tmp)?;
2885 let Location::GPR(tmp) = tmp else {
2886 unreachable!()
2887 };
2888 Location::Memory(tmp, 0)
2889 };
2890 self.assembler.emit_ld(sz_src, true, dst, src)?;
2891 }
2892 _ => {
2893 let src =
2894 self.location_to_reg(sz_src, src, &mut temps, ImmType::None, true, None)?;
2895 let dest =
2896 self.location_to_reg(sz_dst, dst, &mut temps, ImmType::None, false, None)?;
2897 self.assembler.emit_extend(sz_src, true, src, dest)?;
2898 if dst != dest {
2899 self.move_location(sz_dst, dest, dst)?;
2900 }
2901 }
2902 }
2903
2904 for r in temps {
2905 self.release_gpr(r);
2906 }
2907
2908 Ok(())
2909 }
2910
2911 fn emit_imul_imm32(
2912 &mut self,
2913 size: Size,
2914 imm32: u32,
2915 gpr: Self::GPR,
2916 ) -> Result<(), CompileError> {
2917 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2918 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2919 })?;
2920 self.assembler
2921 .emit_mov_imm(Location::GPR(tmp), imm32 as _)?;
2922 self.assembler.emit_mul(
2923 size,
2924 Location::GPR(gpr),
2925 Location::GPR(tmp),
2926 Location::GPR(gpr),
2927 )?;
2928 self.release_gpr(tmp);
2929 Ok(())
2930 }
2931
2932 fn emit_binop_add32(
2933 &mut self,
2934 loc_a: Location,
2935 loc_b: Location,
2936 ret: Location,
2937 ) -> Result<(), CompileError> {
2938 self.emit_relaxed_binop3(
2939 Assembler::emit_add,
2940 Size::S32,
2941 loc_a,
2942 loc_b,
2943 ret,
2944 ImmType::Bits12,
2945 )
2946 }
2947
2948 fn emit_binop_sub32(
2949 &mut self,
2950 loc_a: Location,
2951 loc_b: Location,
2952 ret: Location,
2953 ) -> Result<(), CompileError> {
2954 self.emit_relaxed_binop3(
2955 Assembler::emit_sub,
2956 Size::S32,
2957 loc_a,
2958 loc_b,
2959 ret,
2960 ImmType::Bits12Subtraction,
2961 )
2962 }
2963
2964 fn emit_binop_mul32(
2965 &mut self,
2966 loc_a: Location,
2967 loc_b: Location,
2968 ret: Location,
2969 ) -> Result<(), CompileError> {
2970 self.emit_relaxed_binop3(
2971 Assembler::emit_mul,
2972 Size::S32,
2973 loc_a,
2974 loc_b,
2975 ret,
2976 ImmType::None,
2977 )
2978 }
2979
2980 fn emit_binop_udiv32(
2981 &mut self,
2982 loc_a: Location,
2983 loc_b: Location,
2984 ret: Location,
2985 integer_division_by_zero: Label,
2986 ) -> Result<usize, CompileError> {
2987 let mut temps = vec![];
2988 let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
2989 let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
2990 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
2991
2992 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
2993 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2994 })?;
2995 temps.push(jmp_tmp);
2996
2997 self.assembler
2998 .emit_on_false_label_far(src2, integer_division_by_zero, jmp_tmp)?;
2999 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
3000 self.assembler.emit_udiv(Size::S32, src1, src2, dest)?;
3001 if ret != dest {
3002 self.move_location(Size::S32, dest, ret)?;
3003 }
3004 for r in temps {
3005 self.release_gpr(r);
3006 }
3007 Ok(offset)
3008 }
3009
3010 fn emit_binop_sdiv32(
3011 &mut self,
3012 loc_a: Location,
3013 loc_b: Location,
3014 ret: Location,
3015 integer_division_by_zero: Label,
3016 integer_overflow: Label,
3017 ) -> Result<usize, CompileError> {
3018 let mut temps = vec![];
3019 let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
3020 let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
3021 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3022
3023 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
3024 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3025 })?;
3026 temps.push(jmp_tmp);
3027
3028 self.assembler
3029 .emit_on_false_label_far(src2, integer_division_by_zero, jmp_tmp)?;
3030 let label_nooverflow = self.assembler.get_label();
3031 let tmp = self.location_to_reg(
3032 Size::S32,
3033 Location::Imm32(i32::MIN as u32),
3034 &mut temps,
3035 ImmType::None,
3036 true,
3037 None,
3038 )?;
3039 self.assembler.emit_cmp(Condition::Ne, tmp, src1, tmp)?;
3040 self.assembler
3041 .emit_on_true_label(tmp, label_nooverflow, jmp_tmp)?;
3042 self.move_location(Size::S32, Location::Imm32(-1i32 as _), tmp)?;
3043 self.assembler.emit_cmp(Condition::Eq, tmp, src2, tmp)?;
3044 self.assembler
3045 .emit_on_true_label_far(tmp, integer_overflow, jmp_tmp)?;
3046 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
3047 self.assembler.emit_label(label_nooverflow)?;
3048 self.assembler.emit_sdiv(Size::S32, src1, src2, dest)?;
3049 if ret != dest {
3050 self.move_location(Size::S32, dest, ret)?;
3051 }
3052 for r in temps {
3053 self.release_gpr(r);
3054 }
3055 Ok(offset)
3056 }
3057
3058 fn emit_binop_urem32(
3059 &mut self,
3060 loc_a: Location,
3061 loc_b: Location,
3062 ret: Location,
3063 integer_division_by_zero: Label,
3064 ) -> Result<usize, CompileError> {
3065 let mut temps = vec![];
3066 let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
3067 let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
3068 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3069
3070 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
3071 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3072 })?;
3073 temps.push(jmp_tmp);
3074
3075 self.assembler
3076 .emit_on_false_label_far(src2, integer_division_by_zero, jmp_tmp)?;
3077 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
3078 self.assembler.emit_urem(Size::S32, src1, src2, dest)?;
3079 if ret != dest {
3080 self.move_location(Size::S32, dest, ret)?;
3081 }
3082 for r in temps {
3083 self.release_gpr(r);
3084 }
3085 Ok(offset)
3086 }
3087
3088 fn emit_binop_srem32(
3089 &mut self,
3090 loc_a: Location,
3091 loc_b: Location,
3092 ret: Location,
3093 integer_division_by_zero: Label,
3094 ) -> Result<usize, CompileError> {
3095 let mut temps = vec![];
3096 let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
3097 let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
3098 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3099
3100 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
3101 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3102 })?;
3103 temps.push(jmp_tmp);
3104
3105 self.assembler
3106 .emit_on_false_label_far(src2, integer_division_by_zero, jmp_tmp)?;
3107 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
3108 self.assembler.emit_srem(Size::S32, src1, src2, dest)?;
3109 if ret != dest {
3110 self.move_location(Size::S32, dest, ret)?;
3111 }
3112 for r in temps {
3113 self.release_gpr(r);
3114 }
3115 Ok(offset)
3116 }
3117
3118 fn emit_binop_and32(
3119 &mut self,
3120 loc_a: Location,
3121 loc_b: Location,
3122 ret: Location,
3123 ) -> Result<(), CompileError> {
3124 self.emit_relaxed_binop3(
3125 Assembler::emit_and,
3126 Size::S32,
3127 loc_a,
3128 loc_b,
3129 ret,
3130 ImmType::Bits12,
3131 )
3132 }
3133
3134 fn emit_binop_or32(
3135 &mut self,
3136 loc_a: Location,
3137 loc_b: Location,
3138 ret: Location,
3139 ) -> Result<(), CompileError> {
3140 self.emit_relaxed_binop3(
3141 Assembler::emit_or,
3142 Size::S32,
3143 loc_a,
3144 loc_b,
3145 ret,
3146 ImmType::Bits12,
3147 )
3148 }
3149
3150 fn emit_binop_xor32(
3151 &mut self,
3152 loc_a: Location,
3153 loc_b: Location,
3154 ret: Location,
3155 ) -> Result<(), CompileError> {
3156 self.emit_relaxed_binop3(
3157 Assembler::emit_xor,
3158 Size::S32,
3159 loc_a,
3160 loc_b,
3161 ret,
3162 ImmType::Bits12,
3163 )
3164 }
3165
3166 fn i32_cmp_ge_s(
3167 &mut self,
3168 loc_a: Location,
3169 loc_b: Location,
3170 ret: Location,
3171 ) -> Result<(), CompileError> {
3172 self.emit_cmpop_i32_dynamic_b(Condition::Ge, loc_a, loc_b, ret, true)
3173 }
3174
3175 fn i32_cmp_gt_s(
3176 &mut self,
3177 loc_a: Location,
3178 loc_b: Location,
3179 ret: Location,
3180 ) -> Result<(), CompileError> {
3181 self.emit_cmpop_i32_dynamic_b(Condition::Gt, loc_a, loc_b, ret, true)
3182 }
3183
3184 fn i32_cmp_le_s(
3185 &mut self,
3186 loc_a: Location,
3187 loc_b: Location,
3188 ret: Location,
3189 ) -> Result<(), CompileError> {
3190 self.emit_cmpop_i32_dynamic_b(Condition::Le, loc_a, loc_b, ret, true)
3191 }
3192
3193 fn i32_cmp_lt_s(
3194 &mut self,
3195 loc_a: Location,
3196 loc_b: Location,
3197 ret: Location,
3198 ) -> Result<(), CompileError> {
3199 self.emit_cmpop_i32_dynamic_b(Condition::Lt, loc_a, loc_b, ret, true)
3200 }
3201
3202 fn i32_cmp_ge_u(
3203 &mut self,
3204 loc_a: Location,
3205 loc_b: Location,
3206 ret: Location,
3207 ) -> Result<(), CompileError> {
3208 self.emit_cmpop_i32_dynamic_b(Condition::Geu, loc_a, loc_b, ret, false)
3209 }
3210
3211 fn i32_cmp_gt_u(
3212 &mut self,
3213 loc_a: Location,
3214 loc_b: Location,
3215 ret: Location,
3216 ) -> Result<(), CompileError> {
3217 self.emit_cmpop_i32_dynamic_b(Condition::Gtu, loc_a, loc_b, ret, false)
3218 }
3219
3220 fn i32_cmp_le_u(
3221 &mut self,
3222 loc_a: Location,
3223 loc_b: Location,
3224 ret: Location,
3225 ) -> Result<(), CompileError> {
3226 self.emit_cmpop_i32_dynamic_b(Condition::Leu, loc_a, loc_b, ret, false)
3227 }
3228
3229 fn i32_cmp_lt_u(
3230 &mut self,
3231 loc_a: Location,
3232 loc_b: Location,
3233 ret: Location,
3234 ) -> Result<(), CompileError> {
3235 self.emit_cmpop_i32_dynamic_b(Condition::Ltu, loc_a, loc_b, ret, false)
3236 }
3237
3238 fn i32_cmp_ne(
3239 &mut self,
3240 loc_a: Location,
3241 loc_b: Location,
3242 ret: Location,
3243 ) -> Result<(), CompileError> {
3244 self.emit_cmpop_i32_dynamic_b(Condition::Ne, loc_a, loc_b, ret, true)
3245 }
3246
3247 fn i32_cmp_eq(
3248 &mut self,
3249 loc_a: Location,
3250 loc_b: Location,
3251 ret: Location,
3252 ) -> Result<(), CompileError> {
3253 self.emit_cmpop_i32_dynamic_b(Condition::Eq, loc_a, loc_b, ret, true)
3254 }
3255
3256 fn i32_clz(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
3257 self.emit_clz(Size::S32, loc, ret)
3258 }
3259
3260 fn i32_ctz(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
3261 self.emit_ctz(Size::S32, loc, ret)
3262 }
3263
3264 fn i32_popcnt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
3265 self.emit_popcnt(Size::S32, loc, ret)
3266 }
3267
3268 fn i32_shl(
3269 &mut self,
3270 loc_a: Location,
3271 loc_b: Location,
3272 ret: Location,
3273 ) -> Result<(), CompileError> {
3274 self.emit_relaxed_binop3(
3275 Assembler::emit_sll,
3276 Size::S32,
3277 loc_a,
3278 loc_b,
3279 ret,
3280 ImmType::Shift32,
3281 )
3282 }
3283
3284 fn i32_shr(
3285 &mut self,
3286 loc_a: Location,
3287 loc_b: Location,
3288 ret: Location,
3289 ) -> Result<(), CompileError> {
3290 self.emit_relaxed_binop3(
3291 Assembler::emit_srl,
3292 Size::S32,
3293 loc_a,
3294 loc_b,
3295 ret,
3296 ImmType::Shift32,
3297 )
3298 }
3299
3300 fn i32_sar(
3301 &mut self,
3302 loc_a: Location,
3303 loc_b: Location,
3304 ret: Location,
3305 ) -> Result<(), CompileError> {
3306 self.emit_relaxed_binop3(
3307 Assembler::emit_sra,
3308 Size::S32,
3309 loc_a,
3310 loc_b,
3311 ret,
3312 ImmType::Shift32,
3313 )
3314 }
3315
3316 fn i32_rol(
3317 &mut self,
3318 loc_a: Location,
3319 loc_b: Location,
3320 ret: Location,
3321 ) -> Result<(), CompileError> {
3322 self.emit_rol(Size::S32, loc_a, loc_b, ret, ImmType::Shift32)
3323 }
3324
3325 fn i32_ror(
3326 &mut self,
3327 loc_a: Location,
3328 loc_b: Location,
3329 ret: Location,
3330 ) -> Result<(), CompileError> {
3331 self.emit_ror(Size::S32, loc_a, loc_b, ret, ImmType::Shift32)
3332 }
3333
3334 fn i32_load(
3335 &mut self,
3336 addr: Location,
3337 memarg: &MemArg,
3338 ret: Location,
3339 _need_check: bool,
3340 imported_memories: bool,
3341 offset: i32,
3342 heap_access_oob: Label,
3343 unaligned_atomic: Label,
3344 ) -> Result<(), CompileError> {
3345 self.memory_op(
3346 addr,
3347 memarg,
3348 false,
3349 4,
3350 imported_memories,
3351 offset,
3352 heap_access_oob,
3353 unaligned_atomic,
3354 |this, addr| this.emit_maybe_unaligned_load(Size::S32, true, ret, addr),
3355 )
3356 }
3357
3358 fn i32_load_8u(
3359 &mut self,
3360 addr: Location,
3361 memarg: &MemArg,
3362 ret: Location,
3363 _need_check: bool,
3364 imported_memories: bool,
3365 offset: i32,
3366 heap_access_oob: Label,
3367 unaligned_atomic: Label,
3368 ) -> Result<(), CompileError> {
3369 self.memory_op(
3370 addr,
3371 memarg,
3372 false,
3373 1,
3374 imported_memories,
3375 offset,
3376 heap_access_oob,
3377 unaligned_atomic,
3378 |this, addr| this.emit_maybe_unaligned_load(Size::S8, false, ret, addr),
3379 )
3380 }
3381
3382 fn i32_load_8s(
3383 &mut self,
3384 addr: Location,
3385 memarg: &MemArg,
3386 ret: Location,
3387 _need_check: bool,
3388 imported_memories: bool,
3389 offset: i32,
3390 heap_access_oob: Label,
3391 unaligned_atomic: Label,
3392 ) -> Result<(), CompileError> {
3393 self.memory_op(
3394 addr,
3395 memarg,
3396 false,
3397 1,
3398 imported_memories,
3399 offset,
3400 heap_access_oob,
3401 unaligned_atomic,
3402 |this, addr| this.emit_maybe_unaligned_load(Size::S8, true, ret, addr),
3403 )
3404 }
3405
3406 fn i32_load_16u(
3407 &mut self,
3408 addr: Location,
3409 memarg: &MemArg,
3410 ret: Location,
3411 _need_check: bool,
3412 imported_memories: bool,
3413 offset: i32,
3414 heap_access_oob: Label,
3415 unaligned_atomic: Label,
3416 ) -> Result<(), CompileError> {
3417 self.memory_op(
3418 addr,
3419 memarg,
3420 false,
3421 2,
3422 imported_memories,
3423 offset,
3424 heap_access_oob,
3425 unaligned_atomic,
3426 |this, addr| this.emit_maybe_unaligned_load(Size::S16, false, ret, addr),
3427 )
3428 }
3429
3430 fn i32_load_16s(
3431 &mut self,
3432 addr: Location,
3433 memarg: &MemArg,
3434 ret: Location,
3435 _need_check: bool,
3436 imported_memories: bool,
3437 offset: i32,
3438 heap_access_oob: Label,
3439 unaligned_atomic: Label,
3440 ) -> Result<(), CompileError> {
3441 self.memory_op(
3442 addr,
3443 memarg,
3444 false,
3445 2,
3446 imported_memories,
3447 offset,
3448 heap_access_oob,
3449 unaligned_atomic,
3450 |this, addr| this.emit_maybe_unaligned_load(Size::S16, true, ret, addr),
3451 )
3452 }
3453
3454 fn i32_atomic_load(
3455 &mut self,
3456 addr: Location,
3457 memarg: &MemArg,
3458 ret: Location,
3459 _need_check: bool,
3460 imported_memories: bool,
3461 offset: i32,
3462 heap_access_oob: Label,
3463 unaligned_atomic: Label,
3464 ) -> Result<(), CompileError> {
3465 self.memory_op(
3466 addr,
3467 memarg,
3468 true,
3469 4,
3470 imported_memories,
3471 offset,
3472 heap_access_oob,
3473 unaligned_atomic,
3474 |this, addr| this.emit_relaxed_load(Size::S32, true, ret, Location::Memory(addr, 0)),
3475 )
3476 }
3477
3478 fn i32_atomic_load_8u(
3479 &mut self,
3480 addr: Location,
3481 memarg: &MemArg,
3482 ret: Location,
3483 _need_check: bool,
3484 imported_memories: bool,
3485 offset: i32,
3486 heap_access_oob: Label,
3487 unaligned_atomic: Label,
3488 ) -> Result<(), CompileError> {
3489 self.memory_op(
3490 addr,
3491 memarg,
3492 true,
3493 1,
3494 imported_memories,
3495 offset,
3496 heap_access_oob,
3497 unaligned_atomic,
3498 |this, addr| this.emit_relaxed_load(Size::S8, false, ret, Location::Memory(addr, 0)),
3499 )
3500 }
3501
3502 fn i32_atomic_load_16u(
3503 &mut self,
3504 addr: Location,
3505 memarg: &MemArg,
3506 ret: Location,
3507 _need_check: bool,
3508 imported_memories: bool,
3509 offset: i32,
3510 heap_access_oob: Label,
3511 unaligned_atomic: Label,
3512 ) -> Result<(), CompileError> {
3513 self.memory_op(
3514 addr,
3515 memarg,
3516 true,
3517 2,
3518 imported_memories,
3519 offset,
3520 heap_access_oob,
3521 unaligned_atomic,
3522 |this, addr| this.emit_relaxed_load(Size::S16, false, ret, Location::Memory(addr, 0)),
3523 )
3524 }
3525
3526 fn i32_save(
3527 &mut self,
3528 value: Location,
3529 memarg: &MemArg,
3530 addr: Location,
3531 _need_check: bool,
3532 imported_memories: bool,
3533 offset: i32,
3534 heap_access_oob: Label,
3535 unaligned_atomic: Label,
3536 ) -> Result<(), CompileError> {
3537 self.memory_op(
3538 addr,
3539 memarg,
3540 false,
3541 4,
3542 imported_memories,
3543 offset,
3544 heap_access_oob,
3545 unaligned_atomic,
3546 |this, addr| this.emit_maybe_unaligned_store(Size::S32, value, addr),
3547 )
3548 }
3549
3550 fn i32_save_8(
3551 &mut self,
3552 value: Location,
3553 memarg: &MemArg,
3554 addr: Location,
3555 _need_check: bool,
3556 imported_memories: bool,
3557 offset: i32,
3558 heap_access_oob: Label,
3559 unaligned_atomic: Label,
3560 ) -> Result<(), CompileError> {
3561 self.memory_op(
3562 addr,
3563 memarg,
3564 false,
3565 1,
3566 imported_memories,
3567 offset,
3568 heap_access_oob,
3569 unaligned_atomic,
3570 |this, addr| this.emit_maybe_unaligned_store(Size::S8, value, addr),
3571 )
3572 }
3573
3574 fn i32_save_16(
3575 &mut self,
3576 value: Location,
3577 memarg: &MemArg,
3578 addr: Location,
3579 _need_check: bool,
3580 imported_memories: bool,
3581 offset: i32,
3582 heap_access_oob: Label,
3583 unaligned_atomic: Label,
3584 ) -> Result<(), CompileError> {
3585 self.memory_op(
3586 addr,
3587 memarg,
3588 false,
3589 2,
3590 imported_memories,
3591 offset,
3592 heap_access_oob,
3593 unaligned_atomic,
3594 |this, addr| this.emit_maybe_unaligned_store(Size::S16, value, addr),
3595 )
3596 }
3597
3598 fn i32_atomic_save(
3599 &mut self,
3600 value: Location,
3601 memarg: &MemArg,
3602 addr: Location,
3603 _need_check: bool,
3604 imported_memories: bool,
3605 offset: i32,
3606 heap_access_oob: Label,
3607 unaligned_atomic: Label,
3608 ) -> Result<(), CompileError> {
3609 self.memory_op(
3610 addr,
3611 memarg,
3612 true,
3613 4,
3614 imported_memories,
3615 offset,
3616 heap_access_oob,
3617 unaligned_atomic,
3618 |this, addr| this.emit_relaxed_store(Size::S32, value, Location::Memory(addr, 0)),
3619 )?;
3620 self.assembler.emit_rwfence()
3621 }
3622
3623 fn i32_atomic_save_8(
3624 &mut self,
3625 value: Location,
3626 memarg: &MemArg,
3627 addr: Location,
3628 _need_check: bool,
3629 imported_memories: bool,
3630 offset: i32,
3631 heap_access_oob: Label,
3632 unaligned_atomic: Label,
3633 ) -> Result<(), CompileError> {
3634 self.memory_op(
3635 addr,
3636 memarg,
3637 true,
3638 1,
3639 imported_memories,
3640 offset,
3641 heap_access_oob,
3642 unaligned_atomic,
3643 |this, addr| this.emit_relaxed_store(Size::S8, value, Location::Memory(addr, 0)),
3644 )?;
3645 self.assembler.emit_rwfence()
3646 }
3647
3648 fn i32_atomic_save_16(
3649 &mut self,
3650 value: Location,
3651 memarg: &MemArg,
3652 addr: Location,
3653 _need_check: bool,
3654 imported_memories: bool,
3655 offset: i32,
3656 heap_access_oob: Label,
3657 unaligned_atomic: Label,
3658 ) -> Result<(), CompileError> {
3659 self.memory_op(
3660 addr,
3661 memarg,
3662 true,
3663 2,
3664 imported_memories,
3665 offset,
3666 heap_access_oob,
3667 unaligned_atomic,
3668 |this, addr| this.emit_relaxed_store(Size::S16, value, Location::Memory(addr, 0)),
3669 )?;
3670 self.assembler.emit_rwfence()
3671 }
3672
3673 fn i32_atomic_add(
3674 &mut self,
3675 loc: Location,
3676 target: Location,
3677 memarg: &MemArg,
3678 ret: Location,
3679 _need_check: bool,
3680 imported_memories: bool,
3681 offset: i32,
3682 heap_access_oob: Label,
3683 unaligned_atomic: Label,
3684 ) -> Result<(), CompileError> {
3685 self.memory_op(
3686 target,
3687 memarg,
3688 true,
3689 4,
3690 imported_memories,
3691 offset,
3692 heap_access_oob,
3693 unaligned_atomic,
3694 |this, addr| {
3695 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Add, Size::S32, ret, addr, loc)
3696 },
3697 )
3698 }
3699
3700 fn i32_atomic_add_8u(
3701 &mut self,
3702 loc: Location,
3703 target: Location,
3704 memarg: &MemArg,
3705 ret: Location,
3706 _need_check: bool,
3707 imported_memories: bool,
3708 offset: i32,
3709 heap_access_oob: Label,
3710 unaligned_atomic: Label,
3711 ) -> Result<(), CompileError> {
3712 self.memory_op(
3713 target,
3714 memarg,
3715 true,
3716 1,
3717 imported_memories,
3718 offset,
3719 heap_access_oob,
3720 unaligned_atomic,
3721 |this, addr| {
3722 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Add, Size::S8, ret, addr, loc)
3723 },
3724 )
3725 }
3726
3727 fn i32_atomic_add_16u(
3728 &mut self,
3729 loc: Location,
3730 target: Location,
3731 memarg: &MemArg,
3732 ret: Location,
3733 _need_check: bool,
3734 imported_memories: bool,
3735 offset: i32,
3736 heap_access_oob: Label,
3737 unaligned_atomic: Label,
3738 ) -> Result<(), CompileError> {
3739 self.memory_op(
3740 target,
3741 memarg,
3742 true,
3743 2,
3744 imported_memories,
3745 offset,
3746 heap_access_oob,
3747 unaligned_atomic,
3748 |this, addr| {
3749 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Add, Size::S16, ret, addr, loc)
3750 },
3751 )
3752 }
3753
3754 fn i32_atomic_sub(
3755 &mut self,
3756 loc: Location,
3757 target: Location,
3758 memarg: &MemArg,
3759 ret: Location,
3760 _need_check: bool,
3761 imported_memories: bool,
3762 offset: i32,
3763 heap_access_oob: Label,
3764 unaligned_atomic: Label,
3765 ) -> Result<(), CompileError> {
3766 self.memory_op(
3767 target,
3768 memarg,
3769 true,
3770 4,
3771 imported_memories,
3772 offset,
3773 heap_access_oob,
3774 unaligned_atomic,
3775 |this, addr| {
3776 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Sub, Size::S32, ret, addr, loc)
3777 },
3778 )
3779 }
3780
3781 fn i32_atomic_sub_8u(
3782 &mut self,
3783 loc: Location,
3784 target: Location,
3785 memarg: &MemArg,
3786 ret: Location,
3787 _need_check: bool,
3788 imported_memories: bool,
3789 offset: i32,
3790 heap_access_oob: Label,
3791 unaligned_atomic: Label,
3792 ) -> Result<(), CompileError> {
3793 self.memory_op(
3794 target,
3795 memarg,
3796 true,
3797 1,
3798 imported_memories,
3799 offset,
3800 heap_access_oob,
3801 unaligned_atomic,
3802 |this, addr| {
3803 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Sub, Size::S8, ret, addr, loc)
3804 },
3805 )
3806 }
3807
3808 fn i32_atomic_sub_16u(
3809 &mut self,
3810 loc: Location,
3811 target: Location,
3812 memarg: &MemArg,
3813 ret: Location,
3814 _need_check: bool,
3815 imported_memories: bool,
3816 offset: i32,
3817 heap_access_oob: Label,
3818 unaligned_atomic: Label,
3819 ) -> Result<(), CompileError> {
3820 self.memory_op(
3821 target,
3822 memarg,
3823 true,
3824 2,
3825 imported_memories,
3826 offset,
3827 heap_access_oob,
3828 unaligned_atomic,
3829 |this, addr| {
3830 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Sub, Size::S16, ret, addr, loc)
3831 },
3832 )
3833 }
3834
3835 fn i32_atomic_and(
3836 &mut self,
3837 loc: Location,
3838 target: Location,
3839 memarg: &MemArg,
3840 ret: Location,
3841 _need_check: bool,
3842 imported_memories: bool,
3843 offset: i32,
3844 heap_access_oob: Label,
3845 unaligned_atomic: Label,
3846 ) -> Result<(), CompileError> {
3847 self.memory_op(
3848 target,
3849 memarg,
3850 true,
3851 4,
3852 imported_memories,
3853 offset,
3854 heap_access_oob,
3855 unaligned_atomic,
3856 |this, addr| {
3857 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::And, Size::S32, ret, addr, loc)
3858 },
3859 )
3860 }
3861
3862 fn i32_atomic_and_8u(
3863 &mut self,
3864 loc: Location,
3865 target: Location,
3866 memarg: &MemArg,
3867 ret: Location,
3868 _need_check: bool,
3869 imported_memories: bool,
3870 offset: i32,
3871 heap_access_oob: Label,
3872 unaligned_atomic: Label,
3873 ) -> Result<(), CompileError> {
3874 self.memory_op(
3875 target,
3876 memarg,
3877 true,
3878 1,
3879 imported_memories,
3880 offset,
3881 heap_access_oob,
3882 unaligned_atomic,
3883 |this, addr| {
3884 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::And, Size::S8, ret, addr, loc)
3885 },
3886 )
3887 }
3888
3889 fn i32_atomic_and_16u(
3890 &mut self,
3891 loc: Location,
3892 target: Location,
3893 memarg: &MemArg,
3894 ret: Location,
3895 _need_check: bool,
3896 imported_memories: bool,
3897 offset: i32,
3898 heap_access_oob: Label,
3899 unaligned_atomic: Label,
3900 ) -> Result<(), CompileError> {
3901 self.memory_op(
3902 target,
3903 memarg,
3904 true,
3905 2,
3906 imported_memories,
3907 offset,
3908 heap_access_oob,
3909 unaligned_atomic,
3910 |this, addr| {
3911 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::And, Size::S16, ret, addr, loc)
3912 },
3913 )
3914 }
3915
3916 fn i32_atomic_or(
3917 &mut self,
3918 loc: Location,
3919 target: Location,
3920 memarg: &MemArg,
3921 ret: Location,
3922 _need_check: bool,
3923 imported_memories: bool,
3924 offset: i32,
3925 heap_access_oob: Label,
3926 unaligned_atomic: Label,
3927 ) -> Result<(), CompileError> {
3928 self.memory_op(
3929 target,
3930 memarg,
3931 true,
3932 4,
3933 imported_memories,
3934 offset,
3935 heap_access_oob,
3936 unaligned_atomic,
3937 |this, addr| {
3938 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Or, Size::S32, ret, addr, loc)
3939 },
3940 )
3941 }
3942
3943 fn i32_atomic_or_8u(
3944 &mut self,
3945 loc: Location,
3946 target: Location,
3947 memarg: &MemArg,
3948 ret: Location,
3949 _need_check: bool,
3950 imported_memories: bool,
3951 offset: i32,
3952 heap_access_oob: Label,
3953 unaligned_atomic: Label,
3954 ) -> Result<(), CompileError> {
3955 self.memory_op(
3956 target,
3957 memarg,
3958 true,
3959 1,
3960 imported_memories,
3961 offset,
3962 heap_access_oob,
3963 unaligned_atomic,
3964 |this, addr| {
3965 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Or, Size::S8, ret, addr, loc)
3966 },
3967 )
3968 }
3969
3970 fn i32_atomic_or_16u(
3971 &mut self,
3972 loc: Location,
3973 target: Location,
3974 memarg: &MemArg,
3975 ret: Location,
3976 _need_check: bool,
3977 imported_memories: bool,
3978 offset: i32,
3979 heap_access_oob: Label,
3980 unaligned_atomic: Label,
3981 ) -> Result<(), CompileError> {
3982 self.memory_op(
3983 target,
3984 memarg,
3985 true,
3986 2,
3987 imported_memories,
3988 offset,
3989 heap_access_oob,
3990 unaligned_atomic,
3991 |this, addr| {
3992 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Or, Size::S16, ret, addr, loc)
3993 },
3994 )
3995 }
3996
3997 fn i32_atomic_xor(
3998 &mut self,
3999 loc: Location,
4000 target: Location,
4001 memarg: &MemArg,
4002 ret: Location,
4003 _need_check: bool,
4004 imported_memories: bool,
4005 offset: i32,
4006 heap_access_oob: Label,
4007 unaligned_atomic: Label,
4008 ) -> Result<(), CompileError> {
4009 self.memory_op(
4010 target,
4011 memarg,
4012 true,
4013 4,
4014 imported_memories,
4015 offset,
4016 heap_access_oob,
4017 unaligned_atomic,
4018 |this, addr| {
4019 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Xor, Size::S32, ret, addr, loc)
4020 },
4021 )
4022 }
4023
4024 fn i32_atomic_xor_8u(
4025 &mut self,
4026 loc: Location,
4027 target: Location,
4028 memarg: &MemArg,
4029 ret: Location,
4030 _need_check: bool,
4031 imported_memories: bool,
4032 offset: i32,
4033 heap_access_oob: Label,
4034 unaligned_atomic: Label,
4035 ) -> Result<(), CompileError> {
4036 self.memory_op(
4037 target,
4038 memarg,
4039 true,
4040 1,
4041 imported_memories,
4042 offset,
4043 heap_access_oob,
4044 unaligned_atomic,
4045 |this, addr| {
4046 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Xor, Size::S8, ret, addr, loc)
4047 },
4048 )
4049 }
4050
4051 fn i32_atomic_xor_16u(
4052 &mut self,
4053 loc: Location,
4054 target: Location,
4055 memarg: &MemArg,
4056 ret: Location,
4057 _need_check: bool,
4058 imported_memories: bool,
4059 offset: i32,
4060 heap_access_oob: Label,
4061 unaligned_atomic: Label,
4062 ) -> Result<(), CompileError> {
4063 self.memory_op(
4064 target,
4065 memarg,
4066 true,
4067 2,
4068 imported_memories,
4069 offset,
4070 heap_access_oob,
4071 unaligned_atomic,
4072 |this, addr| {
4073 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Xor, Size::S16, ret, addr, loc)
4074 },
4075 )
4076 }
4077
4078 fn i32_atomic_xchg(
4079 &mut self,
4080 loc: Location,
4081 target: Location,
4082 memarg: &MemArg,
4083 ret: Location,
4084 _need_check: bool,
4085 imported_memories: bool,
4086 offset: i32,
4087 heap_access_oob: Label,
4088 unaligned_atomic: Label,
4089 ) -> Result<(), CompileError> {
4090 self.memory_op(
4091 target,
4092 memarg,
4093 true,
4094 4,
4095 imported_memories,
4096 offset,
4097 heap_access_oob,
4098 unaligned_atomic,
4099 |this, addr| {
4100 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Exchange, Size::S32, ret, addr, loc)
4101 },
4102 )
4103 }
4104
4105 fn i32_atomic_xchg_8u(
4106 &mut self,
4107 loc: Location,
4108 target: Location,
4109 memarg: &MemArg,
4110 ret: Location,
4111 _need_check: bool,
4112 imported_memories: bool,
4113 offset: i32,
4114 heap_access_oob: Label,
4115 unaligned_atomic: Label,
4116 ) -> Result<(), CompileError> {
4117 self.memory_op(
4118 target,
4119 memarg,
4120 true,
4121 1,
4122 imported_memories,
4123 offset,
4124 heap_access_oob,
4125 unaligned_atomic,
4126 |this, addr| {
4127 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Exchange, Size::S8, ret, addr, loc)
4128 },
4129 )
4130 }
4131
4132 fn i32_atomic_xchg_16u(
4133 &mut self,
4134 loc: Location,
4135 target: Location,
4136 memarg: &MemArg,
4137 ret: Location,
4138 _need_check: bool,
4139 imported_memories: bool,
4140 offset: i32,
4141 heap_access_oob: Label,
4142 unaligned_atomic: Label,
4143 ) -> Result<(), CompileError> {
4144 self.memory_op(
4145 target,
4146 memarg,
4147 true,
4148 2,
4149 imported_memories,
4150 offset,
4151 heap_access_oob,
4152 unaligned_atomic,
4153 |this, addr| {
4154 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Exchange, Size::S16, ret, addr, loc)
4155 },
4156 )
4157 }
4158
4159 fn i32_atomic_cmpxchg(
4160 &mut self,
4161 new: Location,
4162 cmp: Location,
4163 target: Location,
4164 memarg: &MemArg,
4165 ret: Location,
4166 _need_check: bool,
4167 imported_memories: bool,
4168 offset: i32,
4169 heap_access_oob: Label,
4170 unaligned_atomic: Label,
4171 ) -> Result<(), CompileError> {
4172 self.memory_op(
4173 target,
4174 memarg,
4175 true,
4176 4,
4177 imported_memories,
4178 offset,
4179 heap_access_oob,
4180 unaligned_atomic,
4181 |this, addr| this.emit_relaxed_atomic_cmpxchg(Size::S32, ret, addr, new, cmp),
4182 )
4183 }
4184
4185 fn i32_atomic_cmpxchg_8u(
4186 &mut self,
4187 new: Location,
4188 cmp: Location,
4189 target: Location,
4190 memarg: &MemArg,
4191 ret: Location,
4192 _need_check: bool,
4193 imported_memories: bool,
4194 offset: i32,
4195 heap_access_oob: Label,
4196 unaligned_atomic: Label,
4197 ) -> Result<(), CompileError> {
4198 self.memory_op(
4199 target,
4200 memarg,
4201 true,
4202 1,
4203 imported_memories,
4204 offset,
4205 heap_access_oob,
4206 unaligned_atomic,
4207 |this, addr| this.emit_relaxed_atomic_cmpxchg(Size::S8, ret, addr, new, cmp),
4208 )
4209 }
4210
4211 fn i32_atomic_cmpxchg_16u(
4212 &mut self,
4213 new: Location,
4214 cmp: Location,
4215 target: Location,
4216 memarg: &MemArg,
4217 ret: Location,
4218 _need_check: bool,
4219 imported_memories: bool,
4220 offset: i32,
4221 heap_access_oob: Label,
4222 unaligned_atomic: Label,
4223 ) -> Result<(), CompileError> {
4224 self.memory_op(
4225 target,
4226 memarg,
4227 true,
4228 2,
4229 imported_memories,
4230 offset,
4231 heap_access_oob,
4232 unaligned_atomic,
4233 |this, addr| this.emit_relaxed_atomic_cmpxchg(Size::S16, ret, addr, new, cmp),
4234 )
4235 }
4236
4237 fn emit_call_with_reloc(
4238 &mut self,
4239 _calling_convention: CallingConvention,
4240 reloc_target: RelocationTarget,
4241 ) -> Result<Vec<Relocation>, CompileError> {
4242 let mut relocations = vec![];
4243 let next = self.get_label();
4244 let reloc_at = self.assembler.get_offset().0;
4245 self.emit_label(next)?; self.assembler.emit_call_label(next)?;
4247 relocations.push(Relocation {
4248 kind: RelocationKind::RiscvCall,
4249 reloc_target,
4250 offset: reloc_at as u32,
4251 addend: 0,
4252 });
4253 Ok(relocations)
4254 }
4255
4256 fn emit_binop_add64(
4257 &mut self,
4258 loc_a: Location,
4259 loc_b: Location,
4260 ret: Location,
4261 ) -> Result<(), CompileError> {
4262 self.emit_relaxed_binop3(
4263 Assembler::emit_add,
4264 Size::S64,
4265 loc_a,
4266 loc_b,
4267 ret,
4268 ImmType::Bits12,
4269 )
4270 }
4271
4272 fn emit_binop_sub64(
4273 &mut self,
4274 loc_a: Location,
4275 loc_b: Location,
4276 ret: Location,
4277 ) -> Result<(), CompileError> {
4278 self.emit_relaxed_binop3(
4279 Assembler::emit_sub,
4280 Size::S64,
4281 loc_a,
4282 loc_b,
4283 ret,
4284 ImmType::Bits12Subtraction,
4285 )
4286 }
4287
4288 fn emit_binop_mul64(
4289 &mut self,
4290 loc_a: Location,
4291 loc_b: Location,
4292 ret: Location,
4293 ) -> Result<(), CompileError> {
4294 self.emit_relaxed_binop3(
4295 Assembler::emit_mul,
4296 Size::S64,
4297 loc_a,
4298 loc_b,
4299 ret,
4300 ImmType::None,
4301 )
4302 }
4303
4304 fn emit_binop_udiv64(
4305 &mut self,
4306 loc_a: Location,
4307 loc_b: Location,
4308 ret: Location,
4309 integer_division_by_zero: Label,
4310 ) -> Result<usize, CompileError> {
4311 let mut temps = vec![];
4312 let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4313 let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4314 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4315
4316 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
4317 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4318 })?;
4319 temps.push(jmp_tmp);
4320
4321 self.assembler
4322 .emit_on_false_label_far(src2, integer_division_by_zero, jmp_tmp)?;
4323 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
4324 self.assembler.emit_udiv(Size::S64, src1, src2, dest)?;
4325 if ret != dest {
4326 self.move_location(Size::S64, dest, ret)?;
4327 }
4328 for r in temps {
4329 self.release_gpr(r);
4330 }
4331 Ok(offset)
4332 }
4333
4334 fn emit_binop_sdiv64(
4335 &mut self,
4336 loc_a: Location,
4337 loc_b: Location,
4338 ret: Location,
4339 integer_division_by_zero: Label,
4340 integer_overflow: Label,
4341 ) -> Result<usize, CompileError> {
4342 let mut temps = vec![];
4343 let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4344 let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4345 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4346
4347 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
4348 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4349 })?;
4350 temps.push(jmp_tmp);
4351
4352 self.assembler
4353 .emit_on_false_label_far(src2, integer_division_by_zero, jmp_tmp)?;
4354 let label_nooverflow = self.assembler.get_label();
4355 let tmp = self.location_to_reg(
4356 Size::S64,
4357 Location::Imm64(i64::MIN as u64),
4358 &mut temps,
4359 ImmType::None,
4360 true,
4361 None,
4362 )?;
4363
4364 self.assembler.emit_cmp(Condition::Ne, tmp, src1, tmp)?;
4365 self.assembler
4366 .emit_on_true_label(tmp, label_nooverflow, jmp_tmp)?;
4367 self.move_location(Size::S64, Location::Imm64(-1i64 as _), tmp)?;
4368 self.assembler.emit_cmp(Condition::Eq, tmp, src2, tmp)?;
4369 self.assembler
4370 .emit_on_true_label_far(tmp, integer_overflow, jmp_tmp)?;
4371 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
4372 self.assembler.emit_label(label_nooverflow)?;
4373 self.assembler.emit_sdiv(Size::S64, src1, src2, dest)?;
4374 if ret != dest {
4375 self.move_location(Size::S64, dest, ret)?;
4376 }
4377 for r in temps {
4378 self.release_gpr(r);
4379 }
4380 Ok(offset)
4381 }
4382
4383 fn emit_binop_urem64(
4384 &mut self,
4385 loc_a: Location,
4386 loc_b: Location,
4387 ret: Location,
4388 integer_division_by_zero: Label,
4389 ) -> Result<usize, CompileError> {
4390 let mut temps = vec![];
4391 let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4392 let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4393 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4394
4395 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
4396 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4397 })?;
4398 temps.push(jmp_tmp);
4399
4400 self.assembler
4401 .emit_on_false_label_far(src2, integer_division_by_zero, jmp_tmp)?;
4402 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
4403 self.assembler.emit_urem(Size::S64, src1, src2, dest)?;
4404 if ret != dest {
4405 self.move_location(Size::S64, dest, ret)?;
4406 }
4407 for r in temps {
4408 self.release_gpr(r);
4409 }
4410 Ok(offset)
4411 }
4412
4413 fn emit_binop_srem64(
4414 &mut self,
4415 loc_a: Location,
4416 loc_b: Location,
4417 ret: Location,
4418 integer_division_by_zero: Label,
4419 ) -> Result<usize, CompileError> {
4420 let mut temps = vec![];
4421 let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4422 let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4423 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4424
4425 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
4426 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4427 })?;
4428 temps.push(jmp_tmp);
4429
4430 self.assembler
4431 .emit_on_false_label_far(src2, integer_division_by_zero, jmp_tmp)?;
4432 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
4433 self.assembler.emit_srem(Size::S64, src1, src2, dest)?;
4434 if ret != dest {
4435 self.move_location(Size::S64, dest, ret)?;
4436 }
4437 for r in temps {
4438 self.release_gpr(r);
4439 }
4440 Ok(offset)
4441 }
4442
4443 fn emit_binop_and64(
4444 &mut self,
4445 loc_a: Location,
4446 loc_b: Location,
4447 ret: Location,
4448 ) -> Result<(), CompileError> {
4449 self.emit_relaxed_binop3(
4450 Assembler::emit_and,
4451 Size::S64,
4452 loc_a,
4453 loc_b,
4454 ret,
4455 ImmType::Bits12,
4456 )
4457 }
4458
4459 fn emit_binop_or64(
4460 &mut self,
4461 loc_a: Location,
4462 loc_b: Location,
4463 ret: Location,
4464 ) -> Result<(), CompileError> {
4465 self.emit_relaxed_binop3(
4466 Assembler::emit_or,
4467 Size::S64,
4468 loc_a,
4469 loc_b,
4470 ret,
4471 ImmType::Bits12,
4472 )
4473 }
4474
4475 fn emit_binop_xor64(
4476 &mut self,
4477 loc_a: Location,
4478 loc_b: Location,
4479 ret: Location,
4480 ) -> Result<(), CompileError> {
4481 self.emit_relaxed_binop3(
4482 Assembler::emit_xor,
4483 Size::S64,
4484 loc_a,
4485 loc_b,
4486 ret,
4487 ImmType::Bits12,
4488 )
4489 }
4490
4491 fn i64_cmp_ge_s(
4492 &mut self,
4493 loc_a: Location,
4494 loc_b: Location,
4495 ret: Location,
4496 ) -> Result<(), CompileError> {
4497 self.emit_cmpop_i64_dynamic_b(Condition::Ge, loc_a, loc_b, ret)
4498 }
4499
4500 fn i64_cmp_gt_s(
4501 &mut self,
4502 loc_a: Location,
4503 loc_b: Location,
4504 ret: Location,
4505 ) -> Result<(), CompileError> {
4506 self.emit_cmpop_i64_dynamic_b(Condition::Gt, loc_a, loc_b, ret)
4507 }
4508
4509 fn i64_cmp_le_s(
4510 &mut self,
4511 loc_a: Location,
4512 loc_b: Location,
4513 ret: Location,
4514 ) -> Result<(), CompileError> {
4515 self.emit_cmpop_i64_dynamic_b(Condition::Le, loc_a, loc_b, ret)
4516 }
4517
4518 fn i64_cmp_lt_s(
4519 &mut self,
4520 loc_a: Location,
4521 loc_b: Location,
4522 ret: Location,
4523 ) -> Result<(), CompileError> {
4524 self.emit_cmpop_i64_dynamic_b(Condition::Lt, loc_a, loc_b, ret)
4525 }
4526
4527 fn i64_cmp_ge_u(
4528 &mut self,
4529 loc_a: Location,
4530 loc_b: Location,
4531 ret: Location,
4532 ) -> Result<(), CompileError> {
4533 self.emit_cmpop_i64_dynamic_b(Condition::Geu, loc_a, loc_b, ret)
4534 }
4535
4536 fn i64_cmp_gt_u(
4537 &mut self,
4538 loc_a: Location,
4539 loc_b: Location,
4540 ret: Location,
4541 ) -> Result<(), CompileError> {
4542 self.emit_cmpop_i64_dynamic_b(Condition::Gtu, loc_a, loc_b, ret)
4543 }
4544
4545 fn i64_cmp_le_u(
4546 &mut self,
4547 loc_a: Location,
4548 loc_b: Location,
4549 ret: Location,
4550 ) -> Result<(), CompileError> {
4551 self.emit_cmpop_i64_dynamic_b(Condition::Leu, loc_a, loc_b, ret)
4552 }
4553
4554 fn i64_cmp_lt_u(
4555 &mut self,
4556 loc_a: Location,
4557 loc_b: Location,
4558 ret: Location,
4559 ) -> Result<(), CompileError> {
4560 self.emit_cmpop_i64_dynamic_b(Condition::Ltu, loc_a, loc_b, ret)
4561 }
4562
4563 fn i64_cmp_ne(
4564 &mut self,
4565 loc_a: Location,
4566 loc_b: Location,
4567 ret: Location,
4568 ) -> Result<(), CompileError> {
4569 self.emit_cmpop_i64_dynamic_b(Condition::Ne, loc_a, loc_b, ret)
4570 }
4571
4572 fn i64_cmp_eq(
4573 &mut self,
4574 loc_a: Location,
4575 loc_b: Location,
4576 ret: Location,
4577 ) -> Result<(), CompileError> {
4578 self.emit_cmpop_i64_dynamic_b(Condition::Eq, loc_a, loc_b, ret)
4579 }
4580
4581 fn i64_clz(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
4582 self.emit_clz(Size::S64, loc, ret)
4583 }
4584
4585 fn i64_ctz(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
4586 self.emit_ctz(Size::S64, loc, ret)
4587 }
4588
4589 fn i64_popcnt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
4590 self.emit_popcnt(Size::S64, loc, ret)
4591 }
4592
4593 fn i64_shl(
4594 &mut self,
4595 loc_a: Location,
4596 loc_b: Location,
4597 ret: Location,
4598 ) -> Result<(), CompileError> {
4599 self.emit_relaxed_binop3(
4600 Assembler::emit_sll,
4601 Size::S64,
4602 loc_a,
4603 loc_b,
4604 ret,
4605 ImmType::Shift64,
4606 )
4607 }
4608
4609 fn i64_shr(
4610 &mut self,
4611 loc_a: Location,
4612 loc_b: Location,
4613 ret: Location,
4614 ) -> Result<(), CompileError> {
4615 self.emit_relaxed_binop3(
4616 Assembler::emit_srl,
4617 Size::S64,
4618 loc_a,
4619 loc_b,
4620 ret,
4621 ImmType::Shift64,
4622 )
4623 }
4624
4625 fn i64_sar(
4626 &mut self,
4627 loc_a: Location,
4628 loc_b: Location,
4629 ret: Location,
4630 ) -> Result<(), CompileError> {
4631 self.emit_relaxed_binop3(
4632 Assembler::emit_sra,
4633 Size::S64,
4634 loc_a,
4635 loc_b,
4636 ret,
4637 ImmType::Shift64,
4638 )
4639 }
4640
4641 fn i64_rol(
4642 &mut self,
4643 loc_a: Location,
4644 loc_b: Location,
4645 ret: Location,
4646 ) -> Result<(), CompileError> {
4647 self.emit_rol(Size::S64, loc_a, loc_b, ret, ImmType::Shift64)
4648 }
4649
4650 fn i64_ror(
4651 &mut self,
4652 loc_a: Location,
4653 loc_b: Location,
4654 ret: Location,
4655 ) -> Result<(), CompileError> {
4656 self.emit_ror(Size::S64, loc_a, loc_b, ret, ImmType::Shift64)
4657 }
4658
4659 fn i64_load(
4660 &mut self,
4661 addr: Location,
4662 memarg: &MemArg,
4663 ret: Location,
4664 _need_check: bool,
4665 imported_memories: bool,
4666 offset: i32,
4667 heap_access_oob: Label,
4668 unaligned_atomic: Label,
4669 ) -> Result<(), CompileError> {
4670 self.memory_op(
4671 addr,
4672 memarg,
4673 false,
4674 8,
4675 imported_memories,
4676 offset,
4677 heap_access_oob,
4678 unaligned_atomic,
4679 |this, addr| this.emit_maybe_unaligned_load(Size::S64, true, ret, addr),
4680 )
4681 }
4682
4683 fn i64_load_8u(
4684 &mut self,
4685 addr: Location,
4686 memarg: &MemArg,
4687 ret: Location,
4688 _need_check: bool,
4689 imported_memories: bool,
4690 offset: i32,
4691 heap_access_oob: Label,
4692 unaligned_atomic: Label,
4693 ) -> Result<(), CompileError> {
4694 self.memory_op(
4695 addr,
4696 memarg,
4697 false,
4698 1,
4699 imported_memories,
4700 offset,
4701 heap_access_oob,
4702 unaligned_atomic,
4703 |this, addr| this.emit_maybe_unaligned_load(Size::S8, false, ret, addr),
4704 )
4705 }
4706
4707 fn i64_load_8s(
4708 &mut self,
4709 addr: Location,
4710 memarg: &MemArg,
4711 ret: Location,
4712 _need_check: bool,
4713 imported_memories: bool,
4714 offset: i32,
4715 heap_access_oob: Label,
4716 unaligned_atomic: Label,
4717 ) -> Result<(), CompileError> {
4718 self.memory_op(
4719 addr,
4720 memarg,
4721 false,
4722 1,
4723 imported_memories,
4724 offset,
4725 heap_access_oob,
4726 unaligned_atomic,
4727 |this, addr| this.emit_maybe_unaligned_load(Size::S8, true, ret, addr),
4728 )
4729 }
4730
4731 fn i64_load_32u(
4732 &mut self,
4733 addr: Location,
4734 memarg: &MemArg,
4735 ret: Location,
4736 _need_check: bool,
4737 imported_memories: bool,
4738 offset: i32,
4739 heap_access_oob: Label,
4740 unaligned_atomic: Label,
4741 ) -> Result<(), CompileError> {
4742 self.memory_op(
4743 addr,
4744 memarg,
4745 false,
4746 4,
4747 imported_memories,
4748 offset,
4749 heap_access_oob,
4750 unaligned_atomic,
4751 |this, addr| this.emit_maybe_unaligned_load(Size::S32, false, ret, addr),
4752 )
4753 }
4754
4755 fn i64_load_32s(
4756 &mut self,
4757 addr: Location,
4758 memarg: &MemArg,
4759 ret: Location,
4760 _need_check: bool,
4761 imported_memories: bool,
4762 offset: i32,
4763 heap_access_oob: Label,
4764 unaligned_atomic: Label,
4765 ) -> Result<(), CompileError> {
4766 self.memory_op(
4767 addr,
4768 memarg,
4769 false,
4770 4,
4771 imported_memories,
4772 offset,
4773 heap_access_oob,
4774 unaligned_atomic,
4775 |this, addr| this.emit_maybe_unaligned_load(Size::S32, true, ret, addr),
4776 )
4777 }
4778
4779 fn i64_load_16u(
4780 &mut self,
4781 addr: Location,
4782 memarg: &MemArg,
4783 ret: Location,
4784 _need_check: bool,
4785 imported_memories: bool,
4786 offset: i32,
4787 heap_access_oob: Label,
4788 unaligned_atomic: Label,
4789 ) -> Result<(), CompileError> {
4790 self.memory_op(
4791 addr,
4792 memarg,
4793 false,
4794 2,
4795 imported_memories,
4796 offset,
4797 heap_access_oob,
4798 unaligned_atomic,
4799 |this, addr| this.emit_maybe_unaligned_load(Size::S16, false, ret, addr),
4800 )
4801 }
4802
4803 fn i64_load_16s(
4804 &mut self,
4805 addr: Location,
4806 memarg: &MemArg,
4807 ret: Location,
4808 _need_check: bool,
4809 imported_memories: bool,
4810 offset: i32,
4811 heap_access_oob: Label,
4812 unaligned_atomic: Label,
4813 ) -> Result<(), CompileError> {
4814 self.memory_op(
4815 addr,
4816 memarg,
4817 false,
4818 2,
4819 imported_memories,
4820 offset,
4821 heap_access_oob,
4822 unaligned_atomic,
4823 |this, addr| this.emit_maybe_unaligned_load(Size::S16, true, ret, addr),
4824 )
4825 }
4826
4827 fn i64_atomic_load(
4828 &mut self,
4829 addr: Location,
4830 memarg: &MemArg,
4831 ret: Location,
4832 _need_check: bool,
4833 imported_memories: bool,
4834 offset: i32,
4835 heap_access_oob: Label,
4836 unaligned_atomic: Label,
4837 ) -> Result<(), CompileError> {
4838 self.memory_op(
4839 addr,
4840 memarg,
4841 true,
4842 8,
4843 imported_memories,
4844 offset,
4845 heap_access_oob,
4846 unaligned_atomic,
4847 |this, addr| this.emit_relaxed_load(Size::S64, true, ret, Location::Memory(addr, 0)),
4848 )
4849 }
4850
4851 fn i64_atomic_load_8u(
4852 &mut self,
4853 addr: Location,
4854 memarg: &MemArg,
4855 ret: Location,
4856 _need_check: bool,
4857 imported_memories: bool,
4858 offset: i32,
4859 heap_access_oob: Label,
4860 unaligned_atomic: Label,
4861 ) -> Result<(), CompileError> {
4862 self.memory_op(
4863 addr,
4864 memarg,
4865 true,
4866 1,
4867 imported_memories,
4868 offset,
4869 heap_access_oob,
4870 unaligned_atomic,
4871 |this, addr| this.emit_relaxed_load(Size::S8, false, ret, Location::Memory(addr, 0)),
4872 )
4873 }
4874
4875 fn i64_atomic_load_16u(
4876 &mut self,
4877 addr: Location,
4878 memarg: &MemArg,
4879 ret: Location,
4880 _need_check: bool,
4881 imported_memories: bool,
4882 offset: i32,
4883 heap_access_oob: Label,
4884 unaligned_atomic: Label,
4885 ) -> Result<(), CompileError> {
4886 self.memory_op(
4887 addr,
4888 memarg,
4889 true,
4890 2,
4891 imported_memories,
4892 offset,
4893 heap_access_oob,
4894 unaligned_atomic,
4895 |this, addr| this.emit_relaxed_load(Size::S16, false, ret, Location::Memory(addr, 0)),
4896 )
4897 }
4898
4899 fn i64_atomic_load_32u(
4900 &mut self,
4901 addr: Location,
4902 memarg: &MemArg,
4903 ret: Location,
4904 _need_check: bool,
4905 imported_memories: bool,
4906 offset: i32,
4907 heap_access_oob: Label,
4908 unaligned_atomic: Label,
4909 ) -> Result<(), CompileError> {
4910 self.memory_op(
4911 addr,
4912 memarg,
4913 true,
4914 4,
4915 imported_memories,
4916 offset,
4917 heap_access_oob,
4918 unaligned_atomic,
4919 |this, addr| this.emit_relaxed_load(Size::S32, false, ret, Location::Memory(addr, 0)),
4920 )
4921 }
4922
4923 fn i64_save(
4924 &mut self,
4925 value: Location,
4926 memarg: &MemArg,
4927 addr: Location,
4928 _need_check: bool,
4929 imported_memories: bool,
4930 offset: i32,
4931 heap_access_oob: Label,
4932 unaligned_atomic: Label,
4933 ) -> Result<(), CompileError> {
4934 self.memory_op(
4935 addr,
4936 memarg,
4937 false,
4938 8,
4939 imported_memories,
4940 offset,
4941 heap_access_oob,
4942 unaligned_atomic,
4943 |this, addr| this.emit_maybe_unaligned_store(Size::S64, value, addr),
4944 )
4945 }
4946
4947 fn i64_save_8(
4948 &mut self,
4949 value: Location,
4950 memarg: &MemArg,
4951 addr: Location,
4952 _need_check: bool,
4953 imported_memories: bool,
4954 offset: i32,
4955 heap_access_oob: Label,
4956 unaligned_atomic: Label,
4957 ) -> Result<(), CompileError> {
4958 self.memory_op(
4959 addr,
4960 memarg,
4961 false,
4962 1,
4963 imported_memories,
4964 offset,
4965 heap_access_oob,
4966 unaligned_atomic,
4967 |this, addr| this.emit_maybe_unaligned_store(Size::S8, value, addr),
4968 )
4969 }
4970
4971 fn i64_save_16(
4972 &mut self,
4973 value: Location,
4974 memarg: &MemArg,
4975 addr: Location,
4976 _need_check: bool,
4977 imported_memories: bool,
4978 offset: i32,
4979 heap_access_oob: Label,
4980 unaligned_atomic: Label,
4981 ) -> Result<(), CompileError> {
4982 self.memory_op(
4983 addr,
4984 memarg,
4985 false,
4986 2,
4987 imported_memories,
4988 offset,
4989 heap_access_oob,
4990 unaligned_atomic,
4991 |this, addr| this.emit_maybe_unaligned_store(Size::S16, value, addr),
4992 )
4993 }
4994
4995 fn i64_save_32(
4996 &mut self,
4997 value: Location,
4998 memarg: &MemArg,
4999 addr: Location,
5000 _need_check: bool,
5001 imported_memories: bool,
5002 offset: i32,
5003 heap_access_oob: Label,
5004 unaligned_atomic: Label,
5005 ) -> Result<(), CompileError> {
5006 self.memory_op(
5007 addr,
5008 memarg,
5009 false,
5010 4,
5011 imported_memories,
5012 offset,
5013 heap_access_oob,
5014 unaligned_atomic,
5015 |this, addr| this.emit_maybe_unaligned_store(Size::S32, value, addr),
5016 )
5017 }
5018
5019 fn i64_atomic_save(
5020 &mut self,
5021 value: Location,
5022 memarg: &MemArg,
5023 addr: Location,
5024 _need_check: bool,
5025 imported_memories: bool,
5026 offset: i32,
5027 heap_access_oob: Label,
5028 unaligned_atomic: Label,
5029 ) -> Result<(), CompileError> {
5030 self.memory_op(
5031 addr,
5032 memarg,
5033 true,
5034 8,
5035 imported_memories,
5036 offset,
5037 heap_access_oob,
5038 unaligned_atomic,
5039 |this, addr| this.emit_relaxed_store(Size::S64, value, Location::Memory(addr, 0)),
5040 )?;
5041 self.assembler.emit_rwfence()
5042 }
5043
5044 fn i64_atomic_save_8(
5045 &mut self,
5046 value: Location,
5047 memarg: &MemArg,
5048 addr: Location,
5049 _need_check: bool,
5050 imported_memories: bool,
5051 offset: i32,
5052 heap_access_oob: Label,
5053 unaligned_atomic: Label,
5054 ) -> Result<(), CompileError> {
5055 self.memory_op(
5056 addr,
5057 memarg,
5058 true,
5059 1,
5060 imported_memories,
5061 offset,
5062 heap_access_oob,
5063 unaligned_atomic,
5064 |this, addr| this.emit_relaxed_store(Size::S8, value, Location::Memory(addr, 0)),
5065 )?;
5066 self.assembler.emit_rwfence()
5067 }
5068
5069 fn i64_atomic_save_16(
5070 &mut self,
5071 value: Location,
5072 memarg: &MemArg,
5073 addr: Location,
5074 _need_check: bool,
5075 imported_memories: bool,
5076 offset: i32,
5077 heap_access_oob: Label,
5078 unaligned_atomic: Label,
5079 ) -> Result<(), CompileError> {
5080 self.memory_op(
5081 addr,
5082 memarg,
5083 true,
5084 2,
5085 imported_memories,
5086 offset,
5087 heap_access_oob,
5088 unaligned_atomic,
5089 |this, addr| this.emit_relaxed_store(Size::S16, value, Location::Memory(addr, 0)),
5090 )?;
5091 self.assembler.emit_rwfence()
5092 }
5093
5094 fn i64_atomic_save_32(
5095 &mut self,
5096 value: Location,
5097 memarg: &MemArg,
5098 addr: Location,
5099 _need_check: bool,
5100 imported_memories: bool,
5101 offset: i32,
5102 heap_access_oob: Label,
5103 unaligned_atomic: Label,
5104 ) -> Result<(), CompileError> {
5105 self.memory_op(
5106 addr,
5107 memarg,
5108 true,
5109 4,
5110 imported_memories,
5111 offset,
5112 heap_access_oob,
5113 unaligned_atomic,
5114 |this, addr| this.emit_relaxed_store(Size::S32, value, Location::Memory(addr, 0)),
5115 )?;
5116 self.assembler.emit_rwfence()
5117 }
5118
5119 fn i64_atomic_add(
5120 &mut self,
5121 loc: Location,
5122 target: Location,
5123 memarg: &MemArg,
5124 ret: Location,
5125 _need_check: bool,
5126 imported_memories: bool,
5127 offset: i32,
5128 heap_access_oob: Label,
5129 unaligned_atomic: Label,
5130 ) -> Result<(), CompileError> {
5131 self.memory_op(
5132 target,
5133 memarg,
5134 true,
5135 8,
5136 imported_memories,
5137 offset,
5138 heap_access_oob,
5139 unaligned_atomic,
5140 |this, addr| {
5141 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Add, Size::S64, ret, addr, loc)
5142 },
5143 )
5144 }
5145
5146 fn i64_atomic_add_8u(
5147 &mut self,
5148 loc: Location,
5149 target: Location,
5150 memarg: &MemArg,
5151 ret: Location,
5152 _need_check: bool,
5153 imported_memories: bool,
5154 offset: i32,
5155 heap_access_oob: Label,
5156 unaligned_atomic: Label,
5157 ) -> Result<(), CompileError> {
5158 self.memory_op(
5159 target,
5160 memarg,
5161 true,
5162 1,
5163 imported_memories,
5164 offset,
5165 heap_access_oob,
5166 unaligned_atomic,
5167 |this, addr| {
5168 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Add, Size::S8, ret, addr, loc)
5169 },
5170 )
5171 }
5172
5173 fn i64_atomic_add_16u(
5174 &mut self,
5175 loc: Location,
5176 target: Location,
5177 memarg: &MemArg,
5178 ret: Location,
5179 _need_check: bool,
5180 imported_memories: bool,
5181 offset: i32,
5182 heap_access_oob: Label,
5183 unaligned_atomic: Label,
5184 ) -> Result<(), CompileError> {
5185 self.memory_op(
5186 target,
5187 memarg,
5188 true,
5189 2,
5190 imported_memories,
5191 offset,
5192 heap_access_oob,
5193 unaligned_atomic,
5194 |this, addr| {
5195 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Add, Size::S16, ret, addr, loc)
5196 },
5197 )
5198 }
5199
5200 fn i64_atomic_add_32u(
5201 &mut self,
5202 loc: Location,
5203 target: Location,
5204 memarg: &MemArg,
5205 ret: Location,
5206 _need_check: bool,
5207 imported_memories: bool,
5208 offset: i32,
5209 heap_access_oob: Label,
5210 unaligned_atomic: Label,
5211 ) -> Result<(), CompileError> {
5212 self.memory_op(
5213 target,
5214 memarg,
5215 true,
5216 4,
5217 imported_memories,
5218 offset,
5219 heap_access_oob,
5220 unaligned_atomic,
5221 |this, addr| {
5222 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Add, Size::S32, ret, addr, loc)
5223 },
5224 )
5225 }
5226
5227 fn i64_atomic_sub(
5228 &mut self,
5229 loc: Location,
5230 target: Location,
5231 memarg: &MemArg,
5232 ret: Location,
5233 _need_check: bool,
5234 imported_memories: bool,
5235 offset: i32,
5236 heap_access_oob: Label,
5237 unaligned_atomic: Label,
5238 ) -> Result<(), CompileError> {
5239 self.memory_op(
5240 target,
5241 memarg,
5242 true,
5243 8,
5244 imported_memories,
5245 offset,
5246 heap_access_oob,
5247 unaligned_atomic,
5248 |this, addr| {
5249 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Sub, Size::S64, ret, addr, loc)
5250 },
5251 )
5252 }
5253
5254 fn i64_atomic_sub_8u(
5255 &mut self,
5256 loc: Location,
5257 target: Location,
5258 memarg: &MemArg,
5259 ret: Location,
5260 _need_check: bool,
5261 imported_memories: bool,
5262 offset: i32,
5263 heap_access_oob: Label,
5264 unaligned_atomic: Label,
5265 ) -> Result<(), CompileError> {
5266 self.memory_op(
5267 target,
5268 memarg,
5269 true,
5270 1,
5271 imported_memories,
5272 offset,
5273 heap_access_oob,
5274 unaligned_atomic,
5275 |this, addr| {
5276 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Sub, Size::S8, ret, addr, loc)
5277 },
5278 )
5279 }
5280
5281 fn i64_atomic_sub_16u(
5282 &mut self,
5283 loc: Location,
5284 target: Location,
5285 memarg: &MemArg,
5286 ret: Location,
5287 _need_check: bool,
5288 imported_memories: bool,
5289 offset: i32,
5290 heap_access_oob: Label,
5291 unaligned_atomic: Label,
5292 ) -> Result<(), CompileError> {
5293 self.memory_op(
5294 target,
5295 memarg,
5296 true,
5297 2,
5298 imported_memories,
5299 offset,
5300 heap_access_oob,
5301 unaligned_atomic,
5302 |this, addr| {
5303 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Sub, Size::S16, ret, addr, loc)
5304 },
5305 )
5306 }
5307
5308 fn i64_atomic_sub_32u(
5309 &mut self,
5310 loc: Location,
5311 target: Location,
5312 memarg: &MemArg,
5313 ret: Location,
5314 _need_check: bool,
5315 imported_memories: bool,
5316 offset: i32,
5317 heap_access_oob: Label,
5318 unaligned_atomic: Label,
5319 ) -> Result<(), CompileError> {
5320 self.memory_op(
5321 target,
5322 memarg,
5323 true,
5324 4,
5325 imported_memories,
5326 offset,
5327 heap_access_oob,
5328 unaligned_atomic,
5329 |this, addr| {
5330 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Sub, Size::S32, ret, addr, loc)
5331 },
5332 )
5333 }
5334
5335 fn i64_atomic_and(
5336 &mut self,
5337 loc: Location,
5338 target: Location,
5339 memarg: &MemArg,
5340 ret: Location,
5341 _need_check: bool,
5342 imported_memories: bool,
5343 offset: i32,
5344 heap_access_oob: Label,
5345 unaligned_atomic: Label,
5346 ) -> Result<(), CompileError> {
5347 self.memory_op(
5348 target,
5349 memarg,
5350 true,
5351 8,
5352 imported_memories,
5353 offset,
5354 heap_access_oob,
5355 unaligned_atomic,
5356 |this, addr| {
5357 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::And, Size::S64, ret, addr, loc)
5358 },
5359 )
5360 }
5361
5362 fn i64_atomic_and_8u(
5363 &mut self,
5364 loc: Location,
5365 target: Location,
5366 memarg: &MemArg,
5367 ret: Location,
5368 _need_check: bool,
5369 imported_memories: bool,
5370 offset: i32,
5371 heap_access_oob: Label,
5372 unaligned_atomic: Label,
5373 ) -> Result<(), CompileError> {
5374 self.memory_op(
5375 target,
5376 memarg,
5377 true,
5378 1,
5379 imported_memories,
5380 offset,
5381 heap_access_oob,
5382 unaligned_atomic,
5383 |this, addr| {
5384 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::And, Size::S8, ret, addr, loc)
5385 },
5386 )
5387 }
5388
5389 fn i64_atomic_and_16u(
5390 &mut self,
5391 loc: Location,
5392 target: Location,
5393 memarg: &MemArg,
5394 ret: Location,
5395 _need_check: bool,
5396 imported_memories: bool,
5397 offset: i32,
5398 heap_access_oob: Label,
5399 unaligned_atomic: Label,
5400 ) -> Result<(), CompileError> {
5401 self.memory_op(
5402 target,
5403 memarg,
5404 true,
5405 2,
5406 imported_memories,
5407 offset,
5408 heap_access_oob,
5409 unaligned_atomic,
5410 |this, addr| {
5411 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::And, Size::S16, ret, addr, loc)
5412 },
5413 )
5414 }
5415
5416 fn i64_atomic_and_32u(
5417 &mut self,
5418 loc: Location,
5419 target: Location,
5420 memarg: &MemArg,
5421 ret: Location,
5422 _need_check: bool,
5423 imported_memories: bool,
5424 offset: i32,
5425 heap_access_oob: Label,
5426 unaligned_atomic: Label,
5427 ) -> Result<(), CompileError> {
5428 self.memory_op(
5429 target,
5430 memarg,
5431 true,
5432 4,
5433 imported_memories,
5434 offset,
5435 heap_access_oob,
5436 unaligned_atomic,
5437 |this, addr| {
5438 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::And, Size::S32, ret, addr, loc)
5439 },
5440 )
5441 }
5442
5443 fn i64_atomic_or(
5444 &mut self,
5445 loc: Location,
5446 target: Location,
5447 memarg: &MemArg,
5448 ret: Location,
5449 _need_check: bool,
5450 imported_memories: bool,
5451 offset: i32,
5452 heap_access_oob: Label,
5453 unaligned_atomic: Label,
5454 ) -> Result<(), CompileError> {
5455 self.memory_op(
5456 target,
5457 memarg,
5458 true,
5459 8,
5460 imported_memories,
5461 offset,
5462 heap_access_oob,
5463 unaligned_atomic,
5464 |this, addr| {
5465 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Or, Size::S64, ret, addr, loc)
5466 },
5467 )
5468 }
5469
5470 fn i64_atomic_or_8u(
5471 &mut self,
5472 loc: Location,
5473 target: Location,
5474 memarg: &MemArg,
5475 ret: Location,
5476 _need_check: bool,
5477 imported_memories: bool,
5478 offset: i32,
5479 heap_access_oob: Label,
5480 unaligned_atomic: Label,
5481 ) -> Result<(), CompileError> {
5482 self.memory_op(
5483 target,
5484 memarg,
5485 true,
5486 1,
5487 imported_memories,
5488 offset,
5489 heap_access_oob,
5490 unaligned_atomic,
5491 |this, addr| {
5492 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Or, Size::S8, ret, addr, loc)
5493 },
5494 )
5495 }
5496
5497 fn i64_atomic_or_16u(
5498 &mut self,
5499 loc: Location,
5500 target: Location,
5501 memarg: &MemArg,
5502 ret: Location,
5503 _need_check: bool,
5504 imported_memories: bool,
5505 offset: i32,
5506 heap_access_oob: Label,
5507 unaligned_atomic: Label,
5508 ) -> Result<(), CompileError> {
5509 self.memory_op(
5510 target,
5511 memarg,
5512 true,
5513 2,
5514 imported_memories,
5515 offset,
5516 heap_access_oob,
5517 unaligned_atomic,
5518 |this, addr| {
5519 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Or, Size::S16, ret, addr, loc)
5520 },
5521 )
5522 }
5523
5524 fn i64_atomic_or_32u(
5525 &mut self,
5526 loc: Location,
5527 target: Location,
5528 memarg: &MemArg,
5529 ret: Location,
5530 _need_check: bool,
5531 imported_memories: bool,
5532 offset: i32,
5533 heap_access_oob: Label,
5534 unaligned_atomic: Label,
5535 ) -> Result<(), CompileError> {
5536 self.memory_op(
5537 target,
5538 memarg,
5539 true,
5540 4,
5541 imported_memories,
5542 offset,
5543 heap_access_oob,
5544 unaligned_atomic,
5545 |this, addr| {
5546 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Or, Size::S32, ret, addr, loc)
5547 },
5548 )
5549 }
5550
5551 fn i64_atomic_xor(
5552 &mut self,
5553 loc: Location,
5554 target: Location,
5555 memarg: &MemArg,
5556 ret: Location,
5557 _need_check: bool,
5558 imported_memories: bool,
5559 offset: i32,
5560 heap_access_oob: Label,
5561 unaligned_atomic: Label,
5562 ) -> Result<(), CompileError> {
5563 self.memory_op(
5564 target,
5565 memarg,
5566 true,
5567 8,
5568 imported_memories,
5569 offset,
5570 heap_access_oob,
5571 unaligned_atomic,
5572 |this, addr| {
5573 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Xor, Size::S64, ret, addr, loc)
5574 },
5575 )
5576 }
5577
5578 fn i64_atomic_xor_8u(
5579 &mut self,
5580 loc: Location,
5581 target: Location,
5582 memarg: &MemArg,
5583 ret: Location,
5584 _need_check: bool,
5585 imported_memories: bool,
5586 offset: i32,
5587 heap_access_oob: Label,
5588 unaligned_atomic: Label,
5589 ) -> Result<(), CompileError> {
5590 self.memory_op(
5591 target,
5592 memarg,
5593 true,
5594 1,
5595 imported_memories,
5596 offset,
5597 heap_access_oob,
5598 unaligned_atomic,
5599 |this, addr| {
5600 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Xor, Size::S8, ret, addr, loc)
5601 },
5602 )
5603 }
5604
5605 fn i64_atomic_xor_16u(
5606 &mut self,
5607 loc: Location,
5608 target: Location,
5609 memarg: &MemArg,
5610 ret: Location,
5611 _need_check: bool,
5612 imported_memories: bool,
5613 offset: i32,
5614 heap_access_oob: Label,
5615 unaligned_atomic: Label,
5616 ) -> Result<(), CompileError> {
5617 self.memory_op(
5618 target,
5619 memarg,
5620 true,
5621 2,
5622 imported_memories,
5623 offset,
5624 heap_access_oob,
5625 unaligned_atomic,
5626 |this, addr| {
5627 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Xor, Size::S16, ret, addr, loc)
5628 },
5629 )
5630 }
5631
5632 fn i64_atomic_xor_32u(
5633 &mut self,
5634 loc: Location,
5635 target: Location,
5636 memarg: &MemArg,
5637 ret: Location,
5638 _need_check: bool,
5639 imported_memories: bool,
5640 offset: i32,
5641 heap_access_oob: Label,
5642 unaligned_atomic: Label,
5643 ) -> Result<(), CompileError> {
5644 self.memory_op(
5645 target,
5646 memarg,
5647 true,
5648 4,
5649 imported_memories,
5650 offset,
5651 heap_access_oob,
5652 unaligned_atomic,
5653 |this, addr| {
5654 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Xor, Size::S32, ret, addr, loc)
5655 },
5656 )
5657 }
5658
5659 fn i64_atomic_xchg(
5660 &mut self,
5661 loc: Location,
5662 target: Location,
5663 memarg: &MemArg,
5664 ret: Location,
5665 _need_check: bool,
5666 imported_memories: bool,
5667 offset: i32,
5668 heap_access_oob: Label,
5669 unaligned_atomic: Label,
5670 ) -> Result<(), CompileError> {
5671 self.memory_op(
5672 target,
5673 memarg,
5674 true,
5675 8,
5676 imported_memories,
5677 offset,
5678 heap_access_oob,
5679 unaligned_atomic,
5680 |this, addr| {
5681 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Exchange, Size::S64, ret, addr, loc)
5682 },
5683 )
5684 }
5685
5686 fn i64_atomic_xchg_8u(
5687 &mut self,
5688 loc: Location,
5689 target: Location,
5690 memarg: &MemArg,
5691 ret: Location,
5692 _need_check: bool,
5693 imported_memories: bool,
5694 offset: i32,
5695 heap_access_oob: Label,
5696 unaligned_atomic: Label,
5697 ) -> Result<(), CompileError> {
5698 self.memory_op(
5699 target,
5700 memarg,
5701 true,
5702 1,
5703 imported_memories,
5704 offset,
5705 heap_access_oob,
5706 unaligned_atomic,
5707 |this, addr| {
5708 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Exchange, Size::S8, ret, addr, loc)
5709 },
5710 )
5711 }
5712
5713 fn i64_atomic_xchg_16u(
5714 &mut self,
5715 loc: Location,
5716 target: Location,
5717 memarg: &MemArg,
5718 ret: Location,
5719 _need_check: bool,
5720 imported_memories: bool,
5721 offset: i32,
5722 heap_access_oob: Label,
5723 unaligned_atomic: Label,
5724 ) -> Result<(), CompileError> {
5725 self.memory_op(
5726 target,
5727 memarg,
5728 true,
5729 2,
5730 imported_memories,
5731 offset,
5732 heap_access_oob,
5733 unaligned_atomic,
5734 |this, addr| {
5735 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Exchange, Size::S16, ret, addr, loc)
5736 },
5737 )
5738 }
5739
5740 fn i64_atomic_xchg_32u(
5741 &mut self,
5742 loc: Location,
5743 target: Location,
5744 memarg: &MemArg,
5745 ret: Location,
5746 _need_check: bool,
5747 imported_memories: bool,
5748 offset: i32,
5749 heap_access_oob: Label,
5750 unaligned_atomic: Label,
5751 ) -> Result<(), CompileError> {
5752 self.memory_op(
5753 target,
5754 memarg,
5755 true,
5756 4,
5757 imported_memories,
5758 offset,
5759 heap_access_oob,
5760 unaligned_atomic,
5761 |this, addr| {
5762 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Exchange, Size::S32, ret, addr, loc)
5763 },
5764 )
5765 }
5766
5767 fn i64_atomic_cmpxchg(
5768 &mut self,
5769 new: Location,
5770 cmp: Location,
5771 target: Location,
5772 memarg: &MemArg,
5773 ret: Location,
5774 _need_check: bool,
5775 imported_memories: bool,
5776 offset: i32,
5777 heap_access_oob: Label,
5778 unaligned_atomic: Label,
5779 ) -> Result<(), CompileError> {
5780 self.memory_op(
5781 target,
5782 memarg,
5783 true,
5784 8,
5785 imported_memories,
5786 offset,
5787 heap_access_oob,
5788 unaligned_atomic,
5789 |this, addr| this.emit_relaxed_atomic_cmpxchg(Size::S64, ret, addr, new, cmp),
5790 )
5791 }
5792
5793 fn i64_atomic_cmpxchg_8u(
5794 &mut self,
5795 new: Location,
5796 cmp: Location,
5797 target: Location,
5798 memarg: &MemArg,
5799 ret: Location,
5800 _need_check: bool,
5801 imported_memories: bool,
5802 offset: i32,
5803 heap_access_oob: Label,
5804 unaligned_atomic: Label,
5805 ) -> Result<(), CompileError> {
5806 self.memory_op(
5807 target,
5808 memarg,
5809 true,
5810 1,
5811 imported_memories,
5812 offset,
5813 heap_access_oob,
5814 unaligned_atomic,
5815 |this, addr| this.emit_relaxed_atomic_cmpxchg(Size::S8, ret, addr, new, cmp),
5816 )
5817 }
5818
5819 fn i64_atomic_cmpxchg_16u(
5820 &mut self,
5821 new: Location,
5822 cmp: Location,
5823 target: Location,
5824 memarg: &MemArg,
5825 ret: Location,
5826 _need_check: bool,
5827 imported_memories: bool,
5828 offset: i32,
5829 heap_access_oob: Label,
5830 unaligned_atomic: Label,
5831 ) -> Result<(), CompileError> {
5832 self.memory_op(
5833 target,
5834 memarg,
5835 true,
5836 2,
5837 imported_memories,
5838 offset,
5839 heap_access_oob,
5840 unaligned_atomic,
5841 |this, addr| this.emit_relaxed_atomic_cmpxchg(Size::S16, ret, addr, new, cmp),
5842 )
5843 }
5844
5845 fn i64_atomic_cmpxchg_32u(
5846 &mut self,
5847 new: Location,
5848 cmp: Location,
5849 target: Location,
5850 memarg: &MemArg,
5851 ret: Location,
5852 _need_check: bool,
5853 imported_memories: bool,
5854 offset: i32,
5855 heap_access_oob: Label,
5856 unaligned_atomic: Label,
5857 ) -> Result<(), CompileError> {
5858 self.memory_op(
5859 target,
5860 memarg,
5861 true,
5862 4,
5863 imported_memories,
5864 offset,
5865 heap_access_oob,
5866 unaligned_atomic,
5867 |this, addr| this.emit_relaxed_atomic_cmpxchg(Size::S32, ret, addr, new, cmp),
5868 )
5869 }
5870
5871 fn f32_load(
5872 &mut self,
5873 addr: Location,
5874 memarg: &MemArg,
5875 ret: Location,
5876 _need_check: bool,
5877 imported_memories: bool,
5878 offset: i32,
5879 heap_access_oob: Label,
5880 unaligned_atomic: Label,
5881 ) -> Result<(), CompileError> {
5882 self.memory_op(
5883 addr,
5884 memarg,
5885 false,
5886 4,
5887 imported_memories,
5888 offset,
5889 heap_access_oob,
5890 unaligned_atomic,
5891 |this, addr| this.emit_relaxed_load(Size::S32, false, ret, Location::Memory(addr, 0)),
5892 )
5893 }
5894
5895 fn f32_save(
5896 &mut self,
5897 value: Location,
5898 memarg: &MemArg,
5899 addr: Location,
5900 canonicalize: bool,
5901 _need_check: bool,
5902 imported_memories: bool,
5903 offset: i32,
5904 heap_access_oob: Label,
5905 unaligned_atomic: Label,
5906 ) -> Result<(), CompileError> {
5907 self.memory_op(
5908 addr,
5909 memarg,
5910 false,
5911 4,
5912 imported_memories,
5913 offset,
5914 heap_access_oob,
5915 unaligned_atomic,
5916 |this, addr| {
5917 if !canonicalize {
5918 this.emit_relaxed_store(Size::S32, value, Location::Memory(addr, 0))
5919 } else {
5920 this.canonicalize_nan(Size::S32, value, Location::Memory(addr, 0))
5921 }
5922 },
5923 )
5924 }
5925
5926 fn f64_load(
5927 &mut self,
5928 addr: Location,
5929 memarg: &MemArg,
5930 ret: Location,
5931 _need_check: bool,
5932 imported_memories: bool,
5933 offset: i32,
5934 heap_access_oob: Label,
5935 unaligned_atomic: Label,
5936 ) -> Result<(), CompileError> {
5937 self.memory_op(
5938 addr,
5939 memarg,
5940 false,
5941 8,
5942 imported_memories,
5943 offset,
5944 heap_access_oob,
5945 unaligned_atomic,
5946 |this, addr| this.emit_relaxed_load(Size::S64, false, ret, Location::Memory(addr, 0)),
5947 )
5948 }
5949
5950 fn f64_save(
5951 &mut self,
5952 value: Location,
5953 memarg: &MemArg,
5954 addr: Location,
5955 canonicalize: bool,
5956 _need_check: bool,
5957 imported_memories: bool,
5958 offset: i32,
5959 heap_access_oob: Label,
5960 unaligned_atomic: Label,
5961 ) -> Result<(), CompileError> {
5962 self.memory_op(
5963 addr,
5964 memarg,
5965 false,
5966 8,
5967 imported_memories,
5968 offset,
5969 heap_access_oob,
5970 unaligned_atomic,
5971 |this, addr| {
5972 if !canonicalize {
5973 this.emit_relaxed_store(Size::S64, value, Location::Memory(addr, 0))
5974 } else {
5975 this.canonicalize_nan(Size::S64, value, Location::Memory(addr, 0))
5976 }
5977 },
5978 )
5979 }
5980
5981 fn convert_f64_i64(
5982 &mut self,
5983 loc: Location,
5984 signed: bool,
5985 ret: Location,
5986 ) -> Result<(), CompileError> {
5987 self.convert_int_to_float(loc, Size::S64, ret, Size::S64, signed)
5988 }
5989
5990 fn convert_f64_i32(
5991 &mut self,
5992 loc: Location,
5993 signed: bool,
5994 ret: Location,
5995 ) -> Result<(), CompileError> {
5996 self.convert_int_to_float(loc, Size::S32, ret, Size::S64, signed)
5997 }
5998
5999 fn convert_f32_i64(
6000 &mut self,
6001 loc: Location,
6002 signed: bool,
6003 ret: Location,
6004 ) -> Result<(), CompileError> {
6005 self.convert_int_to_float(loc, Size::S64, ret, Size::S32, signed)
6006 }
6007
6008 fn convert_f32_i32(
6009 &mut self,
6010 loc: Location,
6011 signed: bool,
6012 ret: Location,
6013 ) -> Result<(), CompileError> {
6014 self.convert_int_to_float(loc, Size::S32, ret, Size::S32, signed)
6015 }
6016
6017 fn convert_i64_f64(
6018 &mut self,
6019 loc: Location,
6020 ret: Location,
6021 signed: bool,
6022 sat: bool,
6023 ) -> Result<(), CompileError> {
6024 self.convert_float_to_int(loc, Size::S64, ret, Size::S64, signed, sat)
6025 }
6026
6027 fn convert_i32_f64(
6028 &mut self,
6029 loc: Location,
6030 ret: Location,
6031 signed: bool,
6032 sat: bool,
6033 ) -> Result<(), CompileError> {
6034 self.convert_float_to_int(loc, Size::S64, ret, Size::S32, signed, sat)
6035 }
6036
6037 fn convert_i64_f32(
6038 &mut self,
6039 loc: Location,
6040 ret: Location,
6041 signed: bool,
6042 sat: bool,
6043 ) -> Result<(), CompileError> {
6044 self.convert_float_to_int(loc, Size::S32, ret, Size::S64, signed, sat)
6045 }
6046
6047 fn convert_i32_f32(
6048 &mut self,
6049 loc: Location,
6050 ret: Location,
6051 signed: bool,
6052 sat: bool,
6053 ) -> Result<(), CompileError> {
6054 self.convert_float_to_int(loc, Size::S32, ret, Size::S32, signed, sat)
6055 }
6056
6057 fn convert_f64_f32(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6058 self.convert_float_to_float(loc, Size::S32, ret, Size::S64)
6059 }
6060
6061 fn convert_f32_f64(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6062 self.convert_float_to_float(loc, Size::S64, ret, Size::S32)
6063 }
6064
6065 fn f64_neg(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6066 self.emit_relaxed_binop_fp(Assembler::emit_fneg, Size::S64, loc, ret, true)
6067 }
6068
6069 fn f64_abs(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6070 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
6071 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6072 })?;
6073 let mask = self.acquire_temp_gpr().ok_or_else(|| {
6074 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6075 })?;
6076
6077 self.move_location(Size::S64, loc, Location::GPR(tmp))?;
6078 self.assembler
6079 .emit_mov_imm(Location::GPR(mask), 0x7fffffffffffffffi64)?;
6080 self.assembler.emit_and(
6081 Size::S64,
6082 Location::GPR(tmp),
6083 Location::GPR(mask),
6084 Location::GPR(tmp),
6085 )?;
6086 self.move_location(Size::S64, Location::GPR(tmp), ret)?;
6087
6088 self.release_gpr(tmp);
6089 self.release_gpr(mask);
6090 Ok(())
6091 }
6092
6093 fn emit_i64_copysign(&mut self, tmp1: Self::GPR, tmp2: Self::GPR) -> Result<(), CompileError> {
6094 let mask = self.acquire_temp_gpr().ok_or_else(|| {
6095 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6096 })?;
6097
6098 self.assembler
6099 .emit_mov_imm(Location::GPR(mask), 0x7fffffffffffffffu64 as _)?;
6100 self.assembler.emit_and(
6101 Size::S64,
6102 Location::GPR(tmp1),
6103 Location::GPR(mask),
6104 Location::GPR(tmp1),
6105 )?;
6106
6107 self.assembler
6108 .emit_mov_imm(Location::GPR(mask), 0x8000000000000000u64 as _)?;
6109 self.assembler.emit_and(
6110 Size::S64,
6111 Location::GPR(tmp2),
6112 Location::GPR(mask),
6113 Location::GPR(tmp2),
6114 )?;
6115
6116 self.release_gpr(mask);
6117 self.assembler.emit_or(
6118 Size::S64,
6119 Location::GPR(tmp1),
6120 Location::GPR(tmp2),
6121 Location::GPR(tmp1),
6122 )
6123 }
6124
6125 fn f64_sqrt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6126 self.emit_relaxed_binop_fp(Assembler::emit_fsqrt, Size::S64, loc, ret, true)
6127 }
6128
6129 fn f64_trunc(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6130 self.emit_relaxed_fcvt_with_rounding(RoundingMode::Rtz, Size::S64, loc, ret)
6131 }
6132
6133 fn f64_ceil(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6134 self.emit_relaxed_fcvt_with_rounding(RoundingMode::Rup, Size::S64, loc, ret)
6135 }
6136
6137 fn f64_floor(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6138 self.emit_relaxed_fcvt_with_rounding(RoundingMode::Rdn, Size::S64, loc, ret)
6139 }
6140
6141 fn f64_nearest(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6142 self.emit_relaxed_fcvt_with_rounding(RoundingMode::Rne, Size::S64, loc, ret)
6143 }
6144
6145 fn f64_cmp_ge(
6146 &mut self,
6147 loc_a: Location,
6148 loc_b: Location,
6149 ret: Location,
6150 ) -> Result<(), CompileError> {
6151 self.emit_relaxed_fcmp(Condition::Ge, Size::S64, loc_a, loc_b, ret)
6152 }
6153
6154 fn f64_cmp_gt(
6155 &mut self,
6156 loc_a: Location,
6157 loc_b: Location,
6158 ret: Location,
6159 ) -> Result<(), CompileError> {
6160 self.emit_relaxed_fcmp(Condition::Gt, Size::S64, loc_a, loc_b, ret)
6161 }
6162
6163 fn f64_cmp_le(
6164 &mut self,
6165 loc_a: Location,
6166 loc_b: Location,
6167 ret: Location,
6168 ) -> Result<(), CompileError> {
6169 self.emit_relaxed_fcmp(Condition::Le, Size::S64, loc_a, loc_b, ret)
6170 }
6171
6172 fn f64_cmp_lt(
6173 &mut self,
6174 loc_a: Location,
6175 loc_b: Location,
6176 ret: Location,
6177 ) -> Result<(), CompileError> {
6178 self.emit_relaxed_fcmp(Condition::Lt, Size::S64, loc_a, loc_b, ret)
6179 }
6180
6181 fn f64_cmp_ne(
6182 &mut self,
6183 loc_a: Location,
6184 loc_b: Location,
6185 ret: Location,
6186 ) -> Result<(), CompileError> {
6187 self.emit_relaxed_fcmp(Condition::Ne, Size::S64, loc_a, loc_b, ret)
6188 }
6189
6190 fn f64_cmp_eq(
6191 &mut self,
6192 loc_a: Location,
6193 loc_b: Location,
6194 ret: Location,
6195 ) -> Result<(), CompileError> {
6196 self.emit_relaxed_fcmp(Condition::Eq, Size::S64, loc_a, loc_b, ret)
6197 }
6198
6199 fn f64_min(
6200 &mut self,
6201 loc_a: Location,
6202 loc_b: Location,
6203 ret: Location,
6204 ) -> Result<(), CompileError> {
6205 self.emit_relaxed_binop3_fp(
6206 Assembler::emit_fmin,
6207 Size::S64,
6208 loc_a,
6209 loc_b,
6210 ret,
6211 ImmType::None,
6212 true,
6213 )
6214 }
6215
6216 fn f64_max(
6217 &mut self,
6218 loc_a: Location,
6219 loc_b: Location,
6220 ret: Location,
6221 ) -> Result<(), CompileError> {
6222 self.emit_relaxed_binop3_fp(
6223 Assembler::emit_fmax,
6224 Size::S64,
6225 loc_a,
6226 loc_b,
6227 ret,
6228 ImmType::None,
6229 true,
6230 )
6231 }
6232
6233 fn f64_add(
6234 &mut self,
6235 loc_a: Location,
6236 loc_b: Location,
6237 ret: Location,
6238 ) -> Result<(), CompileError> {
6239 self.emit_relaxed_binop3_fp(
6240 Assembler::emit_add,
6241 Size::S64,
6242 loc_a,
6243 loc_b,
6244 ret,
6245 ImmType::None,
6246 false,
6247 )
6248 }
6249
6250 fn f64_sub(
6251 &mut self,
6252 loc_a: Location,
6253 loc_b: Location,
6254 ret: Location,
6255 ) -> Result<(), CompileError> {
6256 self.emit_relaxed_binop3_fp(
6257 Assembler::emit_sub,
6258 Size::S64,
6259 loc_a,
6260 loc_b,
6261 ret,
6262 ImmType::None,
6263 false,
6264 )
6265 }
6266
6267 fn f64_mul(
6268 &mut self,
6269 loc_a: Location,
6270 loc_b: Location,
6271 ret: Location,
6272 ) -> Result<(), CompileError> {
6273 self.emit_relaxed_binop3_fp(
6274 Assembler::emit_mul,
6275 Size::S64,
6276 loc_a,
6277 loc_b,
6278 ret,
6279 ImmType::None,
6280 false,
6281 )
6282 }
6283
6284 fn f64_div(
6285 &mut self,
6286 loc_a: Location,
6287 loc_b: Location,
6288 ret: Location,
6289 ) -> Result<(), CompileError> {
6290 self.emit_relaxed_binop3_fp(
6291 Assembler::emit_fdiv,
6292 Size::S64,
6293 loc_a,
6294 loc_b,
6295 ret,
6296 ImmType::None,
6297 false,
6298 )
6299 }
6300
6301 fn f32_neg(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6302 self.emit_relaxed_binop_fp(Assembler::emit_fneg, Size::S32, loc, ret, true)
6303 }
6304
6305 fn f32_abs(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6306 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
6307 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6308 })?;
6309 let mask = self.acquire_temp_gpr().ok_or_else(|| {
6310 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6311 })?;
6312
6313 self.move_location(Size::S32, loc, Location::GPR(tmp))?;
6314 self.assembler
6315 .emit_mov_imm(Location::GPR(mask), 0x7fffffffi64)?;
6316 self.assembler.emit_and(
6317 Size::S32,
6318 Location::GPR(tmp),
6319 Location::GPR(mask),
6320 Location::GPR(tmp),
6321 )?;
6322 self.move_location(Size::S32, Location::GPR(tmp), ret)?;
6323
6324 self.release_gpr(tmp);
6325 self.release_gpr(mask);
6326 Ok(())
6327 }
6328
6329 fn emit_i32_copysign(&mut self, tmp1: Self::GPR, tmp2: Self::GPR) -> Result<(), CompileError> {
6330 let mask = self.acquire_temp_gpr().ok_or_else(|| {
6331 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6332 })?;
6333
6334 self.assembler
6335 .emit_mov_imm(Location::GPR(mask), 0x7fffffffu32 as _)?;
6336 self.assembler.emit_and(
6337 Size::S32,
6338 Location::GPR(tmp1),
6339 Location::GPR(mask),
6340 Location::GPR(tmp1),
6341 )?;
6342
6343 self.assembler
6344 .emit_mov_imm(Location::GPR(mask), 0x80000000u32 as _)?;
6345 self.assembler.emit_and(
6346 Size::S32,
6347 Location::GPR(tmp2),
6348 Location::GPR(mask),
6349 Location::GPR(tmp2),
6350 )?;
6351
6352 self.release_gpr(mask);
6353 self.assembler.emit_or(
6354 Size::S32,
6355 Location::GPR(tmp1),
6356 Location::GPR(tmp2),
6357 Location::GPR(tmp1),
6358 )
6359 }
6360
6361 fn f32_sqrt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6362 self.emit_relaxed_binop_fp(Assembler::emit_fsqrt, Size::S32, loc, ret, true)
6363 }
6364
6365 fn f32_trunc(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6366 self.emit_relaxed_fcvt_with_rounding(RoundingMode::Rtz, Size::S32, loc, ret)
6367 }
6368
6369 fn f32_ceil(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6370 self.emit_relaxed_fcvt_with_rounding(RoundingMode::Rup, Size::S32, loc, ret)
6371 }
6372
6373 fn f32_floor(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6374 self.emit_relaxed_fcvt_with_rounding(RoundingMode::Rdn, Size::S32, loc, ret)
6375 }
6376
6377 fn f32_nearest(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6378 self.emit_relaxed_fcvt_with_rounding(RoundingMode::Rne, Size::S32, loc, ret)
6379 }
6380
6381 fn f32_cmp_ge(
6382 &mut self,
6383 loc_a: Location,
6384 loc_b: Location,
6385 ret: Location,
6386 ) -> Result<(), CompileError> {
6387 self.emit_relaxed_fcmp(Condition::Ge, Size::S32, loc_a, loc_b, ret)
6388 }
6389
6390 fn f32_cmp_gt(
6391 &mut self,
6392 loc_a: Location,
6393 loc_b: Location,
6394 ret: Location,
6395 ) -> Result<(), CompileError> {
6396 self.emit_relaxed_fcmp(Condition::Gt, Size::S32, loc_a, loc_b, ret)
6397 }
6398
6399 fn f32_cmp_le(
6400 &mut self,
6401 loc_a: Location,
6402 loc_b: Location,
6403 ret: Location,
6404 ) -> Result<(), CompileError> {
6405 self.emit_relaxed_fcmp(Condition::Le, Size::S32, loc_a, loc_b, ret)
6406 }
6407
6408 fn f32_cmp_lt(
6409 &mut self,
6410 loc_a: Location,
6411 loc_b: Location,
6412 ret: Location,
6413 ) -> Result<(), CompileError> {
6414 self.emit_relaxed_fcmp(Condition::Lt, Size::S32, loc_a, loc_b, ret)
6415 }
6416
6417 fn f32_cmp_ne(
6418 &mut self,
6419 loc_a: Location,
6420 loc_b: Location,
6421 ret: Location,
6422 ) -> Result<(), CompileError> {
6423 self.emit_relaxed_fcmp(Condition::Ne, Size::S32, loc_a, loc_b, ret)
6424 }
6425
6426 fn f32_cmp_eq(
6427 &mut self,
6428 loc_a: Location,
6429 loc_b: Location,
6430 ret: Location,
6431 ) -> Result<(), CompileError> {
6432 self.emit_relaxed_fcmp(Condition::Eq, Size::S32, loc_a, loc_b, ret)
6433 }
6434
6435 fn f32_min(
6436 &mut self,
6437 loc_a: Location,
6438 loc_b: Location,
6439 ret: Location,
6440 ) -> Result<(), CompileError> {
6441 self.emit_relaxed_binop3_fp(
6442 Assembler::emit_fmin,
6443 Size::S32,
6444 loc_a,
6445 loc_b,
6446 ret,
6447 ImmType::None,
6448 true,
6449 )
6450 }
6451
6452 fn f32_max(
6453 &mut self,
6454 loc_a: Location,
6455 loc_b: Location,
6456 ret: Location,
6457 ) -> Result<(), CompileError> {
6458 self.emit_relaxed_binop3_fp(
6459 Assembler::emit_fmax,
6460 Size::S32,
6461 loc_a,
6462 loc_b,
6463 ret,
6464 ImmType::None,
6465 true,
6466 )
6467 }
6468
6469 fn f32_add(
6470 &mut self,
6471 loc_a: Location,
6472 loc_b: Location,
6473 ret: Location,
6474 ) -> Result<(), CompileError> {
6475 self.emit_relaxed_binop3_fp(
6476 Assembler::emit_add,
6477 Size::S32,
6478 loc_a,
6479 loc_b,
6480 ret,
6481 ImmType::None,
6482 false,
6483 )
6484 }
6485
6486 fn f32_sub(
6487 &mut self,
6488 loc_a: Location,
6489 loc_b: Location,
6490 ret: Location,
6491 ) -> Result<(), CompileError> {
6492 self.emit_relaxed_binop3_fp(
6493 Assembler::emit_sub,
6494 Size::S32,
6495 loc_a,
6496 loc_b,
6497 ret,
6498 ImmType::None,
6499 false,
6500 )
6501 }
6502
6503 fn f32_mul(
6504 &mut self,
6505 loc_a: Location,
6506 loc_b: Location,
6507 ret: Location,
6508 ) -> Result<(), CompileError> {
6509 self.emit_relaxed_binop3_fp(
6510 Assembler::emit_mul,
6511 Size::S32,
6512 loc_a,
6513 loc_b,
6514 ret,
6515 ImmType::None,
6516 false,
6517 )
6518 }
6519
6520 fn f32_div(
6521 &mut self,
6522 loc_a: Location,
6523 loc_b: Location,
6524 ret: Location,
6525 ) -> Result<(), CompileError> {
6526 self.emit_relaxed_binop3_fp(
6527 Assembler::emit_fdiv,
6528 Size::S32,
6529 loc_a,
6530 loc_b,
6531 ret,
6532 ImmType::None,
6533 false,
6534 )
6535 }
6536
6537 fn gen_std_trampoline(
6538 &self,
6539 sig: &FunctionType,
6540 calling_convention: CallingConvention,
6541 ) -> Result<FunctionBody, CompileError> {
6542 gen_std_trampoline_riscv(sig, calling_convention)
6543 }
6544
6545 fn gen_std_dynamic_import_trampoline(
6546 &self,
6547 vmoffsets: &VMOffsets,
6548 sig: &FunctionType,
6549 _calling_convention: CallingConvention,
6550 ) -> Result<FunctionBody, CompileError> {
6551 gen_std_dynamic_import_trampoline_riscv(vmoffsets, sig)
6552 }
6553 fn gen_import_call_trampoline(
6556 &self,
6557 vmoffsets: &VMOffsets,
6558 index: FunctionIndex,
6559 sig: &FunctionType,
6560 calling_convention: CallingConvention,
6561 ) -> Result<CustomSection, CompileError> {
6562 gen_import_call_trampoline_riscv(vmoffsets, index, sig, calling_convention)
6563 }
6564
6565 #[cfg(feature = "unwind")]
6566 fn gen_dwarf_unwind_info(&mut self, code_len: usize) -> Option<UnwindInstructions> {
6567 let mut instructions = vec![];
6568 for &(instruction_offset, ref inst) in &self.unwind_ops {
6569 let instruction_offset = instruction_offset as u32;
6570 match *inst {
6571 UnwindOps::PushFP { up_to_sp } => {
6572 instructions.push((
6573 instruction_offset,
6574 CallFrameInstruction::CfaOffset(up_to_sp as i32),
6575 ));
6576 instructions.push((
6577 instruction_offset,
6578 CallFrameInstruction::Offset(RiscV::FP, -(up_to_sp as i32)),
6579 ));
6580 }
6581 UnwindOps::DefineNewFrame => {
6582 instructions.push((
6583 instruction_offset,
6584 CallFrameInstruction::CfaRegister(RiscV::X8),
6585 ));
6586 }
6587 UnwindOps::SaveRegister { reg, bp_neg_offset } => instructions.push((
6588 instruction_offset,
6589 CallFrameInstruction::Offset(reg.dwarf_index(), -bp_neg_offset),
6590 )),
6591 UnwindOps::SubtractFP { up_to_sp } => {
6592 instructions.push((
6593 instruction_offset,
6594 CallFrameInstruction::CfaOffset(up_to_sp as i32),
6595 ));
6596 }
6597 UnwindOps::Push2Regs { .. } => unimplemented!(),
6598 }
6599 }
6600 Some(UnwindInstructions {
6601 instructions,
6602 len: code_len as u32,
6603 })
6604 }
6605 #[cfg(not(feature = "unwind"))]
6606
6607 fn gen_dwarf_unwind_info(&mut self, _code_len: usize) -> Option<UnwindInstructions> {
6608 None
6609 }
6610
6611 fn gen_windows_unwind_info(&mut self, _code_len: usize) -> Option<Vec<u8>> {
6612 None
6613 }
6614}