1use dynasmrt::{DynasmError, VecAssembler, riscv::RiscvRelocation};
4use fixedbitset::FixedBitSet;
5#[cfg(feature = "unwind")]
6use gimli::{RiscV, write::CallFrameInstruction};
7
8use wasmer_compiler::{
9 CANONICAL_NAN_F32, CANONICAL_NAN_F64,
10 types::{
11 address_map::InstructionAddressMap,
12 function::FunctionBody,
13 relocation::{Relocation, RelocationKind, RelocationTarget},
14 section::CustomSection,
15 },
16 wasmparser::MemArg,
17};
18use wasmer_types::{
19 CompileError, FunctionIndex, FunctionType, SourceLoc, TrapCode, TrapInformation, VMOffsets,
20 target::{CallingConvention, Target},
21};
22
23use crate::{
24 codegen_error,
25 common_decl::*,
26 emitter_riscv::*,
27 location::{Location as AbstractLocation, Reg},
28 machine::*,
29 riscv_decl::{FPR, GPR},
30 unwind::{UnwindInstructions, UnwindOps, UnwindRegister},
31};
32
33type Assembler = VecAssembler<RiscvRelocation>;
34type Location = AbstractLocation<GPR, FPR>;
35
36use std::{
37 collections::HashMap,
38 ops::{Deref, DerefMut},
39};
40pub struct AssemblerRiscv {
42 pub inner: Assembler,
44}
45
46impl AssemblerRiscv {
47 pub fn new(base_addr: usize, _target: Option<Target>) -> Result<Self, CompileError> {
49 Ok(Self {
51 inner: Assembler::new(base_addr),
52 })
53 }
54
55 pub fn finalize(self) -> Result<Vec<u8>, DynasmError> {
57 self.inner.finalize()
58 }
59}
60
61impl Deref for AssemblerRiscv {
62 type Target = Assembler;
63 fn deref(&self) -> &Self::Target {
64 &self.inner
65 }
66}
67
68impl DerefMut for AssemblerRiscv {
69 fn deref_mut(&mut self) -> &mut Self::Target {
70 &mut self.inner
71 }
72}
73
74pub struct MachineRiscv {
76 assembler: AssemblerRiscv,
77 allow_unaligned_memory_accesses: bool,
78 used_gprs: FixedBitSet,
79 used_fprs: FixedBitSet,
80 trap_table: TrapTable,
81 instructions_address_map: Vec<InstructionAddressMap>,
84 src_loc: u32,
86 unwind_ops: Vec<(usize, UnwindOps<GPR, FPR>)>,
88}
89
90const SCRATCH_REG: GPR = GPR::X28;
91
92impl MachineRiscv {
93 pub fn new(
95 target: Option<Target>,
96 allow_unaligned_memory_accesses: bool,
97 ) -> Result<Self, CompileError> {
98 Ok(MachineRiscv {
100 assembler: AssemblerRiscv::new(0, target)?,
101 allow_unaligned_memory_accesses,
102 used_gprs: FixedBitSet::with_capacity(32),
103 used_fprs: FixedBitSet::with_capacity(32),
104 trap_table: TrapTable::default(),
105 instructions_address_map: vec![],
106 src_loc: 0,
107 unwind_ops: vec![],
108 })
109 }
110
111 fn used_gprs_contains(&self, r: &GPR) -> bool {
112 self.used_gprs.contains(r.into_index())
113 }
114 fn used_gprs_insert(&mut self, r: GPR) {
115 self.used_gprs.insert(r.into_index());
116 }
117 fn used_gprs_remove(&mut self, r: &GPR) -> bool {
118 let ret = self.used_gprs_contains(r);
119 self.used_gprs.set(r.into_index(), false);
120 ret
121 }
122
123 fn used_fp_contains(&self, r: &FPR) -> bool {
124 self.used_fprs.contains(r.into_index())
125 }
126 fn used_fprs_insert(&mut self, r: FPR) {
127 self.used_fprs.insert(r.into_index());
128 }
129 fn used_fprs_remove(&mut self, r: &FPR) -> bool {
130 let ret = self.used_fp_contains(r);
131 self.used_fprs.set(r.into_index(), false);
132 ret
133 }
134
135 fn location_to_reg(
136 &mut self,
137 sz: Size,
138 src: Location,
139 temps: &mut Vec<GPR>,
140 allow_imm: ImmType,
141 read_val: bool,
142 wanted: Option<GPR>,
143 ) -> Result<Location, CompileError> {
144 match src {
145 Location::GPR(_) | Location::SIMD(_) => Ok(src),
146 Location::Memory(reg, val) => {
147 let tmp = if let Some(wanted) = wanted {
148 wanted
149 } else {
150 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
151 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
152 })?;
153 temps.push(tmp);
154 tmp
155 };
156 if read_val {
157 if ImmType::Bits12.compatible_imm(val as _) {
158 self.assembler.emit_ld(sz, false, Location::GPR(tmp), src)?;
159 } else {
160 if reg == tmp {
161 codegen_error!("singlepass reg == tmp unreachable");
162 }
163 self.assembler.emit_mov_imm(Location::GPR(tmp), val as _)?;
164 self.assembler.emit_add(
165 Size::S64,
166 Location::GPR(reg),
167 Location::GPR(tmp),
168 Location::GPR(tmp),
169 )?;
170 self.assembler.emit_ld(
171 sz,
172 false,
173 Location::GPR(tmp),
174 Location::Memory(tmp, 0),
175 )?;
176 }
177 }
178 Ok(Location::GPR(tmp))
179 }
180 _ if src.is_imm() => {
181 let imm = src.imm_value_scalar().unwrap();
182 if imm == 0 {
183 Ok(Location::GPR(GPR::XZero))
184 } else if allow_imm.compatible_imm(imm) {
185 Ok(src)
186 } else {
187 let tmp = if let Some(wanted) = wanted {
188 wanted
189 } else {
190 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
191 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
192 })?;
193 temps.push(tmp);
194 tmp
195 };
196 self.assembler.emit_mov_imm(Location::GPR(tmp), imm as _)?;
197 Ok(Location::GPR(tmp))
198 }
199 }
200 _ => todo!("unsupported location"),
201 }
202 }
203
204 fn location_to_fpr(
205 &mut self,
206 sz: Size,
207 src: Location,
208 temps: &mut Vec<FPR>,
209 allow_imm: ImmType,
210 read_val: bool,
211 ) -> Result<Location, CompileError> {
212 match src {
213 Location::SIMD(_) => Ok(src),
214 Location::GPR(_) => {
215 let tmp = self.acquire_temp_simd().ok_or_else(|| {
216 CompileError::Codegen("singlepass cannot acquire temp fpr".to_owned())
217 })?;
218 temps.push(tmp);
219 if read_val {
220 self.assembler.emit_mov(sz, src, Location::SIMD(tmp))?;
221 }
222 Ok(Location::SIMD(tmp))
223 }
224 Location::Memory(_, _) => {
225 let tmp = self.acquire_temp_simd().ok_or_else(|| {
226 CompileError::Codegen("singlepass cannot acquire temp fpr".to_owned())
227 })?;
228 temps.push(tmp);
229 if read_val {
230 self.assembler
231 .emit_ld(sz, false, Location::SIMD(tmp), src)?;
232 }
233 Ok(Location::SIMD(tmp))
234 }
235 _ if src.is_imm() => {
236 let tmp = self.acquire_temp_simd().ok_or_else(|| {
237 CompileError::Codegen("singlepass cannot acquire temp fpr".to_owned())
238 })?;
239 temps.push(tmp);
240
241 let mut gpr_temps = vec![];
242 let dst =
243 self.location_to_reg(sz, src, &mut gpr_temps, allow_imm, read_val, None)?;
244 self.assembler.emit_mov(sz, dst, Location::SIMD(tmp))?;
245 for r in gpr_temps {
246 self.release_gpr(r);
247 }
248
249 Ok(Location::SIMD(tmp))
250 }
251 _ => todo!("unsupported location"),
252 }
253 }
254
255 fn emit_relaxed_binop(
256 &mut self,
257 op: fn(&mut Assembler, Size, Location, Location) -> Result<(), CompileError>,
258 sz: Size,
259 src: Location,
260 dst: Location,
261 ) -> Result<(), CompileError> {
262 let mut temps = vec![];
263 let src = self.location_to_reg(sz, src, &mut temps, ImmType::None, true, None)?;
264 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
265 op(&mut self.assembler, sz, src, dest)?;
266 if dst != dest {
267 self.move_location(sz, dest, dst)?;
268 }
269 for r in temps {
270 self.release_gpr(r);
271 }
272 Ok(())
273 }
274
275 fn emit_relaxed_binop_fp(
276 &mut self,
277 op: fn(&mut Assembler, Size, Location, Location) -> Result<(), CompileError>,
278 sz: Size,
279 src: Location,
280 dst: Location,
281 putback: bool,
282 ) -> Result<(), CompileError> {
283 let mut temps = vec![];
284 let src = self.location_to_fpr(sz, src, &mut temps, ImmType::None, true)?;
285 let dest = self.location_to_fpr(sz, dst, &mut temps, ImmType::None, !putback)?;
286 op(&mut self.assembler, sz, src, dest)?;
287 if dst != dest && putback {
288 self.move_location(sz, dest, dst)?;
289 }
290 for r in temps {
291 self.release_simd(r);
292 }
293 Ok(())
294 }
295
296 fn emit_relaxed_binop3(
297 &mut self,
298 op: fn(&mut Assembler, Size, Location, Location, Location) -> Result<(), CompileError>,
299 sz: Size,
300 src1: Location,
301 src2: Location,
302 dst: Location,
303 allow_imm: ImmType,
304 ) -> Result<(), CompileError> {
305 let mut temps = vec![];
306 let src1 = self.location_to_reg(sz, src1, &mut temps, ImmType::None, true, None)?;
307 let src2 = self.location_to_reg(sz, src2, &mut temps, allow_imm, true, None)?;
308 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
309 op(&mut self.assembler, sz, src1, src2, dest)?;
310 if dst != dest {
311 self.move_location(sz, dest, dst)?;
312 }
313 for r in temps {
314 self.release_gpr(r);
315 }
316 Ok(())
317 }
318
319 fn emit_relaxed_atomic_binop3(
320 &mut self,
321 op: AtomicBinaryOp,
322 sz: Size,
323 dst: Location,
324 addr: GPR,
325 src: Location,
326 ) -> Result<(), CompileError> {
327 let mut temps = vec![];
328 let source = self.location_to_reg(sz, src, &mut temps, ImmType::None, false, None)?;
329 let dest = self.location_to_reg(Size::S64, dst, &mut temps, ImmType::None, false, None)?;
330 let (Location::GPR(source), Location::GPR(dest)) = (source, dest) else {
331 panic!("emit_relaxed_atomic_binop3 expects locations in registers");
332 };
333
334 match sz {
337 Size::S32 | Size::S64 => {
338 if op == AtomicBinaryOp::Sub {
339 self.assembler.emit_neg(
340 Size::S64,
341 Location::GPR(source),
342 Location::GPR(source),
343 )?;
344 self.assembler.emit_atomic_binop(
345 AtomicBinaryOp::Add,
346 sz,
347 dest,
348 addr,
349 source,
350 )?;
351 } else {
352 self.assembler
353 .emit_atomic_binop(op, sz, dest, addr, source)?;
354 }
355 self.assembler.emit_rwfence()?;
356 }
357 Size::S8 | Size::S16 => {
358 let aligned_addr = self.acquire_temp_gpr().ok_or_else(|| {
359 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
360 })?;
361 temps.push(aligned_addr);
362 let bit_offset = self.acquire_temp_gpr().ok_or_else(|| {
363 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
364 })?;
365 temps.push(bit_offset);
366 let bit_mask = self.acquire_temp_gpr().ok_or_else(|| {
367 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
368 })?;
369 temps.push(bit_mask);
370
371 self.assembler.emit_and(
372 Size::S64,
373 Location::GPR(addr),
374 Location::Imm64(-4i64 as _),
375 Location::GPR(aligned_addr),
376 )?;
377 self.assembler.emit_and(
378 Size::S64,
379 Location::GPR(addr),
380 Location::Imm64(3),
381 Location::GPR(bit_offset),
382 )?;
383 self.assembler.emit_sll(
384 Size::S64,
385 Location::GPR(bit_offset),
386 Location::Imm64(3),
387 Location::GPR(bit_offset),
388 )?;
389 self.assembler.emit_mov_imm(
390 Location::GPR(bit_mask),
391 if sz == Size::S8 {
392 u8::MAX as _
393 } else {
394 u16::MAX as _
395 },
396 )?;
397 self.assembler.emit_and(
398 Size::S32,
399 Location::GPR(source),
400 Location::GPR(bit_mask),
401 Location::GPR(source),
402 )?;
403 self.assembler.emit_sll(
404 Size::S64,
405 Location::GPR(bit_mask),
406 Location::GPR(bit_offset),
407 Location::GPR(bit_mask),
408 )?;
409 self.assembler.emit_sll(
410 Size::S64,
411 Location::GPR(source),
412 Location::GPR(bit_offset),
413 Location::GPR(source),
414 )?;
415
416 match op {
417 AtomicBinaryOp::Add | AtomicBinaryOp::Sub | AtomicBinaryOp::Exchange => {
418 let loaded_value = self.acquire_temp_gpr().ok_or_else(|| {
419 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
420 })?;
421 temps.push(loaded_value);
422 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
423 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
424 })?;
425 temps.push(tmp);
426
427 let label_retry = self.get_label();
429 self.emit_label(label_retry)?;
430
431 self.assembler
432 .emit_reserved_ld(Size::S32, loaded_value, aligned_addr)?;
433
434 match op {
435 AtomicBinaryOp::Add => self.assembler.emit_add(
436 Size::S64,
437 Location::GPR(loaded_value),
438 Location::GPR(source),
439 Location::GPR(tmp),
440 )?,
441 AtomicBinaryOp::Sub => self.assembler.emit_sub(
442 Size::S64,
443 Location::GPR(loaded_value),
444 Location::GPR(source),
445 Location::GPR(tmp),
446 )?,
447 AtomicBinaryOp::Exchange => self.assembler.emit_mov(
448 Size::S64,
449 Location::GPR(source),
450 Location::GPR(tmp),
451 )?,
452 _ => unreachable!(),
453 }
454
455 self.assembler.emit_xor(
456 Size::S64,
457 Location::GPR(tmp),
458 Location::GPR(loaded_value),
459 Location::GPR(tmp),
460 )?;
461 self.assembler.emit_and(
462 Size::S64,
463 Location::GPR(tmp),
464 Location::GPR(bit_mask),
465 Location::GPR(tmp),
466 )?;
467 self.assembler.emit_xor(
468 Size::S64,
469 Location::GPR(tmp),
470 Location::GPR(loaded_value),
471 Location::GPR(tmp),
472 )?;
473 self.assembler
474 .emit_reserved_sd(Size::S32, tmp, aligned_addr, tmp)?;
475 let tmp2 = self.acquire_temp_gpr().ok_or_else(|| {
476 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
477 })?;
478 temps.push(tmp2);
479 self.assembler
480 .emit_on_true_label(Location::GPR(tmp), label_retry, tmp2)?;
481
482 self.assembler.emit_rwfence()?;
483
484 self.assembler.emit_and(
486 Size::S32,
487 Location::GPR(loaded_value),
488 Location::GPR(bit_mask),
489 Location::GPR(dest),
490 )?;
491 self.assembler.emit_srl(
492 Size::S32,
493 Location::GPR(dest),
494 Location::GPR(bit_offset),
495 Location::GPR(dest),
496 )?;
497 }
498 AtomicBinaryOp::Or | AtomicBinaryOp::Xor => {
499 self.assembler.emit_atomic_binop(
500 op,
501 Size::S32,
502 dest,
503 aligned_addr,
504 source,
505 )?;
506 self.assembler.emit_rwfence()?;
507 self.assembler.emit_and(
508 Size::S32,
509 Location::GPR(dest),
510 Location::GPR(bit_mask),
511 Location::GPR(dest),
512 )?;
513 self.assembler.emit_srl(
514 Size::S32,
515 Location::GPR(dest),
516 Location::GPR(bit_offset),
517 Location::GPR(dest),
518 )?;
519 }
520 AtomicBinaryOp::And => {
521 self.assembler.emit_not(
522 Size::S64,
523 Location::GPR(bit_mask),
524 Location::GPR(bit_mask),
525 )?;
526 self.assembler.emit_or(
527 Size::S64,
528 Location::GPR(bit_mask),
529 Location::GPR(source),
530 Location::GPR(source),
531 )?;
532 self.assembler.emit_not(
533 Size::S64,
534 Location::GPR(bit_mask),
535 Location::GPR(bit_mask),
536 )?;
537 self.assembler.emit_atomic_binop(
538 op,
539 Size::S32,
540 dest,
541 aligned_addr,
542 source,
543 )?;
544 self.assembler.emit_rwfence()?;
545 self.assembler.emit_and(
546 Size::S32,
547 Location::GPR(dest),
548 Location::GPR(bit_mask),
549 Location::GPR(dest),
550 )?;
551 self.assembler.emit_srl(
552 Size::S32,
553 Location::GPR(dest),
554 Location::GPR(bit_offset),
555 Location::GPR(dest),
556 )?;
557 }
558 }
559 }
560 }
561
562 if dst != Location::GPR(dest) {
563 self.move_location(sz, Location::GPR(dest), dst)?;
564 }
565
566 for r in temps {
567 self.release_gpr(r);
568 }
569 Ok(())
570 }
571
572 fn emit_relaxed_atomic_cmpxchg(
573 &mut self,
574 size: Size,
575 dst: Location,
576 addr: GPR,
577 new: Location,
578 cmp: Location,
579 ) -> Result<(), CompileError> {
580 let mut temps = vec![];
581 let cmp = self.location_to_reg(size, cmp, &mut temps, ImmType::None, true, None)?;
582 let new = self.location_to_reg(size, new, &mut temps, ImmType::None, true, None)?;
583 let (Location::GPR(cmp), Location::GPR(new)) = (cmp, new) else {
584 panic!("emit_relaxed_atomic_cmpxchg expects locations in registers");
585 };
586
587 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
588 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
589 })?;
590 temps.push(jmp_tmp);
591
592 match size {
593 Size::S32 | Size::S64 => {
594 let value = self.acquire_temp_gpr().ok_or_else(|| {
595 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
596 })?;
597 temps.push(value);
598 let cond = self.acquire_temp_gpr().ok_or_else(|| {
599 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
600 })?;
601 temps.push(cond);
602
603 let label_retry = self.get_label();
605 let label_after_retry = self.get_label();
606 self.emit_label(label_retry)?;
607
608 self.assembler.emit_reserved_ld(size, value, addr)?;
609 self.assembler.emit_cmp(
610 Condition::Eq,
611 Location::GPR(value),
612 Location::GPR(cmp),
613 Location::GPR(cond),
614 )?;
615 self.assembler.emit_on_false_label(
616 Location::GPR(cond),
617 label_after_retry,
618 jmp_tmp,
619 )?;
620 self.assembler.emit_reserved_sd(size, cond, addr, new)?;
621 self.assembler
622 .emit_on_true_label(Location::GPR(cond), label_retry, jmp_tmp)?;
623
624 self.assembler.emit_rwfence()?;
626 self.emit_label(label_after_retry)?;
627
628 self.assembler.emit_mov(size, Location::GPR(value), dst)?;
629 }
630 Size::S8 | Size::S16 => {
631 let value = self.acquire_temp_gpr().ok_or_else(|| {
632 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
633 })?;
634 temps.push(value);
635 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
636 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
637 })?;
638 temps.push(tmp);
639 let bit_offset = self.acquire_temp_gpr().ok_or_else(|| {
640 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
641 })?;
642 temps.push(bit_offset);
643 let bit_mask = self.acquire_temp_gpr().ok_or_else(|| {
644 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
645 })?;
646 temps.push(bit_mask);
647 let cond = self.acquire_temp_gpr().ok_or_else(|| {
648 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
649 })?;
650 temps.push(cond);
651
652 self.assembler.emit_and(
654 Size::S64,
655 Location::GPR(addr),
656 Location::Imm64(3),
657 Location::GPR(bit_offset),
658 )?;
659 self.assembler.emit_and(
660 Size::S64,
661 Location::GPR(addr),
662 Location::Imm64(-4i64 as _),
663 Location::GPR(addr),
664 )?;
665 self.assembler.emit_sll(
666 Size::S64,
667 Location::GPR(bit_offset),
668 Location::Imm64(3),
669 Location::GPR(bit_offset),
670 )?;
671 self.assembler.emit_mov_imm(
672 Location::GPR(bit_mask),
673 if size == Size::S8 {
674 u8::MAX as _
675 } else {
676 u16::MAX as _
677 },
678 )?;
679 self.assembler.emit_and(
680 Size::S32,
681 Location::GPR(new),
682 Location::GPR(bit_mask),
683 Location::GPR(new),
684 )?;
685 self.assembler.emit_sll(
686 Size::S64,
687 Location::GPR(bit_mask),
688 Location::GPR(bit_offset),
689 Location::GPR(bit_mask),
690 )?;
691 self.assembler.emit_sll(
692 Size::S64,
693 Location::GPR(new),
694 Location::GPR(bit_offset),
695 Location::GPR(new),
696 )?;
697 self.assembler.emit_sll(
698 Size::S64,
699 Location::GPR(cmp),
700 Location::GPR(bit_offset),
701 Location::GPR(cmp),
702 )?;
703
704 let label_retry = self.get_label();
706 let label_after_retry = self.get_label();
707 self.emit_label(label_retry)?;
708
709 self.assembler.emit_reserved_ld(Size::S32, value, addr)?;
710 self.assembler.emit_and(
711 Size::S32,
712 Location::GPR(value),
713 Location::GPR(bit_mask),
714 Location::GPR(tmp),
715 )?;
716
717 self.assembler.emit_cmp(
718 Condition::Eq,
719 Location::GPR(tmp),
720 Location::GPR(cmp),
721 Location::GPR(cond),
722 )?;
723 self.assembler.emit_on_false_label(
724 Location::GPR(cond),
725 label_after_retry,
726 jmp_tmp,
727 )?;
728
729 self.assembler.emit_xor(
731 Size::S32,
732 Location::GPR(value),
733 Location::GPR(new),
734 Location::GPR(tmp),
735 )?;
736 self.assembler.emit_and(
737 Size::S32,
738 Location::GPR(tmp),
739 Location::GPR(bit_mask),
740 Location::GPR(tmp),
741 )?;
742 self.assembler.emit_xor(
743 Size::S32,
744 Location::GPR(tmp),
745 Location::GPR(value),
746 Location::GPR(tmp),
747 )?;
748 self.assembler
749 .emit_reserved_sd(Size::S32, cond, addr, tmp)?;
750 self.assembler
751 .emit_on_true_label(Location::GPR(cond), label_retry, jmp_tmp)?;
752
753 self.assembler.emit_rwfence()?;
755 self.emit_label(label_after_retry)?;
756
757 self.assembler.emit_and(
758 Size::S32,
759 Location::GPR(value),
760 Location::GPR(bit_mask),
761 Location::GPR(tmp),
762 )?;
763 self.assembler.emit_srl(
764 Size::S32,
765 Location::GPR(tmp),
766 Location::GPR(bit_offset),
767 Location::GPR(tmp),
768 )?;
769 self.assembler
770 .emit_mov(Size::S32, Location::GPR(tmp), dst)?;
771 }
772 }
773
774 for r in temps {
775 self.release_gpr(r);
776 }
777 Ok(())
778 }
779
780 #[allow(clippy::too_many_arguments)]
781 fn emit_relaxed_binop3_fp(
782 &mut self,
783 op: fn(&mut Assembler, Size, Location, Location, Location) -> Result<(), CompileError>,
784 sz: Size,
785 src1: Location,
786 src2: Location,
787 dst: Location,
788 allow_imm: ImmType,
789 return_nan_if_present: bool,
790 ) -> Result<(), CompileError> {
791 let mut temps = vec![];
792 let mut gprs = vec![];
793 let src1 = self.location_to_fpr(sz, src1, &mut temps, ImmType::None, true)?;
794 let src2 = self.location_to_fpr(sz, src2, &mut temps, allow_imm, true)?;
795 let dest = self.location_to_fpr(sz, dst, &mut temps, ImmType::None, false)?;
796
797 let label_after = self.get_label();
798 if return_nan_if_present {
799 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
800 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
801 })?;
802 gprs.push(tmp);
803 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
804 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
805 })?;
806 gprs.push(jmp_tmp);
807
808 let canonical_nan = match sz {
810 Size::S32 => CANONICAL_NAN_F32 as u64,
811 Size::S64 => CANONICAL_NAN_F64,
812 _ => unreachable!(),
813 };
814
815 self.assembler
816 .emit_mov_imm(Location::GPR(tmp), canonical_nan as _)?;
817 self.assembler.emit_mov(sz, Location::GPR(tmp), dest)?;
818
819 self.assembler
820 .emit_fcmp(Condition::Eq, sz, src1, src1, Location::GPR(tmp))?;
821 self.assembler
822 .emit_on_false_label(Location::GPR(tmp), label_after, jmp_tmp)?;
823
824 self.assembler
825 .emit_fcmp(Condition::Eq, sz, src2, src2, Location::GPR(tmp))?;
826 self.assembler
827 .emit_on_false_label(Location::GPR(tmp), label_after, jmp_tmp)?;
828 }
829
830 op(&mut self.assembler, sz, src1, src2, dest)?;
831 self.emit_label(label_after)?;
832
833 if dst != dest {
834 self.move_location(sz, dest, dst)?;
835 }
836 for r in temps {
837 self.release_simd(r);
838 }
839 for r in gprs {
840 self.release_gpr(r);
841 }
842 Ok(())
843 }
844
845 fn emit_relaxed_cmp(
846 &mut self,
847 c: Condition,
848 loc_a: Location,
849 loc_b: Location,
850 ret: Location,
851 sz: Size,
852 signed: bool,
853 ) -> Result<(), CompileError> {
854 let mut temps = vec![];
856 let loc_a = self.location_to_reg(sz, loc_a, &mut temps, ImmType::None, true, None)?;
857 let loc_b = self.location_to_reg(sz, loc_b, &mut temps, ImmType::None, true, None)?;
858
859 if sz != Size::S64 {
860 self.assembler.emit_extend(sz, signed, loc_a, loc_a)?;
861 self.assembler.emit_extend(sz, signed, loc_b, loc_b)?;
862 }
863
864 let dest = self.location_to_reg(sz, ret, &mut temps, ImmType::None, false, None)?;
865 self.assembler.emit_cmp(c, loc_a, loc_b, dest)?;
866 if ret != dest {
867 self.move_location(sz, dest, ret)?;
868 }
869 for r in temps {
870 self.release_gpr(r);
871 }
872 Ok(())
873 }
874
875 fn emit_cmpop_i32_dynamic_b(
877 &mut self,
878 c: Condition,
879 loc_a: Location,
880 loc_b: Location,
881 ret: Location,
882 signed: bool,
883 ) -> Result<(), CompileError> {
884 self.emit_relaxed_cmp(c, loc_a, loc_b, ret, Size::S32, signed)
885 }
886
887 fn emit_cmpop_i64_dynamic_b(
889 &mut self,
890 c: Condition,
891 loc_a: Location,
892 loc_b: Location,
893 ret: Location,
894 ) -> Result<(), CompileError> {
895 self.emit_relaxed_cmp(c, loc_a, loc_b, ret, Size::S64, false)
896 }
897
898 #[allow(clippy::too_many_arguments)]
902 fn memory_op<F: FnOnce(&mut Self, GPR) -> Result<(), CompileError>>(
903 &mut self,
904 addr: Location,
905 memarg: &MemArg,
906 check_alignment: bool,
907 value_size: usize,
908 imported_memories: bool,
909 offset: i32,
910 heap_access_oob: Label,
911 unaligned_atomic: Label,
912 cb: F,
913 ) -> Result<(), CompileError> {
914 let value_size = value_size as i64;
915 let tmp_addr = self.acquire_temp_gpr().ok_or_else(|| {
916 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
917 })?;
918
919 let (base_loc, bound_loc) = if imported_memories {
921 self.emit_relaxed_binop(
923 Assembler::emit_mov,
924 Size::S64,
925 Location::Memory(self.get_vmctx_reg(), offset),
926 Location::GPR(tmp_addr),
927 )?;
928 (Location::Memory(tmp_addr, 0), Location::Memory(tmp_addr, 8))
929 } else {
930 (
931 Location::Memory(self.get_vmctx_reg(), offset),
932 Location::Memory(self.get_vmctx_reg(), offset + 8),
933 )
934 };
935
936 let tmp_base = self.acquire_temp_gpr().ok_or_else(|| {
937 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
938 })?;
939 let tmp_bound = self.acquire_temp_gpr().ok_or_else(|| {
940 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
941 })?;
942
943 self.emit_relaxed_load(Size::S64, false, Location::GPR(tmp_base), base_loc)?;
945
946 self.emit_relaxed_load(Size::S64, false, Location::GPR(tmp_bound), bound_loc)?;
948
949 self.assembler.emit_add(
953 Size::S64,
954 Location::GPR(tmp_bound),
955 Location::GPR(tmp_base),
956 Location::GPR(tmp_bound),
957 )?;
958 self.assembler.emit_sub(
959 Size::S64,
960 Location::GPR(tmp_bound),
961 Location::Imm64(value_size as _),
962 Location::GPR(tmp_bound),
963 )?;
964
965 self.move_location(Size::S32, addr, Location::GPR(tmp_addr))?;
969
970 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
971 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
972 })?;
973
974 if memarg.offset != 0 {
976 if ImmType::Bits12.compatible_imm(memarg.offset as _) {
977 self.assembler.emit_add(
978 Size::S64,
979 Location::Imm64(memarg.offset),
980 Location::GPR(tmp_addr),
981 Location::GPR(tmp_addr),
982 )?;
983 } else {
984 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
985 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
986 })?;
987 self.assembler
988 .emit_mov_imm(Location::GPR(tmp), memarg.offset as _)?;
989 self.assembler.emit_add(
990 Size::S64,
991 Location::GPR(tmp_addr),
992 Location::GPR(tmp),
993 Location::GPR(tmp_addr),
994 )?;
995 self.release_gpr(tmp);
996 }
997
998 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
1001 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1002 })?;
1003 self.assembler.emit_srl(
1004 Size::S64,
1005 Location::GPR(tmp_addr),
1006 Location::Imm64(32),
1007 Location::GPR(tmp),
1008 )?;
1009 self.assembler
1010 .emit_on_true_label_far(Location::GPR(tmp), heap_access_oob, jmp_tmp)?;
1011 self.release_gpr(tmp);
1012 }
1013
1014 self.assembler.emit_add(
1016 Size::S64,
1017 Location::GPR(tmp_base),
1018 Location::GPR(tmp_addr),
1019 Location::GPR(tmp_addr),
1020 )?;
1021
1022 let cond = tmp_base;
1024
1025 self.assembler.emit_cmp(
1027 Condition::Le,
1028 Location::GPR(tmp_addr),
1029 Location::GPR(tmp_bound),
1030 Location::GPR(cond),
1031 )?;
1032
1033 self.assembler
1035 .emit_on_false_label_far(Location::GPR(cond), heap_access_oob, jmp_tmp)?;
1036
1037 self.release_gpr(tmp_bound);
1038 self.release_gpr(cond);
1039
1040 let align = value_size as u32;
1041 if check_alignment && align != 1 {
1042 self.assembler.emit_and(
1043 Size::S64,
1044 Location::GPR(tmp_addr),
1045 Location::Imm64((align - 1) as u64),
1046 Location::GPR(cond),
1047 )?;
1048 self.assembler.emit_on_true_label_far(
1049 Location::GPR(cond),
1050 unaligned_atomic,
1051 jmp_tmp,
1052 )?;
1053 }
1054 let begin = self.assembler.get_offset().0;
1055 cb(self, tmp_addr)?;
1056 let end = self.assembler.get_offset().0;
1057 self.mark_address_range_with_trap_code(TrapCode::HeapAccessOutOfBounds, begin, end);
1058
1059 self.release_gpr(jmp_tmp);
1060 self.release_gpr(tmp_addr);
1061 Ok(())
1062 }
1063
1064 fn emit_relaxed_load(
1065 &mut self,
1066 sz: Size,
1067 signed: bool,
1068 dst: Location,
1069 src: Location,
1070 ) -> Result<(), CompileError> {
1071 let mut temps = vec![];
1072 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
1073 match src {
1074 Location::Memory(addr, offset) => {
1075 if ImmType::Bits12.compatible_imm(offset as i64) {
1076 self.assembler.emit_ld(sz, signed, dest, src)?;
1077 } else {
1078 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
1079 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1080 })?;
1081 self.assembler
1082 .emit_mov_imm(Location::GPR(tmp), offset as i64)?;
1083 self.assembler.emit_add(
1084 Size::S64,
1085 Location::GPR(addr),
1086 Location::GPR(tmp),
1087 Location::GPR(tmp),
1088 )?;
1089 self.assembler
1090 .emit_ld(sz, signed, dest, Location::Memory(tmp, 0))?;
1091 temps.push(tmp);
1092 }
1093 }
1094 _ => codegen_error!("singlepass emit_relaxed_load unreachable"),
1095 }
1096 if dst != dest {
1097 self.move_location(Size::S64, dest, dst)?;
1099 }
1100 for r in temps {
1101 self.release_gpr(r);
1102 }
1103 Ok(())
1104 }
1105
1106 fn emit_maybe_unaligned_load(
1107 &mut self,
1108 sz: Size,
1109 signed: bool,
1110 dst: Location,
1111 src: GPR,
1112 ) -> Result<(), CompileError> {
1113 if !self.allow_unaligned_memory_accesses {
1114 return self.emit_relaxed_load(sz, signed, dst, Location::Memory(src, 0));
1115 }
1116
1117 if let Size::S8 = sz {
1118 return self.emit_relaxed_load(sz, signed, dst, Location::Memory(src, 0));
1119 }
1120
1121 let label_unaligned = self.get_label();
1122 let label_completed = self.get_label();
1123 let mut temps = vec![];
1124 let tmp_cond = self.acquire_temp_gpr().ok_or_else(|| {
1125 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1126 })?;
1127 temps.push(tmp_cond);
1128 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
1129 self.assembler.emit_and(
1130 Size::S64,
1131 Location::GPR(src),
1132 Location::Imm64((sz.bytes() - 1) as u64),
1133 Location::GPR(tmp_cond),
1134 )?;
1135 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
1136 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1137 })?;
1138 temps.push(jmp_tmp);
1139 self.assembler
1140 .emit_on_true_label(Location::GPR(tmp_cond), label_unaligned, jmp_tmp)?;
1141 self.assembler
1143 .emit_ld(sz, signed, dest, Location::Memory(src, 0))?;
1144 self.jmp_unconditional(label_completed)?;
1145 self.emit_label(label_unaligned)?;
1146
1147 let tmp_value = tmp_cond;
1149
1150 self.assembler
1152 .emit_ld(Size::S8, false, dest, Location::Memory(src, 0))?;
1153 for i in 1..sz.bytes() {
1154 self.assembler.emit_ld(
1155 Size::S8,
1156 if i == sz.bytes() - 1 { signed } else { false },
1157 Location::GPR(tmp_value),
1158 Location::Memory(src, i as _),
1159 )?;
1160 self.assembler.emit_sll(
1161 Size::S64,
1162 Location::GPR(tmp_value),
1163 Location::Imm64(8 * i as u64),
1164 Location::GPR(tmp_value),
1165 )?;
1166 self.assembler
1167 .emit_or(Size::S64, dest, Location::GPR(tmp_value), dest)?;
1168 }
1169
1170 self.emit_label(label_completed)?;
1172 if dst != dest {
1173 self.move_location(Size::S64, dest, dst)?;
1175 }
1176
1177 for tmp in temps {
1178 self.release_gpr(tmp);
1179 }
1180 Ok(())
1181 }
1182
1183 fn emit_relaxed_store(
1184 &mut self,
1185 sz: Size,
1186 dst: Location,
1187 src: Location,
1188 ) -> Result<(), CompileError> {
1189 let mut temps = vec![];
1190 let dest = self.location_to_reg(Size::S64, dst, &mut temps, ImmType::None, true, None)?;
1191 match src {
1192 Location::Memory(addr, offset) => {
1193 if ImmType::Bits12.compatible_imm(offset as i64) {
1194 self.assembler.emit_sd(sz, dest, src)?;
1195 } else {
1196 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
1197 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1198 })?;
1199 self.assembler
1200 .emit_mov_imm(Location::GPR(tmp), offset as i64)?;
1201 self.assembler.emit_add(
1202 Size::S64,
1203 Location::GPR(addr),
1204 Location::GPR(tmp),
1205 Location::GPR(tmp),
1206 )?;
1207 self.assembler.emit_sd(sz, dest, Location::Memory(tmp, 0))?;
1208 temps.push(tmp);
1209 }
1210 }
1211 _ => codegen_error!("singlepass emit_relaxed_store unreachable"),
1212 }
1213 for r in temps {
1214 self.release_gpr(r);
1215 }
1216 Ok(())
1217 }
1218
1219 fn emit_maybe_unaligned_store(
1220 &mut self,
1221 sz: Size,
1222 src: Location,
1223 dst: GPR,
1224 ) -> Result<(), CompileError> {
1225 if !self.allow_unaligned_memory_accesses {
1226 return self.emit_relaxed_store(sz, src, Location::Memory(dst, 0));
1227 }
1228
1229 if let Size::S8 = sz {
1230 return self.emit_relaxed_store(sz, src, Location::Memory(dst, 0));
1235 }
1236
1237 let label_unaligned = self.get_label();
1238 let label_completed = self.get_label();
1239 let mut temps = vec![];
1240 let tmp_cond = self.acquire_temp_gpr().ok_or_else(|| {
1241 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1242 })?;
1243 temps.push(tmp_cond);
1244 let src_value =
1245 self.location_to_reg(Size::S64, src, &mut temps, ImmType::None, true, None)?;
1246 self.assembler.emit_and(
1247 Size::S64,
1248 Location::GPR(dst),
1249 Location::Imm64((sz.bytes() - 1) as u64),
1250 Location::GPR(tmp_cond),
1251 )?;
1252 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
1253 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1254 })?;
1255 temps.push(jmp_tmp);
1256 self.assembler
1257 .emit_on_true_label(Location::GPR(tmp_cond), label_unaligned, jmp_tmp)?;
1258 self.assembler
1260 .emit_sd(sz, src_value, Location::Memory(dst, 0))?;
1261 self.jmp_unconditional(label_completed)?;
1262 self.emit_label(label_unaligned)?;
1263
1264 let size_bytes = sz.bytes();
1266 for i in 0..size_bytes {
1268 self.assembler
1269 .emit_sd(Size::S8, src_value, Location::Memory(dst, i as _))?;
1270 if i != size_bytes - 1 {
1271 self.assembler
1272 .emit_srl(Size::S64, src_value, Location::Imm64(8), src_value)?;
1273 }
1274 }
1275
1276 self.emit_label(label_completed)?;
1278
1279 for tmp in temps {
1280 self.release_gpr(tmp);
1281 }
1282 Ok(())
1283 }
1284
1285 fn emit_rol(
1286 &mut self,
1287 sz: Size,
1288 loc_a: Location,
1289 loc_b: Location,
1290 ret: Location,
1291 allow_imm: ImmType,
1292 ) -> Result<(), CompileError> {
1293 let mut temps = vec![];
1294 let size_bits = sz.bits();
1295
1296 let src2 = if let Some(imm) = loc_b.imm_value_scalar() {
1297 Location::Imm32(size_bits - (imm as u32) % size_bits)
1298 } else {
1299 let tmp1 = self.location_to_reg(
1300 sz,
1301 Location::Imm32(size_bits),
1302 &mut temps,
1303 ImmType::None,
1304 true,
1305 None,
1306 )?;
1307 let tmp2 = self.location_to_reg(sz, loc_b, &mut temps, ImmType::None, true, None)?;
1308 self.assembler.emit_sub(sz, tmp1, tmp2, tmp1)?;
1309 tmp1
1310 };
1311
1312 self.emit_ror(sz, loc_a, src2, ret, allow_imm)?;
1313
1314 for r in temps {
1315 self.release_gpr(r);
1316 }
1317 Ok(())
1318 }
1319
1320 fn emit_ror(
1321 &mut self,
1322 sz: Size,
1323 loc_a: Location,
1324 loc_b: Location,
1325 ret: Location,
1326 allow_imm: ImmType,
1327 ) -> Result<(), CompileError> {
1328 let mut temps = vec![];
1329
1330 let imm = match sz {
1331 Size::S32 | Size::S64 => Location::Imm32(sz.bits() as u32),
1332 _ => codegen_error!("singlepass emit_ror unreachable"),
1333 };
1334 let imm = self.location_to_reg(sz, imm, &mut temps, ImmType::None, false, None)?;
1335
1336 let tmp1 = self.acquire_temp_gpr().ok_or_else(|| {
1337 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1338 })?;
1339 self.emit_relaxed_binop3(
1340 Assembler::emit_srl,
1341 sz,
1342 loc_a,
1343 loc_b,
1344 Location::GPR(tmp1),
1345 allow_imm,
1346 )?;
1347
1348 let tmp2 = self.acquire_temp_gpr().ok_or_else(|| {
1349 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1350 })?;
1351 self.emit_relaxed_binop3(
1352 Assembler::emit_sub,
1353 sz,
1354 imm,
1355 loc_b,
1356 Location::GPR(tmp2),
1357 allow_imm,
1358 )?;
1359 self.emit_relaxed_binop3(
1360 Assembler::emit_sll,
1361 sz,
1362 loc_a,
1363 Location::GPR(tmp2),
1364 Location::GPR(tmp2),
1365 ImmType::Bits12,
1366 )?;
1367 self.assembler.emit_or(
1368 sz,
1369 Location::GPR(tmp1),
1370 Location::GPR(tmp2),
1371 Location::GPR(tmp1),
1372 )?;
1373
1374 self.move_location(sz, Location::GPR(tmp1), ret)?;
1375 self.release_gpr(tmp1);
1376 self.release_gpr(tmp2);
1377 for r in temps {
1378 self.release_gpr(r);
1379 }
1380 Ok(())
1381 }
1382
1383 fn emit_popcnt(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1384 let arg = self.acquire_temp_gpr().ok_or_else(|| {
1385 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1386 })?;
1387 let cnt = self.acquire_temp_gpr().ok_or_else(|| {
1388 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1389 })?;
1390 let temp = self.acquire_temp_gpr().ok_or_else(|| {
1391 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1392 })?;
1393 self.move_location(sz, src, Location::GPR(arg))?;
1394
1395 self.move_location(sz, Location::Imm32(0), Location::GPR(cnt))?;
1396 let one_imm = match sz {
1397 Size::S32 => Location::Imm32(1),
1398 Size::S64 => Location::Imm64(1),
1399 _ => codegen_error!("singlepass emit_popcnt unreachable"),
1400 };
1401
1402 let label_loop = self.assembler.get_label();
1403 let label_exit = self.assembler.get_label();
1404
1405 self.assembler.emit_label(label_loop)?; self.assembler
1407 .emit_and(sz, Location::GPR(arg), one_imm, Location::GPR(temp))?;
1408 self.assembler.emit_add(
1409 sz,
1410 Location::GPR(cnt),
1411 Location::GPR(temp),
1412 Location::GPR(cnt),
1413 )?;
1414 self.assembler
1415 .emit_srl(sz, Location::GPR(arg), one_imm, Location::GPR(arg))?;
1416 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
1417 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1418 })?;
1419 self.assembler
1420 .emit_on_false_label(Location::GPR(arg), label_exit, jmp_tmp)?;
1421 self.release_gpr(jmp_tmp);
1422 self.jmp_unconditional(label_loop)?;
1423
1424 self.assembler.emit_label(label_exit)?; self.move_location(sz, Location::GPR(cnt), dst)?;
1427
1428 self.release_gpr(arg);
1429 self.release_gpr(cnt);
1430 self.release_gpr(temp);
1431 Ok(())
1432 }
1433
1434 fn emit_ctz(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1435 let size_bits = sz.bits();
1436 let arg = self.acquire_temp_gpr().ok_or_else(|| {
1437 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1438 })?;
1439 let cnt = self.acquire_temp_gpr().ok_or_else(|| {
1440 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1441 })?;
1442 let temp = self.acquire_temp_gpr().ok_or_else(|| {
1443 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1444 })?;
1445 self.move_location(sz, src, Location::GPR(arg))?;
1446
1447 let one_imm = match sz {
1448 Size::S32 => Location::Imm32(1),
1449 Size::S64 => Location::Imm64(1),
1450 _ => codegen_error!("singlepass emit_ctz unreachable"),
1451 };
1452
1453 let label_loop = self.assembler.get_label();
1454 let label_exit = self.assembler.get_label();
1455
1456 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
1457 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1458 })?;
1459
1460 self.move_location(sz, Location::Imm32(size_bits), Location::GPR(cnt))?;
1462 self.assembler
1463 .emit_on_false_label(Location::GPR(arg), label_exit, jmp_tmp)?;
1464
1465 self.move_location(sz, Location::Imm32(0), Location::GPR(cnt))?;
1466
1467 self.assembler.emit_label(label_loop)?; self.assembler
1469 .emit_and(sz, Location::GPR(arg), one_imm, Location::GPR(temp))?;
1470 self.assembler
1471 .emit_on_true_label(Location::GPR(temp), label_exit, jmp_tmp)?;
1472
1473 self.release_gpr(jmp_tmp);
1474
1475 self.assembler
1476 .emit_add(sz, Location::GPR(cnt), one_imm, Location::GPR(cnt))?;
1477 self.assembler
1478 .emit_srl(sz, Location::GPR(arg), one_imm, Location::GPR(arg))?;
1479 self.jmp_unconditional(label_loop)?;
1480
1481 self.assembler.emit_label(label_exit)?; self.move_location(sz, Location::GPR(cnt), dst)?;
1484
1485 self.release_gpr(arg);
1486 self.release_gpr(cnt);
1487 self.release_gpr(temp);
1488 Ok(())
1489 }
1490
1491 fn emit_clz(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1492 let size_bits = sz.bits();
1493 let arg = self.acquire_temp_gpr().ok_or_else(|| {
1494 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1495 })?;
1496 let cnt = self.acquire_temp_gpr().ok_or_else(|| {
1497 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1498 })?;
1499 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
1500 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1501 })?;
1502 self.move_location(sz, src, Location::GPR(arg))?;
1503
1504 let one_imm = match sz {
1505 Size::S32 => Location::Imm32(1),
1506 Size::S64 => Location::Imm64(1),
1507 _ => codegen_error!("singlepass emit_ctz unreachable"),
1508 };
1509
1510 let label_loop = self.assembler.get_label();
1511 let label_exit = self.assembler.get_label();
1512
1513 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
1514 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1515 })?;
1516
1517 self.move_location(sz, Location::Imm32(size_bits), Location::GPR(cnt))?;
1519 self.assembler
1520 .emit_on_false_label(Location::GPR(arg), label_exit, jmp_tmp)?;
1521
1522 self.move_location(sz, Location::Imm32(0), Location::GPR(cnt))?;
1523
1524 self.assembler.emit_label(label_loop)?;
1526 self.move_location(
1528 Size::S32,
1529 Location::Imm32(size_bits - 1),
1530 Location::GPR(tmp),
1531 )?;
1532 self.assembler.emit_sub(
1533 Size::S32,
1534 Location::GPR(tmp),
1535 Location::GPR(cnt),
1536 Location::GPR(tmp),
1537 )?;
1538 self.assembler.emit_srl(
1539 sz,
1540 Location::GPR(arg),
1541 Location::GPR(tmp),
1542 Location::GPR(tmp),
1543 )?;
1544 self.assembler
1545 .emit_on_true_label(Location::GPR(tmp), label_exit, jmp_tmp)?;
1546
1547 self.assembler
1548 .emit_add(sz, Location::GPR(cnt), one_imm, Location::GPR(cnt))?;
1549 self.release_gpr(jmp_tmp);
1550 self.jmp_unconditional(label_loop)?;
1551
1552 self.assembler.emit_label(label_exit)?; self.move_location(sz, Location::GPR(cnt), dst)?;
1555
1556 self.release_gpr(arg);
1557 self.release_gpr(cnt);
1558 self.release_gpr(tmp);
1559 Ok(())
1560 }
1561
1562 fn convert_float_to_int(
1563 &mut self,
1564 loc: Location,
1565 size_in: Size,
1566 ret: Location,
1567 size_out: Size,
1568 signed: bool,
1569 sat: bool,
1570 ) -> Result<(), CompileError> {
1571 let mut gprs = vec![];
1572 let mut fprs = vec![];
1573 let src = self.location_to_fpr(size_in, loc, &mut fprs, ImmType::None, true)?;
1574 let dest = self.location_to_reg(size_out, ret, &mut gprs, ImmType::None, false, None)?;
1575
1576 if sat {
1577 let end = self.assembler.get_label();
1580 let cond = self.acquire_temp_gpr().ok_or_else(|| {
1581 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1582 })?;
1583 self.zero_location(size_out, dest)?;
1584 self.assembler
1586 .emit_fcmp(Condition::Eq, size_in, src, src, Location::GPR(cond))?;
1587 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
1588 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1589 })?;
1590 self.assembler
1591 .emit_on_false_label(Location::GPR(cond), end, jmp_tmp)?;
1592 self.release_gpr(jmp_tmp);
1593 self.release_gpr(cond);
1594
1595 self.assembler
1596 .emit_fcvt(signed, size_in, src, size_out, dest)?;
1597 self.emit_label(end)?;
1598 } else {
1599 let old_fcsr = self.acquire_temp_gpr().ok_or_else(|| {
1600 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1601 })?;
1602 self.move_location(
1603 Size::S32,
1604 Location::GPR(GPR::XZero),
1605 Location::GPR(old_fcsr),
1606 )?;
1607 self.assembler.emit_swap_fscr(old_fcsr)?;
1608 self.assembler
1609 .emit_fcvt(signed, size_in, src, size_out, dest)?;
1610 self.trap_float_conversion_errors(size_in, src, old_fcsr, &mut gprs)?;
1611 self.release_gpr(old_fcsr);
1612 }
1613
1614 if ret != dest {
1615 self.move_location(size_out, dest, ret)?;
1616 }
1617 for r in gprs {
1618 self.release_gpr(r);
1619 }
1620 for r in fprs {
1621 self.release_simd(r);
1622 }
1623 Ok(())
1624 }
1625
1626 fn convert_int_to_float(
1627 &mut self,
1628 loc: Location,
1629 size_in: Size,
1630 ret: Location,
1631 size_out: Size,
1632 signed: bool,
1633 ) -> Result<(), CompileError> {
1634 let mut gprs = vec![];
1635 let mut fprs = vec![];
1636 let src = self.location_to_reg(size_in, loc, &mut gprs, ImmType::None, true, None)?;
1637 let dest = self.location_to_fpr(size_out, ret, &mut fprs, ImmType::None, false)?;
1638 self.assembler
1639 .emit_fcvt(signed, size_in, src, size_out, dest)?;
1640 if ret != dest {
1641 self.move_location(Size::S32, dest, ret)?;
1642 }
1643 for r in gprs {
1644 self.release_gpr(r);
1645 }
1646 for r in fprs {
1647 self.release_simd(r);
1648 }
1649 Ok(())
1650 }
1651
1652 fn convert_float_to_float(
1653 &mut self,
1654 loc: Location,
1655 size_in: Size,
1656 ret: Location,
1657 size_out: Size,
1658 ) -> Result<(), CompileError> {
1659 let mut temps = vec![];
1660 let src = self.location_to_fpr(size_in, loc, &mut temps, ImmType::None, true)?;
1661 let dest = self.location_to_fpr(size_out, ret, &mut temps, ImmType::None, false)?;
1662
1663 match (size_in, size_out) {
1664 (Size::S32, Size::S64) => self
1665 .assembler
1666 .emit_fcvt(false, size_in, src, size_out, dest)?,
1667 (Size::S64, Size::S32) => self
1668 .assembler
1669 .emit_fcvt(false, size_in, src, size_out, dest)?,
1670 _ => codegen_error!("singlepass convert_float_to_float unreachable"),
1671 }
1672
1673 if ret != dest {
1674 self.move_location(size_out, dest, ret)?;
1675 }
1676 for r in temps {
1677 self.release_simd(r);
1678 }
1679 Ok(())
1680 }
1681
1682 fn trap_float_conversion_errors(
1683 &mut self,
1684 sz: Size,
1685 f: Location,
1686 old_fcsr: GPR,
1687 temps: &mut Vec<GPR>,
1688 ) -> Result<(), CompileError> {
1689 let fscr = self.acquire_temp_gpr().ok_or_else(|| {
1690 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1691 })?;
1692 self.zero_location(Size::S64, Location::GPR(fscr))?;
1693 temps.push(fscr);
1694
1695 let trap_badconv = self.assembler.get_label();
1696 let end = self.assembler.get_label();
1697
1698 self.assembler.emit_swap_fscr(fscr)?;
1699
1700 self.assembler.emit_srl(
1703 Size::S32,
1704 Location::GPR(fscr),
1705 Location::Imm32(4),
1706 Location::GPR(fscr),
1707 )?;
1708
1709 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
1710 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1711 })?;
1712 temps.push(jmp_tmp);
1713
1714 self.assembler
1715 .emit_on_false_label(Location::GPR(fscr), end, jmp_tmp)?;
1716
1717 let cond = self.acquire_temp_gpr().ok_or_else(|| {
1719 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1720 })?;
1721 temps.push(cond);
1722
1723 self.assembler
1724 .emit_fcmp(Condition::Eq, sz, f, f, Location::GPR(cond))?;
1725 self.assembler
1727 .emit_on_false_label(Location::GPR(cond), trap_badconv, jmp_tmp)?;
1728 self.emit_illegal_op_internal(TrapCode::IntegerOverflow)?;
1729 self.emit_label(trap_badconv)?;
1730 self.emit_illegal_op_internal(TrapCode::BadConversionToInteger)?;
1731
1732 self.emit_label(end)?;
1733 self.assembler.emit_swap_fscr(old_fcsr)?;
1734
1735 Ok(())
1736 }
1737
1738 fn emit_illegal_op_internal(&mut self, trap: TrapCode) -> Result<(), CompileError> {
1739 self.assembler.emit_udf(trap as u8)
1740 }
1741
1742 fn emit_relaxed_fcmp(
1743 &mut self,
1744 c: Condition,
1745 size: Size,
1746 loc_a: Location,
1747 loc_b: Location,
1748 ret: Location,
1749 ) -> Result<(), CompileError> {
1750 let mut fprs = vec![];
1752 let mut gprs = vec![];
1753
1754 let loc_a = self.location_to_fpr(size, loc_a, &mut fprs, ImmType::None, true)?;
1755 let loc_b = self.location_to_fpr(size, loc_b, &mut fprs, ImmType::None, true)?;
1756 let dest = self.location_to_reg(size, ret, &mut gprs, ImmType::None, false, None)?;
1757
1758 self.assembler.emit_fcmp(c, size, loc_a, loc_b, dest)?;
1759 if ret != dest {
1760 self.move_location(size, dest, ret)?;
1761 }
1762 for r in fprs {
1763 self.release_simd(r);
1764 }
1765 for r in gprs {
1766 self.release_gpr(r);
1767 }
1768 Ok(())
1769 }
1770
1771 fn emit_relaxed_fcvt_with_rounding(
1772 &mut self,
1773 rounding: RoundingMode,
1774 size: Size,
1775 loc: Location,
1776 ret: Location,
1777 ) -> Result<(), CompileError> {
1778 let mut fprs = vec![];
1783 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
1784 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1785 })?;
1786
1787 let loc = self.location_to_fpr(size, loc, &mut fprs, ImmType::None, true)?;
1788 let dest = self.location_to_fpr(size, ret, &mut fprs, ImmType::None, false)?;
1789
1790 let cond = self.acquire_temp_gpr().ok_or_else(|| {
1791 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1792 })?;
1793 let tmp1 = self.acquire_temp_simd().ok_or_else(|| {
1794 CompileError::Codegen("singlepass cannot acquire temp fpr".to_owned())
1795 })?;
1796 let tmp2 = self.acquire_temp_simd().ok_or_else(|| {
1797 CompileError::Codegen("singlepass cannot acquire temp fpr".to_owned())
1798 })?;
1799
1800 let label_after = self.get_label();
1801
1802 let canonical_nan = match size {
1804 Size::S32 => CANONICAL_NAN_F32 as u64,
1805 Size::S64 => CANONICAL_NAN_F64,
1806 _ => unreachable!(),
1807 };
1808
1809 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
1810 CompileError::Codegen("singlepass cannot acquire temp fpr".to_owned())
1811 })?;
1812
1813 self.assembler
1814 .emit_mov_imm(Location::GPR(tmp), canonical_nan as _)?;
1815 self.assembler.emit_mov(size, Location::GPR(tmp), dest)?;
1816 self.assembler
1817 .emit_fcmp(Condition::Eq, size, loc, loc, Location::GPR(cond))?;
1818 self.assembler
1819 .emit_on_false_label(Location::GPR(cond), label_after, jmp_tmp)?;
1820
1821 if size == Size::S64 {
1823 self.assembler
1824 .emit_mov_imm(Location::GPR(cond), 0x4330000000000000)?;
1825 self.assembler
1826 .emit_mov(Size::S64, Location::GPR(cond), Location::SIMD(tmp1))?;
1827 self.f64_abs(loc, Location::SIMD(tmp2))?;
1828 } else {
1829 assert!(size == Size::S32);
1830 self.assembler
1831 .emit_mov_imm(Location::GPR(cond), 0x4b000000)?;
1832 self.assembler
1833 .emit_mov(Size::S32, Location::GPR(cond), Location::SIMD(tmp1))?;
1834 self.f32_abs(loc, Location::SIMD(tmp2))?;
1835 }
1836 self.emit_relaxed_fcmp(
1837 Condition::Lt,
1838 size,
1839 Location::SIMD(tmp2),
1840 Location::SIMD(tmp1),
1841 Location::GPR(cond),
1842 )?;
1843
1844 self.assembler.emit_mov(size, loc, dest)?;
1845 self.assembler
1846 .emit_on_false_label(Location::GPR(cond), label_after, jmp_tmp)?;
1847
1848 self.assembler
1850 .emit_fcvt_with_rounding(rounding, size, loc, dest, cond)?;
1851 self.emit_label(label_after)?;
1852
1853 if ret != dest {
1854 self.move_location(size, dest, ret)?;
1855 }
1856
1857 for r in fprs {
1858 self.release_simd(r);
1859 }
1860 self.release_gpr(jmp_tmp);
1861 self.release_gpr(tmp);
1862 self.release_gpr(cond);
1863 self.release_simd(tmp1);
1864 self.release_simd(tmp2);
1865 Ok(())
1866 }
1867
1868 fn emit_unwind_op(&mut self, op: UnwindOps<GPR, FPR>) {
1869 self.unwind_ops.push((self.get_offset().0, op));
1870 }
1871}
1872
1873pub(crate) const RISCV_RETURN_VALUE_REGISTERS: [GPR; 7] = [
1878 GPR::X10,
1879 GPR::X11,
1880 GPR::X12,
1881 GPR::X13,
1882 GPR::X14,
1883 GPR::X15,
1884 GPR::X16,
1885];
1886
1887#[allow(dead_code)]
1888#[derive(PartialEq, Copy, Clone)]
1889pub(crate) enum ImmType {
1890 None,
1891 Bits12,
1892 Bits12Subtraction,
1895 Shift32,
1896 Shift64,
1897}
1898
1899impl ImmType {
1900 pub(crate) fn compatible_imm(&self, imm: i64) -> bool {
1901 match self {
1902 ImmType::None => false,
1903 ImmType::Bits12 => (-0x800..0x800).contains(&imm),
1904 ImmType::Bits12Subtraction => (-0x801..0x801).contains(&imm),
1905 ImmType::Shift32 => (0..32).contains(&imm),
1906 ImmType::Shift64 => (0..64).contains(&imm),
1907 }
1908 }
1909}
1910
1911impl Machine for MachineRiscv {
1912 type GPR = GPR;
1913 type SIMD = FPR;
1914
1915 fn assembler_get_offset(&self) -> Offset {
1916 self.assembler.get_offset()
1917 }
1918
1919 fn get_vmctx_reg(&self) -> Self::GPR {
1920 GPR::X27
1922 }
1923
1924 fn pick_gpr(&self) -> Option<Self::GPR> {
1925 use GPR::*;
1926 static REGS: &[GPR] = &[X5, X6, X7, X29, X30, X31];
1928 for r in REGS {
1929 if !self.used_gprs_contains(r) {
1930 return Some(*r);
1931 }
1932 }
1933 None
1934 }
1935
1936 fn pick_temp_gpr(&self) -> Option<GPR> {
1937 use GPR::*;
1938 static REGS: &[GPR] = &[X17, X16, X15, X14, X13, X12, X11, X10];
1942 for r in REGS {
1943 if !self.used_gprs_contains(r) {
1944 return Some(*r);
1945 }
1946 }
1947 None
1948 }
1949
1950 fn get_used_gprs(&self) -> Vec<Self::GPR> {
1951 GPR::iterator()
1952 .filter(|x| self.used_gprs.contains(x.into_index()))
1953 .cloned()
1954 .collect()
1955 }
1956
1957 fn get_used_simd(&self) -> Vec<Self::SIMD> {
1958 FPR::iterator()
1959 .filter(|x| self.used_fprs.contains(x.into_index()))
1960 .cloned()
1961 .collect()
1962 }
1963
1964 fn acquire_temp_gpr(&mut self) -> Option<Self::GPR> {
1965 let gpr = self.pick_temp_gpr();
1966 if let Some(x) = gpr {
1967 self.used_gprs_insert(x);
1968 }
1969 gpr
1970 }
1971
1972 fn release_gpr(&mut self, gpr: Self::GPR) {
1973 assert!(self.used_gprs_remove(&gpr));
1974 }
1975
1976 fn reserve_unused_temp_gpr(&mut self, gpr: Self::GPR) -> Self::GPR {
1977 assert!(!self.used_gprs_contains(&gpr));
1978 self.used_gprs_insert(gpr);
1979 gpr
1980 }
1981
1982 fn reserve_gpr(&mut self, gpr: Self::GPR) {
1983 self.used_gprs_insert(gpr);
1984 }
1985
1986 fn push_used_gpr(&mut self, used_gprs: &[Self::GPR]) -> Result<usize, CompileError> {
1987 for r in used_gprs.iter() {
1988 self.assembler.emit_push(Size::S64, Location::GPR(*r))?;
1989 }
1990 Ok(used_gprs.len() * 16)
1991 }
1992
1993 fn pop_used_gpr(&mut self, used_gprs: &[Self::GPR]) -> Result<(), CompileError> {
1994 for r in used_gprs.iter().rev() {
1995 self.emit_pop(Size::S64, Location::GPR(*r))?;
1996 }
1997 Ok(())
1998 }
1999
2000 fn pick_simd(&self) -> Option<Self::SIMD> {
2001 use FPR::*;
2002 static REGS: &[FPR] = &[F0, F1, F2, F3, F4, F5, F6, F7];
2003 for r in REGS {
2004 if !self.used_fp_contains(r) {
2005 return Some(*r);
2006 }
2007 }
2008 None
2009 }
2010
2011 fn pick_temp_simd(&self) -> Option<FPR> {
2012 use FPR::*;
2013 static REGS: &[FPR] = &[F28, F29, F31];
2014 for r in REGS {
2015 if !self.used_fp_contains(r) {
2016 return Some(*r);
2017 }
2018 }
2019 None
2020 }
2021
2022 fn acquire_temp_simd(&mut self) -> Option<Self::SIMD> {
2023 let fpr = self.pick_temp_simd();
2024 if let Some(x) = fpr {
2025 self.used_fprs_insert(x);
2026 }
2027 fpr
2028 }
2029
2030 fn reserve_simd(&mut self, fpr: Self::SIMD) {
2031 self.used_fprs_insert(fpr);
2032 }
2033
2034 fn release_simd(&mut self, fpr: Self::SIMD) {
2035 assert!(self.used_fprs_remove(&fpr));
2036 }
2037
2038 fn push_used_simd(&mut self, used_neons: &[Self::SIMD]) -> Result<usize, CompileError> {
2039 let stack_adjust = (used_neons.len() * 8) as u32;
2040 self.extend_stack(stack_adjust)?;
2041
2042 for (i, r) in used_neons.iter().enumerate() {
2043 self.assembler.emit_sd(
2044 Size::S64,
2045 Location::SIMD(*r),
2046 Location::Memory(GPR::Sp, (i * 8) as i32),
2047 )?;
2048 }
2049 Ok(stack_adjust as usize)
2050 }
2051
2052 fn pop_used_simd(&mut self, used_neons: &[Self::SIMD]) -> Result<(), CompileError> {
2053 for (i, r) in used_neons.iter().enumerate() {
2054 self.assembler.emit_ld(
2055 Size::S64,
2056 false,
2057 Location::SIMD(*r),
2058 Location::Memory(GPR::Sp, (i * 8) as i32),
2059 )?;
2060 }
2061 let stack_adjust = (used_neons.len() * 8) as u32;
2062 self.assembler.emit_add(
2063 Size::S64,
2064 Location::GPR(GPR::Sp),
2065 Location::Imm64(stack_adjust as _),
2066 Location::GPR(GPR::Sp),
2067 )
2068 }
2069
2070 fn round_stack_adjust(&self, value: usize) -> usize {
2071 if value & 0xf != 0 {
2072 ((value >> 4) + 1) << 4
2073 } else {
2074 value
2075 }
2076 }
2077
2078 fn set_srcloc(&mut self, offset: u32) {
2079 self.src_loc = offset;
2080 }
2081
2082 fn mark_address_range_with_trap_code(&mut self, code: TrapCode, begin: usize, end: usize) {
2083 for i in begin..end {
2084 self.trap_table.offset_to_code.insert(i, code);
2085 }
2086 self.mark_instruction_address_end(begin);
2087 }
2088
2089 fn mark_address_with_trap_code(&mut self, code: TrapCode) {
2090 let offset = self.assembler.get_offset().0;
2091 self.trap_table.offset_to_code.insert(offset, code);
2092 self.mark_instruction_address_end(offset);
2093 }
2094
2095 fn mark_instruction_with_trap_code(&mut self, code: TrapCode) -> usize {
2096 let offset = self.assembler.get_offset().0;
2097 self.trap_table.offset_to_code.insert(offset, code);
2098 offset
2099 }
2100
2101 fn mark_instruction_address_end(&mut self, begin: usize) {
2102 self.instructions_address_map.push(InstructionAddressMap {
2103 srcloc: SourceLoc::new(self.src_loc),
2104 code_offset: begin,
2105 code_len: self.assembler.get_offset().0 - begin,
2106 });
2107 }
2108
2109 fn insert_stackoverflow(&mut self) {
2110 let offset = 0;
2111 self.trap_table
2112 .offset_to_code
2113 .insert(offset, TrapCode::StackOverflow);
2114 self.mark_instruction_address_end(offset);
2115 }
2116
2117 fn collect_trap_information(&self) -> Vec<TrapInformation> {
2118 self.trap_table
2119 .offset_to_code
2120 .clone()
2121 .into_iter()
2122 .map(|(offset, code)| TrapInformation {
2123 code_offset: offset as u32,
2124 trap_code: code,
2125 })
2126 .collect()
2127 }
2128
2129 fn instructions_address_map(&self) -> Vec<InstructionAddressMap> {
2130 self.instructions_address_map.clone()
2131 }
2132
2133 fn local_on_stack(&mut self, stack_offset: i32) -> Location {
2134 Location::Memory(GPR::Fp, -stack_offset)
2135 }
2136
2137 fn extend_stack(&mut self, delta_stack_offset: u32) -> Result<(), CompileError> {
2138 let delta = if ImmType::Bits12Subtraction.compatible_imm(delta_stack_offset as _) {
2139 Location::Imm64(delta_stack_offset as _)
2140 } else {
2141 self.assembler
2142 .emit_mov_imm(Location::GPR(SCRATCH_REG), delta_stack_offset as _)?;
2143 Location::GPR(SCRATCH_REG)
2144 };
2145 self.assembler.emit_sub(
2146 Size::S64,
2147 Location::GPR(GPR::Sp),
2148 delta,
2149 Location::GPR(GPR::Sp),
2150 )
2151 }
2152
2153 fn truncate_stack(&mut self, delta_stack_offset: u32) -> Result<(), CompileError> {
2154 let delta = if ImmType::Bits12.compatible_imm(delta_stack_offset as _) {
2155 Location::Imm64(delta_stack_offset as _)
2156 } else {
2157 self.assembler
2158 .emit_mov_imm(Location::GPR(SCRATCH_REG), delta_stack_offset as _)?;
2159 Location::GPR(SCRATCH_REG)
2160 };
2161 self.assembler.emit_add(
2162 Size::S64,
2163 Location::GPR(GPR::Sp),
2164 delta,
2165 Location::GPR(GPR::Sp),
2166 )
2167 }
2168
2169 fn zero_location(&mut self, size: Size, location: Location) -> Result<(), CompileError> {
2170 self.move_location(size, Location::GPR(GPR::XZero), location)
2171 }
2172
2173 fn local_pointer(&self) -> Self::GPR {
2174 GPR::Fp
2175 }
2176
2177 fn move_location_for_native(
2178 &mut self,
2179 _size: Size,
2180 loc: Location,
2181 dest: Location,
2182 ) -> Result<(), CompileError> {
2183 match loc {
2184 Location::Imm64(_)
2185 | Location::Imm32(_)
2186 | Location::Imm8(_)
2187 | Location::Memory(_, _) => {
2188 self.move_location(Size::S64, loc, Location::GPR(SCRATCH_REG))?;
2189 self.move_location(Size::S64, Location::GPR(SCRATCH_REG), dest)
2190 }
2191 _ => self.move_location(Size::S64, loc, dest),
2192 }
2193 }
2194
2195 fn is_local_on_stack(&self, idx: usize) -> bool {
2196 idx > 9
2197 }
2198
2199 fn get_local_location(&self, idx: usize, callee_saved_regs_size: usize) -> Location {
2200 match idx {
2202 0 => Location::GPR(GPR::X9),
2203 1 => Location::GPR(GPR::X18),
2204 2 => Location::GPR(GPR::X19),
2205 3 => Location::GPR(GPR::X20),
2206 4 => Location::GPR(GPR::X21),
2207 5 => Location::GPR(GPR::X22),
2208 6 => Location::GPR(GPR::X23),
2209 7 => Location::GPR(GPR::X24),
2210 8 => Location::GPR(GPR::X25),
2211 9 => Location::GPR(GPR::X26),
2212 _ => Location::Memory(GPR::Fp, -(((idx - 9) * 8 + callee_saved_regs_size) as i32)),
2213 }
2214 }
2215
2216 fn move_local(&mut self, stack_offset: i32, location: Location) -> Result<(), CompileError> {
2217 self.move_location(
2218 Size::S64,
2219 location,
2220 Location::Memory(GPR::Fp, -stack_offset),
2221 )?;
2222
2223 match location {
2224 Location::GPR(x) => self.emit_unwind_op(UnwindOps::SaveRegister {
2225 reg: UnwindRegister::GPR(x),
2226 bp_neg_offset: stack_offset,
2227 }),
2228 Location::SIMD(x) => self.emit_unwind_op(UnwindOps::SaveRegister {
2229 reg: UnwindRegister::FPR(x),
2230 bp_neg_offset: stack_offset,
2231 }),
2232 _ => (),
2233 }
2234 Ok(())
2235 }
2236
2237 fn list_to_save(&self, _calling_convention: CallingConvention) -> Vec<Location> {
2238 vec![]
2239 }
2240
2241 fn get_param_registers(&self, _calling_convention: CallingConvention) -> &'static [Self::GPR] {
2242 &[
2243 GPR::X10,
2244 GPR::X11,
2245 GPR::X12,
2246 GPR::X13,
2247 GPR::X14,
2248 GPR::X15,
2249 GPR::X16,
2250 GPR::X17,
2251 ]
2252 }
2253
2254 fn get_param_location(
2255 &self,
2256 idx: usize,
2257 _sz: Size,
2258 stack_args: &mut usize,
2259 calling_convention: CallingConvention,
2260 ) -> Location {
2261 let register_params: &[GPR] = self.get_param_registers(calling_convention);
2262 if let Some(reg) = register_params.get(idx) {
2263 Location::GPR(*reg)
2264 } else {
2265 let loc = Location::Memory(GPR::Sp, *stack_args as i32);
2266 *stack_args += 8;
2267 loc
2268 }
2269 }
2270
2271 fn get_call_param_location(
2272 &self,
2273 return_slots: usize,
2274 idx: usize,
2275 _sz: Size,
2276 stack_args: &mut usize,
2277 calling_convention: CallingConvention,
2278 ) -> Location {
2279 let return_values_memory_size =
2280 8 * return_slots.saturating_sub(RISCV_RETURN_VALUE_REGISTERS.len()) as i32;
2281 self.get_param_registers(calling_convention)
2282 .get(idx)
2283 .map_or_else(
2284 || {
2285 let loc = Location::Memory(
2286 GPR::Fp,
2287 16 + return_values_memory_size + *stack_args as i32,
2288 );
2289 *stack_args += 8;
2290 loc
2291 },
2292 |reg| Location::GPR(*reg),
2293 )
2294 }
2295
2296 fn get_simple_param_location(
2297 &self,
2298 idx: usize,
2299 calling_convention: CallingConvention,
2300 ) -> Self::GPR {
2301 self.get_param_registers(calling_convention)[idx]
2302 }
2303
2304 fn adjust_gpr_param_location(
2305 &mut self,
2306 register: Self::GPR,
2307 size: Size,
2308 ) -> Result<(), CompileError> {
2309 match size {
2314 Size::S64 => Ok(()),
2315 Size::S32 => self.assembler.emit_extend(
2316 Size::S32,
2317 true,
2318 Location::GPR(register),
2319 Location::GPR(register),
2320 ),
2321 Size::S8 | Size::S16 => {
2322 codegen_error!("singlepass adjust_gpr_param_location unreachable")
2323 }
2324 }
2325 }
2326
2327 fn get_return_value_location(
2328 &self,
2329 idx: usize,
2330 stack_location: &mut usize,
2331 _calling_convention: CallingConvention,
2332 ) -> AbstractLocation<Self::GPR, Self::SIMD> {
2333 RISCV_RETURN_VALUE_REGISTERS.get(idx).map_or_else(
2334 || {
2335 let loc = Location::Memory(GPR::Sp, *stack_location as i32);
2336 *stack_location += 8;
2337 loc
2338 },
2339 |reg| Location::GPR(*reg),
2340 )
2341 }
2342
2343 fn get_call_return_value_location(
2344 &self,
2345 idx: usize,
2346 _calling_convention: CallingConvention,
2347 ) -> AbstractLocation<Self::GPR, Self::SIMD> {
2348 RISCV_RETURN_VALUE_REGISTERS.get(idx).map_or_else(
2349 || {
2350 Location::Memory(
2351 GPR::Fp,
2352 (16 + (idx - RISCV_RETURN_VALUE_REGISTERS.len()) * 8) as i32,
2353 )
2354 },
2355 |reg| Location::GPR(*reg),
2356 )
2357 }
2358
2359 fn move_location(
2360 &mut self,
2361 size: Size,
2362 source: Location,
2363 dest: Location,
2364 ) -> Result<(), CompileError> {
2365 match (source, dest) {
2366 (Location::GPR(_), Location::GPR(_)) => self.assembler.emit_mov(size, source, dest),
2367 (Location::Imm32(_) | Location::Imm64(_), Location::GPR(_)) => self
2368 .assembler
2369 .emit_mov_imm(dest, source.imm_value_scalar().unwrap()),
2370 (Location::GPR(_), Location::Memory(addr, offset)) => {
2371 let addr = if ImmType::Bits12.compatible_imm(offset as _) {
2372 dest
2373 } else {
2374 self.assembler
2375 .emit_mov_imm(Location::GPR(SCRATCH_REG), offset as _)?;
2376 self.assembler.emit_add(
2377 Size::S64,
2378 Location::GPR(addr),
2379 Location::GPR(SCRATCH_REG),
2380 Location::GPR(SCRATCH_REG),
2381 )?;
2382 Location::Memory(SCRATCH_REG, 0)
2383 };
2384 self.assembler.emit_sd(size, source, addr)
2385 }
2386 (Location::SIMD(_), Location::Memory(..)) => {
2387 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2388 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2389 })?;
2390 self.move_location(size, source, Location::GPR(tmp))?;
2391 self.move_location(size, Location::GPR(tmp), dest)?;
2392 self.release_gpr(tmp);
2393 Ok(())
2394 }
2395 (Location::Memory(addr, offset), Location::GPR(_)) => {
2396 let addr = if ImmType::Bits12.compatible_imm(offset as _) {
2397 source
2398 } else {
2399 self.assembler
2400 .emit_mov_imm(Location::GPR(SCRATCH_REG), offset as _)?;
2401 self.assembler.emit_add(
2402 Size::S64,
2403 Location::GPR(addr),
2404 Location::GPR(SCRATCH_REG),
2405 Location::GPR(SCRATCH_REG),
2406 )?;
2407 Location::Memory(SCRATCH_REG, 0)
2408 };
2409 self.assembler.emit_ld(size, false, dest, addr)
2410 }
2411 (Location::GPR(_), Location::SIMD(_)) => self.assembler.emit_mov(size, source, dest),
2412 (Location::SIMD(_), Location::GPR(_)) => self.assembler.emit_mov(size, source, dest),
2413 (Location::SIMD(_), Location::SIMD(_)) => self.assembler.emit_mov(size, source, dest),
2414 _ => todo!("unsupported move: {size:?} {source:?} {dest:?}"),
2415 }
2416 }
2417
2418 fn move_location_extend(
2419 &mut self,
2420 size_val: Size,
2421 signed: bool,
2422 source: Location,
2423 size_op: Size,
2424 dest: Location,
2425 ) -> Result<(), CompileError> {
2426 if size_op != Size::S64 {
2427 codegen_error!("singlepass move_location_extend unreachable");
2428 }
2429 let mut temps = vec![];
2430 let dst = self.location_to_reg(size_op, dest, &mut temps, ImmType::None, false, None)?;
2431 let src = match (size_val, signed, source) {
2432 (Size::S64, _, _) => source,
2433 (_, _, Location::GPR(_)) => {
2434 self.assembler.emit_extend(size_val, signed, source, dst)?;
2435 dst
2436 }
2437 (_, _, Location::Memory(_, _)) => {
2438 self.assembler.emit_ld(size_val, signed, dst, source)?;
2439 dst
2440 }
2441 _ => codegen_error!(
2442 "singlepass can't emit move_location_extend {:?} {:?} {:?} => {:?} {:?}",
2443 size_val,
2444 signed,
2445 source,
2446 size_op,
2447 dest
2448 ),
2449 };
2450 if src != dst {
2451 self.move_location(size_op, src, dst)?;
2452 }
2453 if dst != dest {
2454 self.move_location(size_op, dst, dest)?;
2455 }
2456 for r in temps {
2457 self.release_gpr(r);
2458 }
2459 Ok(())
2460 }
2461
2462 fn init_stack_loc(
2463 &mut self,
2464 init_stack_loc_cnt: u64,
2465 last_stack_loc: Location,
2466 ) -> Result<(), CompileError> {
2467 let label = self.assembler.get_label();
2468 let mut temps = vec![];
2469 let dest = self.acquire_temp_gpr().ok_or_else(|| {
2470 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2471 })?;
2472 temps.push(dest);
2473 let cnt = self.location_to_reg(
2474 Size::S64,
2475 Location::Imm64(init_stack_loc_cnt),
2476 &mut temps,
2477 ImmType::None,
2478 true,
2479 None,
2480 )?;
2481 match last_stack_loc {
2482 Location::GPR(_) => codegen_error!("singlepass init_stack_loc unreachable"),
2483 Location::SIMD(_) => codegen_error!("singlepass init_stack_loc unreachable"),
2484 Location::Memory(reg, offset) => {
2485 if ImmType::Bits12.compatible_imm(offset as _) {
2486 self.assembler.emit_add(
2487 Size::S64,
2488 Location::GPR(reg),
2489 Location::Imm64(offset as _),
2490 Location::GPR(dest),
2491 )?;
2492 } else {
2493 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2494 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2495 })?;
2496 self.assembler
2497 .emit_mov_imm(Location::GPR(tmp), offset as _)?;
2498 self.assembler.emit_add(
2499 Size::S64,
2500 Location::GPR(reg),
2501 Location::GPR(tmp),
2502 Location::GPR(dest),
2503 )?;
2504 temps.push(tmp);
2505 }
2506 }
2507 _ => codegen_error!("singlepass can't emit init_stack_loc {:?}", last_stack_loc),
2508 };
2509 self.assembler.emit_label(label)?;
2510 self.assembler.emit_sd(
2511 Size::S64,
2512 Location::GPR(GPR::XZero),
2513 Location::Memory(dest, 0),
2514 )?;
2515 self.assembler
2516 .emit_sub(Size::S64, cnt, Location::Imm64(1), cnt)?;
2517 self.assembler.emit_add(
2518 Size::S64,
2519 Location::GPR(dest),
2520 Location::Imm64(8),
2521 Location::GPR(dest),
2522 )?;
2523 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
2524 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2525 })?;
2526 temps.push(jmp_tmp);
2527 self.assembler.emit_on_true_label(cnt, label, jmp_tmp)?;
2528 for r in temps {
2529 self.release_gpr(r);
2530 }
2531 Ok(())
2532 }
2533
2534 fn restore_saved_area(&mut self, saved_area_offset: i32) -> Result<(), CompileError> {
2535 self.assembler.emit_sub(
2536 Size::S64,
2537 Location::GPR(GPR::Fp),
2538 Location::Imm64(saved_area_offset as _),
2539 Location::GPR(GPR::Sp),
2540 )
2541 }
2542
2543 fn pop_location(&mut self, location: Location) -> Result<(), CompileError> {
2544 self.emit_pop(Size::S64, location)
2545 }
2546
2547 fn assembler_finalize(
2548 self,
2549 assembly_comments: HashMap<usize, AssemblyComment>,
2550 ) -> Result<FinalizedAssembly, CompileError> {
2551 Ok(FinalizedAssembly {
2552 body: self.assembler.finalize().map_err(|e| {
2553 CompileError::Codegen(format!("Assembler failed finalization with: {e:?}"))
2554 })?,
2555 assembly_comments,
2556 })
2557 }
2558
2559 fn get_offset(&self) -> Offset {
2560 self.assembler.get_offset()
2561 }
2562
2563 fn finalize_function(&mut self) -> Result<(), CompileError> {
2564 self.assembler.finalize_function()?;
2565 Ok(())
2566 }
2567
2568 fn emit_function_prolog(&mut self) -> Result<(), CompileError> {
2569 self.assembler.emit_sub(
2570 Size::S64,
2571 Location::GPR(GPR::Sp),
2572 Location::Imm64(16),
2573 Location::GPR(GPR::Sp),
2574 )?;
2575 self.emit_unwind_op(UnwindOps::SubtractFP { up_to_sp: 16 });
2576
2577 self.assembler.emit_sd(
2578 Size::S64,
2579 Location::GPR(GPR::X1), Location::Memory(GPR::Sp, 8),
2581 )?;
2582 self.assembler.emit_sd(
2583 Size::S64,
2584 Location::GPR(GPR::Fp),
2585 Location::Memory(GPR::Sp, 0),
2586 )?;
2587 self.emit_unwind_op(UnwindOps::SaveRegister {
2588 reg: UnwindRegister::GPR(GPR::X1),
2589 bp_neg_offset: 8,
2590 });
2591 self.emit_unwind_op(UnwindOps::SaveRegister {
2592 reg: UnwindRegister::GPR(GPR::Fp),
2593 bp_neg_offset: 16,
2594 });
2595
2596 self.assembler
2597 .emit_mov(Size::S64, Location::GPR(GPR::Sp), Location::GPR(GPR::Fp))?;
2598 self.emit_unwind_op(UnwindOps::DefineNewFrame);
2599 Ok(())
2600 }
2601
2602 fn emit_function_epilog(&mut self) -> Result<(), CompileError> {
2603 self.assembler
2604 .emit_mov(Size::S64, Location::GPR(GPR::Fp), Location::GPR(GPR::Sp))?;
2605 self.assembler.emit_ld(
2606 Size::S64,
2607 false,
2608 Location::GPR(GPR::X1), Location::Memory(GPR::Sp, 8),
2610 )?;
2611 self.assembler.emit_ld(
2612 Size::S64,
2613 false,
2614 Location::GPR(GPR::Fp),
2615 Location::Memory(GPR::Sp, 0),
2616 )?;
2617 self.assembler.emit_add(
2618 Size::S64,
2619 Location::GPR(GPR::Sp),
2620 Location::Imm64(16),
2621 Location::GPR(GPR::Sp),
2622 )?;
2623
2624 Ok(())
2625 }
2626
2627 fn emit_function_return_float(&mut self) -> Result<(), CompileError> {
2628 self.assembler
2629 .emit_mov(Size::S64, Location::GPR(GPR::X10), Location::SIMD(FPR::F10))
2630 }
2631
2632 fn canonicalize_nan(
2633 &mut self,
2634 sz: Size,
2635 input: Location,
2636 output: Location,
2637 ) -> Result<(), CompileError> {
2638 let mut temps = vec![];
2639 match (sz, input, output) {
2641 (Size::S32, Location::SIMD(_), Location::SIMD(_)) => {
2642 self.assembler.emit_fmax(sz, input, input, output)?;
2643 }
2644 (Size::S64, Location::SIMD(_), Location::SIMD(_)) => {
2645 self.assembler.emit_fmax(sz, input, input, output)?;
2646 }
2647 (Size::S32, Location::SIMD(_), _) | (Size::S64, Location::SIMD(_), _) => {
2648 let tmp = self.location_to_fpr(sz, output, &mut temps, ImmType::None, false)?;
2649 self.assembler.emit_fmax(sz, input, input, tmp)?;
2650 self.move_location(sz, tmp, output)?;
2651 }
2652 (Size::S32, Location::Memory(_, _), _) | (Size::S64, Location::Memory(_, _), _) => {
2653 let src = self.location_to_fpr(sz, input, &mut temps, ImmType::None, true)?;
2654 let tmp = self.location_to_fpr(sz, output, &mut temps, ImmType::None, false)?;
2655 self.assembler.emit_fmax(sz, src, src, tmp)?;
2656 if tmp != output {
2657 self.move_location(sz, tmp, output)?;
2658 }
2659 }
2660 _ => codegen_error!(
2661 "singlepass can't emit canonicalize_nan {:?} {:?} {:?}",
2662 sz,
2663 input,
2664 output
2665 ),
2666 }
2667
2668 for r in temps {
2669 self.release_simd(r);
2670 }
2671 Ok(())
2672 }
2673
2674 fn emit_illegal_op(&mut self, trap: TrapCode) -> Result<(), CompileError> {
2675 let offset = self.assembler.get_offset().0;
2676 self.assembler.emit_udf(trap as u8)?;
2677 self.mark_instruction_address_end(offset);
2678 Ok(())
2679 }
2680
2681 fn get_label(&mut self) -> Label {
2682 self.assembler.new_dynamic_label()
2683 }
2684
2685 fn emit_label(&mut self, label: Label) -> Result<(), CompileError> {
2686 self.assembler.emit_label(label)
2687 }
2688
2689 fn get_gpr_for_call(&self) -> Self::GPR {
2690 GPR::X1
2691 }
2692
2693 fn emit_call_register(&mut self, register: Self::GPR) -> Result<(), CompileError> {
2694 self.assembler.emit_call_register(register)
2695 }
2696
2697 fn emit_call_label(&mut self, label: Label) -> Result<(), CompileError> {
2698 self.assembler.emit_call_label(label)
2699 }
2700
2701 fn arch_emit_indirect_call_with_trampoline(
2702 &mut self,
2703 _location: Location,
2704 ) -> Result<(), CompileError> {
2705 codegen_error!("singlepass arch_emit_indirect_call_with_trampoline unimplemented")
2706 }
2707
2708 fn emit_call_location(&mut self, location: Location) -> Result<(), CompileError> {
2709 let mut temps = vec![];
2710 let loc = self.location_to_reg(
2711 Size::S64,
2712 location,
2713 &mut temps,
2714 ImmType::None,
2715 true,
2716 Some(self.get_gpr_for_call()),
2717 )?;
2718 match loc {
2719 Location::GPR(reg) => self.assembler.emit_call_register(reg),
2720 _ => codegen_error!("singlepass can't emit CALL Location"),
2721 }?;
2722 for r in temps {
2723 self.release_gpr(r);
2724 }
2725 Ok(())
2726 }
2727
2728 fn emit_debug_breakpoint(&mut self) -> Result<(), CompileError> {
2729 self.assembler.emit_brk()
2730 }
2731
2732 fn location_add(
2733 &mut self,
2734 size: Size,
2735 source: Location,
2736 dest: Location,
2737 _flags: bool,
2738 ) -> Result<(), CompileError> {
2739 let mut temps = vec![];
2740 let src = self.location_to_reg(size, source, &mut temps, ImmType::Bits12, true, None)?;
2741 let dst = self.location_to_reg(size, dest, &mut temps, ImmType::None, true, None)?;
2742 self.assembler.emit_add(size, dst, src, dst)?;
2743 if dst != dest {
2744 self.move_location(size, dst, dest)?;
2745 }
2746 for r in temps {
2747 self.release_gpr(r);
2748 }
2749 Ok(())
2750 }
2751
2752 fn location_cmp(
2753 &mut self,
2754 _size: Size,
2755 _source: Location,
2756 _dest: Location,
2757 ) -> Result<(), CompileError> {
2758 codegen_error!("singlepass location_cmp not implemented")
2759 }
2760
2761 fn jmp_unconditional(&mut self, label: Label) -> Result<(), CompileError> {
2762 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2763 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2764 })?;
2765 self.assembler.emit_j_label(label, Some(tmp))?;
2766 self.release_gpr(tmp);
2767 Ok(())
2768 }
2769
2770 fn jmp_on_condition(
2771 &mut self,
2772 cond: UnsignedCondition,
2773 size: Size,
2774 loc_a: AbstractLocation<Self::GPR, Self::SIMD>,
2775 loc_b: AbstractLocation<Self::GPR, Self::SIMD>,
2776 label: Label,
2777 ) -> Result<(), CompileError> {
2778 let c = match cond {
2779 UnsignedCondition::Equal => Condition::Eq,
2780 UnsignedCondition::NotEqual => Condition::Ne,
2781 UnsignedCondition::Above => Condition::Gtu,
2782 UnsignedCondition::AboveEqual => Condition::Geu,
2783 UnsignedCondition::Below => Condition::Ltu,
2784 UnsignedCondition::BelowEqual => Condition::Leu,
2785 };
2786
2787 let mut temps = vec![];
2788 let loc_a = self.location_to_reg(size, loc_a, &mut temps, ImmType::None, true, None)?;
2789 let loc_b = self.location_to_reg(size, loc_b, &mut temps, ImmType::None, true, None)?;
2790
2791 if size != Size::S64 {
2792 self.assembler.emit_extend(size, false, loc_a, loc_a)?;
2793 self.assembler.emit_extend(size, false, loc_b, loc_b)?;
2794 }
2795
2796 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2797 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2798 })?;
2799 temps.push(tmp);
2800 self.assembler
2801 .emit_jmp_on_condition(c, loc_a, loc_b, label, tmp)?;
2802
2803 for r in temps {
2804 self.release_gpr(r);
2805 }
2806 Ok(())
2807 }
2808
2809 fn emit_jmp_to_jumptable(&mut self, label: Label, cond: Location) -> Result<(), CompileError> {
2810 let tmp1 = self.acquire_temp_gpr().ok_or_else(|| {
2811 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2812 })?;
2813 let tmp2 = self.acquire_temp_gpr().ok_or_else(|| {
2814 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2815 })?;
2816
2817 self.assembler.emit_load_label(tmp1, label)?;
2818 self.move_location(Size::S32, cond, Location::GPR(tmp2))?;
2819
2820 self.assembler.emit_sll(
2823 Size::S32,
2824 Location::GPR(tmp2),
2825 Location::Imm32(3),
2826 Location::GPR(tmp2),
2827 )?;
2828 self.assembler.emit_add(
2829 Size::S64,
2830 Location::GPR(tmp1),
2831 Location::GPR(tmp2),
2832 Location::GPR(tmp2),
2833 )?;
2834 self.assembler.emit_j_register(tmp2)?;
2835 self.release_gpr(tmp2);
2836 self.release_gpr(tmp1);
2837 Ok(())
2838 }
2839
2840 fn align_for_loop(&mut self) -> Result<(), CompileError> {
2841 Ok(())
2843 }
2844
2845 fn emit_ret(&mut self) -> Result<(), CompileError> {
2846 self.assembler.emit_ret()
2847 }
2848
2849 fn emit_push(&mut self, size: Size, loc: Location) -> Result<(), CompileError> {
2850 self.assembler.emit_push(size, loc)
2851 }
2852
2853 fn emit_pop(&mut self, size: Size, loc: Location) -> Result<(), CompileError> {
2854 self.assembler.emit_pop(size, loc)
2855 }
2856
2857 fn emit_relaxed_mov(
2858 &mut self,
2859 sz: Size,
2860 src: Location,
2861 dst: Location,
2862 ) -> Result<(), CompileError> {
2863 self.emit_relaxed_binop(Assembler::emit_mov, sz, src, dst)
2864 }
2865
2866 fn emit_relaxed_cmp(
2867 &mut self,
2868 _sz: Size,
2869 _src: Location,
2870 _dst: Location,
2871 ) -> Result<(), CompileError> {
2872 todo!();
2873 }
2874
2875 fn emit_memory_fence(&mut self) -> Result<(), CompileError> {
2876 self.assembler.emit_rwfence()
2877 }
2878
2879 fn emit_relaxed_sign_extension(
2880 &mut self,
2881 sz_src: Size,
2882 src: Location,
2883 sz_dst: Size,
2884 dst: Location,
2885 ) -> Result<(), CompileError> {
2886 let mut temps = vec![];
2887
2888 match (src, dst) {
2889 (Location::Memory(reg, offset), Location::GPR(_)) => {
2890 let src = if ImmType::Bits12.compatible_imm(offset as _) {
2891 src
2892 } else {
2893 let tmp =
2894 self.location_to_reg(sz_src, src, &mut temps, ImmType::None, true, None)?;
2895 self.assembler.emit_mov_imm(tmp, offset as _)?;
2896 self.assembler
2897 .emit_add(Size::S64, Location::GPR(reg), tmp, tmp)?;
2898 let Location::GPR(tmp) = tmp else {
2899 unreachable!()
2900 };
2901 Location::Memory(tmp, 0)
2902 };
2903 self.assembler.emit_ld(sz_src, true, dst, src)?;
2904 }
2905 _ => {
2906 let src =
2907 self.location_to_reg(sz_src, src, &mut temps, ImmType::None, true, None)?;
2908 let dest =
2909 self.location_to_reg(sz_dst, dst, &mut temps, ImmType::None, false, None)?;
2910 self.assembler.emit_extend(sz_src, true, src, dest)?;
2911 if dst != dest {
2912 self.move_location(sz_dst, dest, dst)?;
2913 }
2914 }
2915 }
2916
2917 for r in temps {
2918 self.release_gpr(r);
2919 }
2920
2921 Ok(())
2922 }
2923
2924 fn emit_imul_imm32(
2925 &mut self,
2926 size: Size,
2927 imm32: u32,
2928 gpr: Self::GPR,
2929 ) -> Result<(), CompileError> {
2930 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2931 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2932 })?;
2933 self.assembler
2934 .emit_mov_imm(Location::GPR(tmp), imm32 as _)?;
2935 self.assembler.emit_mul(
2936 size,
2937 Location::GPR(gpr),
2938 Location::GPR(tmp),
2939 Location::GPR(gpr),
2940 )?;
2941 self.release_gpr(tmp);
2942 Ok(())
2943 }
2944
2945 fn emit_binop_add32(
2946 &mut self,
2947 loc_a: Location,
2948 loc_b: Location,
2949 ret: Location,
2950 ) -> Result<(), CompileError> {
2951 self.emit_relaxed_binop3(
2952 Assembler::emit_add,
2953 Size::S32,
2954 loc_a,
2955 loc_b,
2956 ret,
2957 ImmType::Bits12,
2958 )
2959 }
2960
2961 fn emit_binop_sub32(
2962 &mut self,
2963 loc_a: Location,
2964 loc_b: Location,
2965 ret: Location,
2966 ) -> Result<(), CompileError> {
2967 self.emit_relaxed_binop3(
2968 Assembler::emit_sub,
2969 Size::S32,
2970 loc_a,
2971 loc_b,
2972 ret,
2973 ImmType::Bits12Subtraction,
2974 )
2975 }
2976
2977 fn emit_binop_mul32(
2978 &mut self,
2979 loc_a: Location,
2980 loc_b: Location,
2981 ret: Location,
2982 ) -> Result<(), CompileError> {
2983 self.emit_relaxed_binop3(
2984 Assembler::emit_mul,
2985 Size::S32,
2986 loc_a,
2987 loc_b,
2988 ret,
2989 ImmType::None,
2990 )
2991 }
2992
2993 fn emit_binop_udiv32(
2994 &mut self,
2995 loc_a: Location,
2996 loc_b: Location,
2997 ret: Location,
2998 integer_division_by_zero: Label,
2999 ) -> Result<usize, CompileError> {
3000 let mut temps = vec![];
3001 let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
3002 let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
3003 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3004
3005 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
3006 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3007 })?;
3008 temps.push(jmp_tmp);
3009
3010 self.assembler
3011 .emit_on_false_label_far(src2, integer_division_by_zero, jmp_tmp)?;
3012 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
3013 self.assembler.emit_udiv(Size::S32, src1, src2, dest)?;
3014 if ret != dest {
3015 self.move_location(Size::S32, dest, ret)?;
3016 }
3017 for r in temps {
3018 self.release_gpr(r);
3019 }
3020 Ok(offset)
3021 }
3022
3023 fn emit_binop_sdiv32(
3024 &mut self,
3025 loc_a: Location,
3026 loc_b: Location,
3027 ret: Location,
3028 integer_division_by_zero: Label,
3029 integer_overflow: Label,
3030 ) -> Result<usize, CompileError> {
3031 let mut temps = vec![];
3032 let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
3033 let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
3034 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3035
3036 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
3037 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3038 })?;
3039 temps.push(jmp_tmp);
3040
3041 self.assembler
3042 .emit_on_false_label_far(src2, integer_division_by_zero, jmp_tmp)?;
3043 let label_nooverflow = self.assembler.get_label();
3044 let tmp = self.location_to_reg(
3045 Size::S32,
3046 Location::Imm32(i32::MIN as u32),
3047 &mut temps,
3048 ImmType::None,
3049 true,
3050 None,
3051 )?;
3052 self.assembler.emit_cmp(Condition::Ne, tmp, src1, tmp)?;
3053 self.assembler
3054 .emit_on_true_label(tmp, label_nooverflow, jmp_tmp)?;
3055 self.move_location(Size::S32, Location::Imm32(-1i32 as _), tmp)?;
3056 self.assembler.emit_cmp(Condition::Eq, tmp, src2, tmp)?;
3057 self.assembler
3058 .emit_on_true_label_far(tmp, integer_overflow, jmp_tmp)?;
3059 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
3060 self.assembler.emit_label(label_nooverflow)?;
3061 self.assembler.emit_sdiv(Size::S32, src1, src2, dest)?;
3062 if ret != dest {
3063 self.move_location(Size::S32, dest, ret)?;
3064 }
3065 for r in temps {
3066 self.release_gpr(r);
3067 }
3068 Ok(offset)
3069 }
3070
3071 fn emit_binop_urem32(
3072 &mut self,
3073 loc_a: Location,
3074 loc_b: Location,
3075 ret: Location,
3076 integer_division_by_zero: Label,
3077 ) -> Result<usize, CompileError> {
3078 let mut temps = vec![];
3079 let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
3080 let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
3081 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3082
3083 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
3084 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3085 })?;
3086 temps.push(jmp_tmp);
3087
3088 self.assembler
3089 .emit_on_false_label_far(src2, integer_division_by_zero, jmp_tmp)?;
3090 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
3091 self.assembler.emit_urem(Size::S32, src1, src2, dest)?;
3092 if ret != dest {
3093 self.move_location(Size::S32, dest, ret)?;
3094 }
3095 for r in temps {
3096 self.release_gpr(r);
3097 }
3098 Ok(offset)
3099 }
3100
3101 fn emit_binop_srem32(
3102 &mut self,
3103 loc_a: Location,
3104 loc_b: Location,
3105 ret: Location,
3106 integer_division_by_zero: Label,
3107 ) -> Result<usize, CompileError> {
3108 let mut temps = vec![];
3109 let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
3110 let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
3111 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3112
3113 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
3114 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3115 })?;
3116 temps.push(jmp_tmp);
3117
3118 self.assembler
3119 .emit_on_false_label_far(src2, integer_division_by_zero, jmp_tmp)?;
3120 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
3121 self.assembler.emit_srem(Size::S32, src1, src2, dest)?;
3122 if ret != dest {
3123 self.move_location(Size::S32, dest, ret)?;
3124 }
3125 for r in temps {
3126 self.release_gpr(r);
3127 }
3128 Ok(offset)
3129 }
3130
3131 fn emit_binop_and32(
3132 &mut self,
3133 loc_a: Location,
3134 loc_b: Location,
3135 ret: Location,
3136 ) -> Result<(), CompileError> {
3137 self.emit_relaxed_binop3(
3138 Assembler::emit_and,
3139 Size::S32,
3140 loc_a,
3141 loc_b,
3142 ret,
3143 ImmType::Bits12,
3144 )
3145 }
3146
3147 fn emit_binop_or32(
3148 &mut self,
3149 loc_a: Location,
3150 loc_b: Location,
3151 ret: Location,
3152 ) -> Result<(), CompileError> {
3153 self.emit_relaxed_binop3(
3154 Assembler::emit_or,
3155 Size::S32,
3156 loc_a,
3157 loc_b,
3158 ret,
3159 ImmType::Bits12,
3160 )
3161 }
3162
3163 fn emit_binop_xor32(
3164 &mut self,
3165 loc_a: Location,
3166 loc_b: Location,
3167 ret: Location,
3168 ) -> Result<(), CompileError> {
3169 self.emit_relaxed_binop3(
3170 Assembler::emit_xor,
3171 Size::S32,
3172 loc_a,
3173 loc_b,
3174 ret,
3175 ImmType::Bits12,
3176 )
3177 }
3178
3179 fn i32_cmp_ge_s(
3180 &mut self,
3181 loc_a: Location,
3182 loc_b: Location,
3183 ret: Location,
3184 ) -> Result<(), CompileError> {
3185 self.emit_cmpop_i32_dynamic_b(Condition::Ge, loc_a, loc_b, ret, true)
3186 }
3187
3188 fn i32_cmp_gt_s(
3189 &mut self,
3190 loc_a: Location,
3191 loc_b: Location,
3192 ret: Location,
3193 ) -> Result<(), CompileError> {
3194 self.emit_cmpop_i32_dynamic_b(Condition::Gt, loc_a, loc_b, ret, true)
3195 }
3196
3197 fn i32_cmp_le_s(
3198 &mut self,
3199 loc_a: Location,
3200 loc_b: Location,
3201 ret: Location,
3202 ) -> Result<(), CompileError> {
3203 self.emit_cmpop_i32_dynamic_b(Condition::Le, loc_a, loc_b, ret, true)
3204 }
3205
3206 fn i32_cmp_lt_s(
3207 &mut self,
3208 loc_a: Location,
3209 loc_b: Location,
3210 ret: Location,
3211 ) -> Result<(), CompileError> {
3212 self.emit_cmpop_i32_dynamic_b(Condition::Lt, loc_a, loc_b, ret, true)
3213 }
3214
3215 fn i32_cmp_ge_u(
3216 &mut self,
3217 loc_a: Location,
3218 loc_b: Location,
3219 ret: Location,
3220 ) -> Result<(), CompileError> {
3221 self.emit_cmpop_i32_dynamic_b(Condition::Geu, loc_a, loc_b, ret, false)
3222 }
3223
3224 fn i32_cmp_gt_u(
3225 &mut self,
3226 loc_a: Location,
3227 loc_b: Location,
3228 ret: Location,
3229 ) -> Result<(), CompileError> {
3230 self.emit_cmpop_i32_dynamic_b(Condition::Gtu, loc_a, loc_b, ret, false)
3231 }
3232
3233 fn i32_cmp_le_u(
3234 &mut self,
3235 loc_a: Location,
3236 loc_b: Location,
3237 ret: Location,
3238 ) -> Result<(), CompileError> {
3239 self.emit_cmpop_i32_dynamic_b(Condition::Leu, loc_a, loc_b, ret, false)
3240 }
3241
3242 fn i32_cmp_lt_u(
3243 &mut self,
3244 loc_a: Location,
3245 loc_b: Location,
3246 ret: Location,
3247 ) -> Result<(), CompileError> {
3248 self.emit_cmpop_i32_dynamic_b(Condition::Ltu, loc_a, loc_b, ret, false)
3249 }
3250
3251 fn i32_cmp_ne(
3252 &mut self,
3253 loc_a: Location,
3254 loc_b: Location,
3255 ret: Location,
3256 ) -> Result<(), CompileError> {
3257 self.emit_cmpop_i32_dynamic_b(Condition::Ne, loc_a, loc_b, ret, true)
3258 }
3259
3260 fn i32_cmp_eq(
3261 &mut self,
3262 loc_a: Location,
3263 loc_b: Location,
3264 ret: Location,
3265 ) -> Result<(), CompileError> {
3266 self.emit_cmpop_i32_dynamic_b(Condition::Eq, loc_a, loc_b, ret, true)
3267 }
3268
3269 fn i32_clz(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
3270 self.emit_clz(Size::S32, loc, ret)
3271 }
3272
3273 fn i32_ctz(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
3274 self.emit_ctz(Size::S32, loc, ret)
3275 }
3276
3277 fn i32_popcnt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
3278 self.emit_popcnt(Size::S32, loc, ret)
3279 }
3280
3281 fn i32_shl(
3282 &mut self,
3283 loc_a: Location,
3284 loc_b: Location,
3285 ret: Location,
3286 ) -> Result<(), CompileError> {
3287 self.emit_relaxed_binop3(
3288 Assembler::emit_sll,
3289 Size::S32,
3290 loc_a,
3291 loc_b,
3292 ret,
3293 ImmType::Shift32,
3294 )
3295 }
3296
3297 fn i32_shr(
3298 &mut self,
3299 loc_a: Location,
3300 loc_b: Location,
3301 ret: Location,
3302 ) -> Result<(), CompileError> {
3303 self.emit_relaxed_binop3(
3304 Assembler::emit_srl,
3305 Size::S32,
3306 loc_a,
3307 loc_b,
3308 ret,
3309 ImmType::Shift32,
3310 )
3311 }
3312
3313 fn i32_sar(
3314 &mut self,
3315 loc_a: Location,
3316 loc_b: Location,
3317 ret: Location,
3318 ) -> Result<(), CompileError> {
3319 self.emit_relaxed_binop3(
3320 Assembler::emit_sra,
3321 Size::S32,
3322 loc_a,
3323 loc_b,
3324 ret,
3325 ImmType::Shift32,
3326 )
3327 }
3328
3329 fn i32_rol(
3330 &mut self,
3331 loc_a: Location,
3332 loc_b: Location,
3333 ret: Location,
3334 ) -> Result<(), CompileError> {
3335 self.emit_rol(Size::S32, loc_a, loc_b, ret, ImmType::Shift32)
3336 }
3337
3338 fn i32_ror(
3339 &mut self,
3340 loc_a: Location,
3341 loc_b: Location,
3342 ret: Location,
3343 ) -> Result<(), CompileError> {
3344 self.emit_ror(Size::S32, loc_a, loc_b, ret, ImmType::Shift32)
3345 }
3346
3347 fn i32_load(
3348 &mut self,
3349 addr: Location,
3350 memarg: &MemArg,
3351 ret: Location,
3352 _need_check: bool,
3353 imported_memories: bool,
3354 offset: i32,
3355 heap_access_oob: Label,
3356 unaligned_atomic: Label,
3357 ) -> Result<(), CompileError> {
3358 self.memory_op(
3359 addr,
3360 memarg,
3361 false,
3362 4,
3363 imported_memories,
3364 offset,
3365 heap_access_oob,
3366 unaligned_atomic,
3367 |this, addr| this.emit_maybe_unaligned_load(Size::S32, true, ret, addr),
3368 )
3369 }
3370
3371 fn i32_load_8u(
3372 &mut self,
3373 addr: Location,
3374 memarg: &MemArg,
3375 ret: Location,
3376 _need_check: bool,
3377 imported_memories: bool,
3378 offset: i32,
3379 heap_access_oob: Label,
3380 unaligned_atomic: Label,
3381 ) -> Result<(), CompileError> {
3382 self.memory_op(
3383 addr,
3384 memarg,
3385 false,
3386 1,
3387 imported_memories,
3388 offset,
3389 heap_access_oob,
3390 unaligned_atomic,
3391 |this, addr| this.emit_maybe_unaligned_load(Size::S8, false, ret, addr),
3392 )
3393 }
3394
3395 fn i32_load_8s(
3396 &mut self,
3397 addr: Location,
3398 memarg: &MemArg,
3399 ret: Location,
3400 _need_check: bool,
3401 imported_memories: bool,
3402 offset: i32,
3403 heap_access_oob: Label,
3404 unaligned_atomic: Label,
3405 ) -> Result<(), CompileError> {
3406 self.memory_op(
3407 addr,
3408 memarg,
3409 false,
3410 1,
3411 imported_memories,
3412 offset,
3413 heap_access_oob,
3414 unaligned_atomic,
3415 |this, addr| this.emit_maybe_unaligned_load(Size::S8, true, ret, addr),
3416 )
3417 }
3418
3419 fn i32_load_16u(
3420 &mut self,
3421 addr: Location,
3422 memarg: &MemArg,
3423 ret: Location,
3424 _need_check: bool,
3425 imported_memories: bool,
3426 offset: i32,
3427 heap_access_oob: Label,
3428 unaligned_atomic: Label,
3429 ) -> Result<(), CompileError> {
3430 self.memory_op(
3431 addr,
3432 memarg,
3433 false,
3434 2,
3435 imported_memories,
3436 offset,
3437 heap_access_oob,
3438 unaligned_atomic,
3439 |this, addr| this.emit_maybe_unaligned_load(Size::S16, false, ret, addr),
3440 )
3441 }
3442
3443 fn i32_load_16s(
3444 &mut self,
3445 addr: Location,
3446 memarg: &MemArg,
3447 ret: Location,
3448 _need_check: bool,
3449 imported_memories: bool,
3450 offset: i32,
3451 heap_access_oob: Label,
3452 unaligned_atomic: Label,
3453 ) -> Result<(), CompileError> {
3454 self.memory_op(
3455 addr,
3456 memarg,
3457 false,
3458 2,
3459 imported_memories,
3460 offset,
3461 heap_access_oob,
3462 unaligned_atomic,
3463 |this, addr| this.emit_maybe_unaligned_load(Size::S16, true, ret, addr),
3464 )
3465 }
3466
3467 fn i32_atomic_load(
3468 &mut self,
3469 addr: Location,
3470 memarg: &MemArg,
3471 ret: Location,
3472 _need_check: bool,
3473 imported_memories: bool,
3474 offset: i32,
3475 heap_access_oob: Label,
3476 unaligned_atomic: Label,
3477 ) -> Result<(), CompileError> {
3478 self.memory_op(
3479 addr,
3480 memarg,
3481 true,
3482 4,
3483 imported_memories,
3484 offset,
3485 heap_access_oob,
3486 unaligned_atomic,
3487 |this, addr| this.emit_relaxed_load(Size::S32, true, ret, Location::Memory(addr, 0)),
3488 )
3489 }
3490
3491 fn i32_atomic_load_8u(
3492 &mut self,
3493 addr: Location,
3494 memarg: &MemArg,
3495 ret: Location,
3496 _need_check: bool,
3497 imported_memories: bool,
3498 offset: i32,
3499 heap_access_oob: Label,
3500 unaligned_atomic: Label,
3501 ) -> Result<(), CompileError> {
3502 self.memory_op(
3503 addr,
3504 memarg,
3505 true,
3506 1,
3507 imported_memories,
3508 offset,
3509 heap_access_oob,
3510 unaligned_atomic,
3511 |this, addr| this.emit_relaxed_load(Size::S8, false, ret, Location::Memory(addr, 0)),
3512 )
3513 }
3514
3515 fn i32_atomic_load_16u(
3516 &mut self,
3517 addr: Location,
3518 memarg: &MemArg,
3519 ret: Location,
3520 _need_check: bool,
3521 imported_memories: bool,
3522 offset: i32,
3523 heap_access_oob: Label,
3524 unaligned_atomic: Label,
3525 ) -> Result<(), CompileError> {
3526 self.memory_op(
3527 addr,
3528 memarg,
3529 true,
3530 2,
3531 imported_memories,
3532 offset,
3533 heap_access_oob,
3534 unaligned_atomic,
3535 |this, addr| this.emit_relaxed_load(Size::S16, false, ret, Location::Memory(addr, 0)),
3536 )
3537 }
3538
3539 fn i32_save(
3540 &mut self,
3541 value: Location,
3542 memarg: &MemArg,
3543 addr: Location,
3544 _need_check: bool,
3545 imported_memories: bool,
3546 offset: i32,
3547 heap_access_oob: Label,
3548 unaligned_atomic: Label,
3549 ) -> Result<(), CompileError> {
3550 self.memory_op(
3551 addr,
3552 memarg,
3553 false,
3554 4,
3555 imported_memories,
3556 offset,
3557 heap_access_oob,
3558 unaligned_atomic,
3559 |this, addr| this.emit_maybe_unaligned_store(Size::S32, value, addr),
3560 )
3561 }
3562
3563 fn i32_save_8(
3564 &mut self,
3565 value: Location,
3566 memarg: &MemArg,
3567 addr: Location,
3568 _need_check: bool,
3569 imported_memories: bool,
3570 offset: i32,
3571 heap_access_oob: Label,
3572 unaligned_atomic: Label,
3573 ) -> Result<(), CompileError> {
3574 self.memory_op(
3575 addr,
3576 memarg,
3577 false,
3578 1,
3579 imported_memories,
3580 offset,
3581 heap_access_oob,
3582 unaligned_atomic,
3583 |this, addr| this.emit_maybe_unaligned_store(Size::S8, value, addr),
3584 )
3585 }
3586
3587 fn i32_save_16(
3588 &mut self,
3589 value: Location,
3590 memarg: &MemArg,
3591 addr: Location,
3592 _need_check: bool,
3593 imported_memories: bool,
3594 offset: i32,
3595 heap_access_oob: Label,
3596 unaligned_atomic: Label,
3597 ) -> Result<(), CompileError> {
3598 self.memory_op(
3599 addr,
3600 memarg,
3601 false,
3602 2,
3603 imported_memories,
3604 offset,
3605 heap_access_oob,
3606 unaligned_atomic,
3607 |this, addr| this.emit_maybe_unaligned_store(Size::S16, value, addr),
3608 )
3609 }
3610
3611 fn i32_atomic_save(
3612 &mut self,
3613 value: Location,
3614 memarg: &MemArg,
3615 addr: Location,
3616 _need_check: bool,
3617 imported_memories: bool,
3618 offset: i32,
3619 heap_access_oob: Label,
3620 unaligned_atomic: Label,
3621 ) -> Result<(), CompileError> {
3622 self.memory_op(
3623 addr,
3624 memarg,
3625 true,
3626 4,
3627 imported_memories,
3628 offset,
3629 heap_access_oob,
3630 unaligned_atomic,
3631 |this, addr| this.emit_relaxed_store(Size::S32, value, Location::Memory(addr, 0)),
3632 )?;
3633 self.assembler.emit_rwfence()
3634 }
3635
3636 fn i32_atomic_save_8(
3637 &mut self,
3638 value: Location,
3639 memarg: &MemArg,
3640 addr: Location,
3641 _need_check: bool,
3642 imported_memories: bool,
3643 offset: i32,
3644 heap_access_oob: Label,
3645 unaligned_atomic: Label,
3646 ) -> Result<(), CompileError> {
3647 self.memory_op(
3648 addr,
3649 memarg,
3650 true,
3651 1,
3652 imported_memories,
3653 offset,
3654 heap_access_oob,
3655 unaligned_atomic,
3656 |this, addr| this.emit_relaxed_store(Size::S8, value, Location::Memory(addr, 0)),
3657 )?;
3658 self.assembler.emit_rwfence()
3659 }
3660
3661 fn i32_atomic_save_16(
3662 &mut self,
3663 value: Location,
3664 memarg: &MemArg,
3665 addr: Location,
3666 _need_check: bool,
3667 imported_memories: bool,
3668 offset: i32,
3669 heap_access_oob: Label,
3670 unaligned_atomic: Label,
3671 ) -> Result<(), CompileError> {
3672 self.memory_op(
3673 addr,
3674 memarg,
3675 true,
3676 2,
3677 imported_memories,
3678 offset,
3679 heap_access_oob,
3680 unaligned_atomic,
3681 |this, addr| this.emit_relaxed_store(Size::S16, value, Location::Memory(addr, 0)),
3682 )?;
3683 self.assembler.emit_rwfence()
3684 }
3685
3686 fn i32_atomic_add(
3687 &mut self,
3688 loc: Location,
3689 target: Location,
3690 memarg: &MemArg,
3691 ret: Location,
3692 _need_check: bool,
3693 imported_memories: bool,
3694 offset: i32,
3695 heap_access_oob: Label,
3696 unaligned_atomic: Label,
3697 ) -> Result<(), CompileError> {
3698 self.memory_op(
3699 target,
3700 memarg,
3701 true,
3702 4,
3703 imported_memories,
3704 offset,
3705 heap_access_oob,
3706 unaligned_atomic,
3707 |this, addr| {
3708 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Add, Size::S32, ret, addr, loc)
3709 },
3710 )
3711 }
3712
3713 fn i32_atomic_add_8u(
3714 &mut self,
3715 loc: Location,
3716 target: Location,
3717 memarg: &MemArg,
3718 ret: Location,
3719 _need_check: bool,
3720 imported_memories: bool,
3721 offset: i32,
3722 heap_access_oob: Label,
3723 unaligned_atomic: Label,
3724 ) -> Result<(), CompileError> {
3725 self.memory_op(
3726 target,
3727 memarg,
3728 true,
3729 1,
3730 imported_memories,
3731 offset,
3732 heap_access_oob,
3733 unaligned_atomic,
3734 |this, addr| {
3735 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Add, Size::S8, ret, addr, loc)
3736 },
3737 )
3738 }
3739
3740 fn i32_atomic_add_16u(
3741 &mut self,
3742 loc: Location,
3743 target: Location,
3744 memarg: &MemArg,
3745 ret: Location,
3746 _need_check: bool,
3747 imported_memories: bool,
3748 offset: i32,
3749 heap_access_oob: Label,
3750 unaligned_atomic: Label,
3751 ) -> Result<(), CompileError> {
3752 self.memory_op(
3753 target,
3754 memarg,
3755 true,
3756 2,
3757 imported_memories,
3758 offset,
3759 heap_access_oob,
3760 unaligned_atomic,
3761 |this, addr| {
3762 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Add, Size::S16, ret, addr, loc)
3763 },
3764 )
3765 }
3766
3767 fn i32_atomic_sub(
3768 &mut self,
3769 loc: Location,
3770 target: Location,
3771 memarg: &MemArg,
3772 ret: Location,
3773 _need_check: bool,
3774 imported_memories: bool,
3775 offset: i32,
3776 heap_access_oob: Label,
3777 unaligned_atomic: Label,
3778 ) -> Result<(), CompileError> {
3779 self.memory_op(
3780 target,
3781 memarg,
3782 true,
3783 4,
3784 imported_memories,
3785 offset,
3786 heap_access_oob,
3787 unaligned_atomic,
3788 |this, addr| {
3789 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Sub, Size::S32, ret, addr, loc)
3790 },
3791 )
3792 }
3793
3794 fn i32_atomic_sub_8u(
3795 &mut self,
3796 loc: Location,
3797 target: Location,
3798 memarg: &MemArg,
3799 ret: Location,
3800 _need_check: bool,
3801 imported_memories: bool,
3802 offset: i32,
3803 heap_access_oob: Label,
3804 unaligned_atomic: Label,
3805 ) -> Result<(), CompileError> {
3806 self.memory_op(
3807 target,
3808 memarg,
3809 true,
3810 1,
3811 imported_memories,
3812 offset,
3813 heap_access_oob,
3814 unaligned_atomic,
3815 |this, addr| {
3816 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Sub, Size::S8, ret, addr, loc)
3817 },
3818 )
3819 }
3820
3821 fn i32_atomic_sub_16u(
3822 &mut self,
3823 loc: Location,
3824 target: Location,
3825 memarg: &MemArg,
3826 ret: Location,
3827 _need_check: bool,
3828 imported_memories: bool,
3829 offset: i32,
3830 heap_access_oob: Label,
3831 unaligned_atomic: Label,
3832 ) -> Result<(), CompileError> {
3833 self.memory_op(
3834 target,
3835 memarg,
3836 true,
3837 2,
3838 imported_memories,
3839 offset,
3840 heap_access_oob,
3841 unaligned_atomic,
3842 |this, addr| {
3843 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Sub, Size::S16, ret, addr, loc)
3844 },
3845 )
3846 }
3847
3848 fn i32_atomic_and(
3849 &mut self,
3850 loc: Location,
3851 target: Location,
3852 memarg: &MemArg,
3853 ret: Location,
3854 _need_check: bool,
3855 imported_memories: bool,
3856 offset: i32,
3857 heap_access_oob: Label,
3858 unaligned_atomic: Label,
3859 ) -> Result<(), CompileError> {
3860 self.memory_op(
3861 target,
3862 memarg,
3863 true,
3864 4,
3865 imported_memories,
3866 offset,
3867 heap_access_oob,
3868 unaligned_atomic,
3869 |this, addr| {
3870 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::And, Size::S32, ret, addr, loc)
3871 },
3872 )
3873 }
3874
3875 fn i32_atomic_and_8u(
3876 &mut self,
3877 loc: Location,
3878 target: Location,
3879 memarg: &MemArg,
3880 ret: Location,
3881 _need_check: bool,
3882 imported_memories: bool,
3883 offset: i32,
3884 heap_access_oob: Label,
3885 unaligned_atomic: Label,
3886 ) -> Result<(), CompileError> {
3887 self.memory_op(
3888 target,
3889 memarg,
3890 true,
3891 1,
3892 imported_memories,
3893 offset,
3894 heap_access_oob,
3895 unaligned_atomic,
3896 |this, addr| {
3897 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::And, Size::S8, ret, addr, loc)
3898 },
3899 )
3900 }
3901
3902 fn i32_atomic_and_16u(
3903 &mut self,
3904 loc: Location,
3905 target: Location,
3906 memarg: &MemArg,
3907 ret: Location,
3908 _need_check: bool,
3909 imported_memories: bool,
3910 offset: i32,
3911 heap_access_oob: Label,
3912 unaligned_atomic: Label,
3913 ) -> Result<(), CompileError> {
3914 self.memory_op(
3915 target,
3916 memarg,
3917 true,
3918 2,
3919 imported_memories,
3920 offset,
3921 heap_access_oob,
3922 unaligned_atomic,
3923 |this, addr| {
3924 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::And, Size::S16, ret, addr, loc)
3925 },
3926 )
3927 }
3928
3929 fn i32_atomic_or(
3930 &mut self,
3931 loc: Location,
3932 target: Location,
3933 memarg: &MemArg,
3934 ret: Location,
3935 _need_check: bool,
3936 imported_memories: bool,
3937 offset: i32,
3938 heap_access_oob: Label,
3939 unaligned_atomic: Label,
3940 ) -> Result<(), CompileError> {
3941 self.memory_op(
3942 target,
3943 memarg,
3944 true,
3945 4,
3946 imported_memories,
3947 offset,
3948 heap_access_oob,
3949 unaligned_atomic,
3950 |this, addr| {
3951 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Or, Size::S32, ret, addr, loc)
3952 },
3953 )
3954 }
3955
3956 fn i32_atomic_or_8u(
3957 &mut self,
3958 loc: Location,
3959 target: Location,
3960 memarg: &MemArg,
3961 ret: Location,
3962 _need_check: bool,
3963 imported_memories: bool,
3964 offset: i32,
3965 heap_access_oob: Label,
3966 unaligned_atomic: Label,
3967 ) -> Result<(), CompileError> {
3968 self.memory_op(
3969 target,
3970 memarg,
3971 true,
3972 1,
3973 imported_memories,
3974 offset,
3975 heap_access_oob,
3976 unaligned_atomic,
3977 |this, addr| {
3978 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Or, Size::S8, ret, addr, loc)
3979 },
3980 )
3981 }
3982
3983 fn i32_atomic_or_16u(
3984 &mut self,
3985 loc: Location,
3986 target: Location,
3987 memarg: &MemArg,
3988 ret: Location,
3989 _need_check: bool,
3990 imported_memories: bool,
3991 offset: i32,
3992 heap_access_oob: Label,
3993 unaligned_atomic: Label,
3994 ) -> Result<(), CompileError> {
3995 self.memory_op(
3996 target,
3997 memarg,
3998 true,
3999 2,
4000 imported_memories,
4001 offset,
4002 heap_access_oob,
4003 unaligned_atomic,
4004 |this, addr| {
4005 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Or, Size::S16, ret, addr, loc)
4006 },
4007 )
4008 }
4009
4010 fn i32_atomic_xor(
4011 &mut self,
4012 loc: Location,
4013 target: Location,
4014 memarg: &MemArg,
4015 ret: Location,
4016 _need_check: bool,
4017 imported_memories: bool,
4018 offset: i32,
4019 heap_access_oob: Label,
4020 unaligned_atomic: Label,
4021 ) -> Result<(), CompileError> {
4022 self.memory_op(
4023 target,
4024 memarg,
4025 true,
4026 4,
4027 imported_memories,
4028 offset,
4029 heap_access_oob,
4030 unaligned_atomic,
4031 |this, addr| {
4032 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Xor, Size::S32, ret, addr, loc)
4033 },
4034 )
4035 }
4036
4037 fn i32_atomic_xor_8u(
4038 &mut self,
4039 loc: Location,
4040 target: Location,
4041 memarg: &MemArg,
4042 ret: Location,
4043 _need_check: bool,
4044 imported_memories: bool,
4045 offset: i32,
4046 heap_access_oob: Label,
4047 unaligned_atomic: Label,
4048 ) -> Result<(), CompileError> {
4049 self.memory_op(
4050 target,
4051 memarg,
4052 true,
4053 1,
4054 imported_memories,
4055 offset,
4056 heap_access_oob,
4057 unaligned_atomic,
4058 |this, addr| {
4059 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Xor, Size::S8, ret, addr, loc)
4060 },
4061 )
4062 }
4063
4064 fn i32_atomic_xor_16u(
4065 &mut self,
4066 loc: Location,
4067 target: Location,
4068 memarg: &MemArg,
4069 ret: Location,
4070 _need_check: bool,
4071 imported_memories: bool,
4072 offset: i32,
4073 heap_access_oob: Label,
4074 unaligned_atomic: Label,
4075 ) -> Result<(), CompileError> {
4076 self.memory_op(
4077 target,
4078 memarg,
4079 true,
4080 2,
4081 imported_memories,
4082 offset,
4083 heap_access_oob,
4084 unaligned_atomic,
4085 |this, addr| {
4086 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Xor, Size::S16, ret, addr, loc)
4087 },
4088 )
4089 }
4090
4091 fn i32_atomic_xchg(
4092 &mut self,
4093 loc: Location,
4094 target: Location,
4095 memarg: &MemArg,
4096 ret: Location,
4097 _need_check: bool,
4098 imported_memories: bool,
4099 offset: i32,
4100 heap_access_oob: Label,
4101 unaligned_atomic: Label,
4102 ) -> Result<(), CompileError> {
4103 self.memory_op(
4104 target,
4105 memarg,
4106 true,
4107 4,
4108 imported_memories,
4109 offset,
4110 heap_access_oob,
4111 unaligned_atomic,
4112 |this, addr| {
4113 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Exchange, Size::S32, ret, addr, loc)
4114 },
4115 )
4116 }
4117
4118 fn i32_atomic_xchg_8u(
4119 &mut self,
4120 loc: Location,
4121 target: Location,
4122 memarg: &MemArg,
4123 ret: Location,
4124 _need_check: bool,
4125 imported_memories: bool,
4126 offset: i32,
4127 heap_access_oob: Label,
4128 unaligned_atomic: Label,
4129 ) -> Result<(), CompileError> {
4130 self.memory_op(
4131 target,
4132 memarg,
4133 true,
4134 1,
4135 imported_memories,
4136 offset,
4137 heap_access_oob,
4138 unaligned_atomic,
4139 |this, addr| {
4140 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Exchange, Size::S8, ret, addr, loc)
4141 },
4142 )
4143 }
4144
4145 fn i32_atomic_xchg_16u(
4146 &mut self,
4147 loc: Location,
4148 target: Location,
4149 memarg: &MemArg,
4150 ret: Location,
4151 _need_check: bool,
4152 imported_memories: bool,
4153 offset: i32,
4154 heap_access_oob: Label,
4155 unaligned_atomic: Label,
4156 ) -> Result<(), CompileError> {
4157 self.memory_op(
4158 target,
4159 memarg,
4160 true,
4161 2,
4162 imported_memories,
4163 offset,
4164 heap_access_oob,
4165 unaligned_atomic,
4166 |this, addr| {
4167 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Exchange, Size::S16, ret, addr, loc)
4168 },
4169 )
4170 }
4171
4172 fn i32_atomic_cmpxchg(
4173 &mut self,
4174 new: Location,
4175 cmp: Location,
4176 target: Location,
4177 memarg: &MemArg,
4178 ret: Location,
4179 _need_check: bool,
4180 imported_memories: bool,
4181 offset: i32,
4182 heap_access_oob: Label,
4183 unaligned_atomic: Label,
4184 ) -> Result<(), CompileError> {
4185 self.memory_op(
4186 target,
4187 memarg,
4188 true,
4189 4,
4190 imported_memories,
4191 offset,
4192 heap_access_oob,
4193 unaligned_atomic,
4194 |this, addr| this.emit_relaxed_atomic_cmpxchg(Size::S32, ret, addr, new, cmp),
4195 )
4196 }
4197
4198 fn i32_atomic_cmpxchg_8u(
4199 &mut self,
4200 new: Location,
4201 cmp: Location,
4202 target: Location,
4203 memarg: &MemArg,
4204 ret: Location,
4205 _need_check: bool,
4206 imported_memories: bool,
4207 offset: i32,
4208 heap_access_oob: Label,
4209 unaligned_atomic: Label,
4210 ) -> Result<(), CompileError> {
4211 self.memory_op(
4212 target,
4213 memarg,
4214 true,
4215 1,
4216 imported_memories,
4217 offset,
4218 heap_access_oob,
4219 unaligned_atomic,
4220 |this, addr| this.emit_relaxed_atomic_cmpxchg(Size::S8, ret, addr, new, cmp),
4221 )
4222 }
4223
4224 fn i32_atomic_cmpxchg_16u(
4225 &mut self,
4226 new: Location,
4227 cmp: Location,
4228 target: Location,
4229 memarg: &MemArg,
4230 ret: Location,
4231 _need_check: bool,
4232 imported_memories: bool,
4233 offset: i32,
4234 heap_access_oob: Label,
4235 unaligned_atomic: Label,
4236 ) -> Result<(), CompileError> {
4237 self.memory_op(
4238 target,
4239 memarg,
4240 true,
4241 2,
4242 imported_memories,
4243 offset,
4244 heap_access_oob,
4245 unaligned_atomic,
4246 |this, addr| this.emit_relaxed_atomic_cmpxchg(Size::S16, ret, addr, new, cmp),
4247 )
4248 }
4249
4250 fn emit_call_with_reloc(
4251 &mut self,
4252 _calling_convention: CallingConvention,
4253 reloc_target: RelocationTarget,
4254 ) -> Result<Vec<Relocation>, CompileError> {
4255 let mut relocations = vec![];
4256 let next = self.get_label();
4257 let reloc_at = self.assembler.get_offset().0;
4258 self.emit_label(next)?; self.assembler.emit_call_label(next)?;
4260 relocations.push(Relocation {
4261 kind: RelocationKind::RiscvCall,
4262 reloc_target,
4263 offset: reloc_at as u32,
4264 addend: 0,
4265 });
4266 Ok(relocations)
4267 }
4268
4269 fn emit_binop_add64(
4270 &mut self,
4271 loc_a: Location,
4272 loc_b: Location,
4273 ret: Location,
4274 ) -> Result<(), CompileError> {
4275 self.emit_relaxed_binop3(
4276 Assembler::emit_add,
4277 Size::S64,
4278 loc_a,
4279 loc_b,
4280 ret,
4281 ImmType::Bits12,
4282 )
4283 }
4284
4285 fn emit_binop_sub64(
4286 &mut self,
4287 loc_a: Location,
4288 loc_b: Location,
4289 ret: Location,
4290 ) -> Result<(), CompileError> {
4291 self.emit_relaxed_binop3(
4292 Assembler::emit_sub,
4293 Size::S64,
4294 loc_a,
4295 loc_b,
4296 ret,
4297 ImmType::Bits12Subtraction,
4298 )
4299 }
4300
4301 fn emit_binop_mul64(
4302 &mut self,
4303 loc_a: Location,
4304 loc_b: Location,
4305 ret: Location,
4306 ) -> Result<(), CompileError> {
4307 self.emit_relaxed_binop3(
4308 Assembler::emit_mul,
4309 Size::S64,
4310 loc_a,
4311 loc_b,
4312 ret,
4313 ImmType::None,
4314 )
4315 }
4316
4317 fn emit_binop_udiv64(
4318 &mut self,
4319 loc_a: Location,
4320 loc_b: Location,
4321 ret: Location,
4322 integer_division_by_zero: Label,
4323 ) -> Result<usize, CompileError> {
4324 let mut temps = vec![];
4325 let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4326 let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4327 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4328
4329 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
4330 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4331 })?;
4332 temps.push(jmp_tmp);
4333
4334 self.assembler
4335 .emit_on_false_label_far(src2, integer_division_by_zero, jmp_tmp)?;
4336 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
4337 self.assembler.emit_udiv(Size::S64, src1, src2, dest)?;
4338 if ret != dest {
4339 self.move_location(Size::S64, dest, ret)?;
4340 }
4341 for r in temps {
4342 self.release_gpr(r);
4343 }
4344 Ok(offset)
4345 }
4346
4347 fn emit_binop_sdiv64(
4348 &mut self,
4349 loc_a: Location,
4350 loc_b: Location,
4351 ret: Location,
4352 integer_division_by_zero: Label,
4353 integer_overflow: Label,
4354 ) -> Result<usize, CompileError> {
4355 let mut temps = vec![];
4356 let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4357 let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4358 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4359
4360 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
4361 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4362 })?;
4363 temps.push(jmp_tmp);
4364
4365 self.assembler
4366 .emit_on_false_label_far(src2, integer_division_by_zero, jmp_tmp)?;
4367 let label_nooverflow = self.assembler.get_label();
4368 let tmp = self.location_to_reg(
4369 Size::S64,
4370 Location::Imm64(i64::MIN as u64),
4371 &mut temps,
4372 ImmType::None,
4373 true,
4374 None,
4375 )?;
4376
4377 self.assembler.emit_cmp(Condition::Ne, tmp, src1, tmp)?;
4378 self.assembler
4379 .emit_on_true_label(tmp, label_nooverflow, jmp_tmp)?;
4380 self.move_location(Size::S64, Location::Imm64(-1i64 as _), tmp)?;
4381 self.assembler.emit_cmp(Condition::Eq, tmp, src2, tmp)?;
4382 self.assembler
4383 .emit_on_true_label_far(tmp, integer_overflow, jmp_tmp)?;
4384 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
4385 self.assembler.emit_label(label_nooverflow)?;
4386 self.assembler.emit_sdiv(Size::S64, src1, src2, dest)?;
4387 if ret != dest {
4388 self.move_location(Size::S64, dest, ret)?;
4389 }
4390 for r in temps {
4391 self.release_gpr(r);
4392 }
4393 Ok(offset)
4394 }
4395
4396 fn emit_binop_urem64(
4397 &mut self,
4398 loc_a: Location,
4399 loc_b: Location,
4400 ret: Location,
4401 integer_division_by_zero: Label,
4402 ) -> Result<usize, CompileError> {
4403 let mut temps = vec![];
4404 let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4405 let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4406 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4407
4408 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
4409 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4410 })?;
4411 temps.push(jmp_tmp);
4412
4413 self.assembler
4414 .emit_on_false_label_far(src2, integer_division_by_zero, jmp_tmp)?;
4415 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
4416 self.assembler.emit_urem(Size::S64, src1, src2, dest)?;
4417 if ret != dest {
4418 self.move_location(Size::S64, dest, ret)?;
4419 }
4420 for r in temps {
4421 self.release_gpr(r);
4422 }
4423 Ok(offset)
4424 }
4425
4426 fn emit_binop_srem64(
4427 &mut self,
4428 loc_a: Location,
4429 loc_b: Location,
4430 ret: Location,
4431 integer_division_by_zero: Label,
4432 ) -> Result<usize, CompileError> {
4433 let mut temps = vec![];
4434 let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4435 let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4436 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4437
4438 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
4439 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4440 })?;
4441 temps.push(jmp_tmp);
4442
4443 self.assembler
4444 .emit_on_false_label_far(src2, integer_division_by_zero, jmp_tmp)?;
4445 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
4446 self.assembler.emit_srem(Size::S64, src1, src2, dest)?;
4447 if ret != dest {
4448 self.move_location(Size::S64, dest, ret)?;
4449 }
4450 for r in temps {
4451 self.release_gpr(r);
4452 }
4453 Ok(offset)
4454 }
4455
4456 fn emit_binop_and64(
4457 &mut self,
4458 loc_a: Location,
4459 loc_b: Location,
4460 ret: Location,
4461 ) -> Result<(), CompileError> {
4462 self.emit_relaxed_binop3(
4463 Assembler::emit_and,
4464 Size::S64,
4465 loc_a,
4466 loc_b,
4467 ret,
4468 ImmType::Bits12,
4469 )
4470 }
4471
4472 fn emit_binop_or64(
4473 &mut self,
4474 loc_a: Location,
4475 loc_b: Location,
4476 ret: Location,
4477 ) -> Result<(), CompileError> {
4478 self.emit_relaxed_binop3(
4479 Assembler::emit_or,
4480 Size::S64,
4481 loc_a,
4482 loc_b,
4483 ret,
4484 ImmType::Bits12,
4485 )
4486 }
4487
4488 fn emit_binop_xor64(
4489 &mut self,
4490 loc_a: Location,
4491 loc_b: Location,
4492 ret: Location,
4493 ) -> Result<(), CompileError> {
4494 self.emit_relaxed_binop3(
4495 Assembler::emit_xor,
4496 Size::S64,
4497 loc_a,
4498 loc_b,
4499 ret,
4500 ImmType::Bits12,
4501 )
4502 }
4503
4504 fn i64_cmp_ge_s(
4505 &mut self,
4506 loc_a: Location,
4507 loc_b: Location,
4508 ret: Location,
4509 ) -> Result<(), CompileError> {
4510 self.emit_cmpop_i64_dynamic_b(Condition::Ge, loc_a, loc_b, ret)
4511 }
4512
4513 fn i64_cmp_gt_s(
4514 &mut self,
4515 loc_a: Location,
4516 loc_b: Location,
4517 ret: Location,
4518 ) -> Result<(), CompileError> {
4519 self.emit_cmpop_i64_dynamic_b(Condition::Gt, loc_a, loc_b, ret)
4520 }
4521
4522 fn i64_cmp_le_s(
4523 &mut self,
4524 loc_a: Location,
4525 loc_b: Location,
4526 ret: Location,
4527 ) -> Result<(), CompileError> {
4528 self.emit_cmpop_i64_dynamic_b(Condition::Le, loc_a, loc_b, ret)
4529 }
4530
4531 fn i64_cmp_lt_s(
4532 &mut self,
4533 loc_a: Location,
4534 loc_b: Location,
4535 ret: Location,
4536 ) -> Result<(), CompileError> {
4537 self.emit_cmpop_i64_dynamic_b(Condition::Lt, loc_a, loc_b, ret)
4538 }
4539
4540 fn i64_cmp_ge_u(
4541 &mut self,
4542 loc_a: Location,
4543 loc_b: Location,
4544 ret: Location,
4545 ) -> Result<(), CompileError> {
4546 self.emit_cmpop_i64_dynamic_b(Condition::Geu, loc_a, loc_b, ret)
4547 }
4548
4549 fn i64_cmp_gt_u(
4550 &mut self,
4551 loc_a: Location,
4552 loc_b: Location,
4553 ret: Location,
4554 ) -> Result<(), CompileError> {
4555 self.emit_cmpop_i64_dynamic_b(Condition::Gtu, loc_a, loc_b, ret)
4556 }
4557
4558 fn i64_cmp_le_u(
4559 &mut self,
4560 loc_a: Location,
4561 loc_b: Location,
4562 ret: Location,
4563 ) -> Result<(), CompileError> {
4564 self.emit_cmpop_i64_dynamic_b(Condition::Leu, loc_a, loc_b, ret)
4565 }
4566
4567 fn i64_cmp_lt_u(
4568 &mut self,
4569 loc_a: Location,
4570 loc_b: Location,
4571 ret: Location,
4572 ) -> Result<(), CompileError> {
4573 self.emit_cmpop_i64_dynamic_b(Condition::Ltu, loc_a, loc_b, ret)
4574 }
4575
4576 fn i64_cmp_ne(
4577 &mut self,
4578 loc_a: Location,
4579 loc_b: Location,
4580 ret: Location,
4581 ) -> Result<(), CompileError> {
4582 self.emit_cmpop_i64_dynamic_b(Condition::Ne, loc_a, loc_b, ret)
4583 }
4584
4585 fn i64_cmp_eq(
4586 &mut self,
4587 loc_a: Location,
4588 loc_b: Location,
4589 ret: Location,
4590 ) -> Result<(), CompileError> {
4591 self.emit_cmpop_i64_dynamic_b(Condition::Eq, loc_a, loc_b, ret)
4592 }
4593
4594 fn i64_clz(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
4595 self.emit_clz(Size::S64, loc, ret)
4596 }
4597
4598 fn i64_ctz(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
4599 self.emit_ctz(Size::S64, loc, ret)
4600 }
4601
4602 fn i64_popcnt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
4603 self.emit_popcnt(Size::S64, loc, ret)
4604 }
4605
4606 fn i64_shl(
4607 &mut self,
4608 loc_a: Location,
4609 loc_b: Location,
4610 ret: Location,
4611 ) -> Result<(), CompileError> {
4612 self.emit_relaxed_binop3(
4613 Assembler::emit_sll,
4614 Size::S64,
4615 loc_a,
4616 loc_b,
4617 ret,
4618 ImmType::Shift64,
4619 )
4620 }
4621
4622 fn i64_shr(
4623 &mut self,
4624 loc_a: Location,
4625 loc_b: Location,
4626 ret: Location,
4627 ) -> Result<(), CompileError> {
4628 self.emit_relaxed_binop3(
4629 Assembler::emit_srl,
4630 Size::S64,
4631 loc_a,
4632 loc_b,
4633 ret,
4634 ImmType::Shift64,
4635 )
4636 }
4637
4638 fn i64_sar(
4639 &mut self,
4640 loc_a: Location,
4641 loc_b: Location,
4642 ret: Location,
4643 ) -> Result<(), CompileError> {
4644 self.emit_relaxed_binop3(
4645 Assembler::emit_sra,
4646 Size::S64,
4647 loc_a,
4648 loc_b,
4649 ret,
4650 ImmType::Shift64,
4651 )
4652 }
4653
4654 fn i64_rol(
4655 &mut self,
4656 loc_a: Location,
4657 loc_b: Location,
4658 ret: Location,
4659 ) -> Result<(), CompileError> {
4660 self.emit_rol(Size::S64, loc_a, loc_b, ret, ImmType::Shift64)
4661 }
4662
4663 fn i64_ror(
4664 &mut self,
4665 loc_a: Location,
4666 loc_b: Location,
4667 ret: Location,
4668 ) -> Result<(), CompileError> {
4669 self.emit_ror(Size::S64, loc_a, loc_b, ret, ImmType::Shift64)
4670 }
4671
4672 fn i64_load(
4673 &mut self,
4674 addr: Location,
4675 memarg: &MemArg,
4676 ret: Location,
4677 _need_check: bool,
4678 imported_memories: bool,
4679 offset: i32,
4680 heap_access_oob: Label,
4681 unaligned_atomic: Label,
4682 ) -> Result<(), CompileError> {
4683 self.memory_op(
4684 addr,
4685 memarg,
4686 false,
4687 8,
4688 imported_memories,
4689 offset,
4690 heap_access_oob,
4691 unaligned_atomic,
4692 |this, addr| this.emit_maybe_unaligned_load(Size::S64, true, ret, addr),
4693 )
4694 }
4695
4696 fn i64_load_8u(
4697 &mut self,
4698 addr: Location,
4699 memarg: &MemArg,
4700 ret: Location,
4701 _need_check: bool,
4702 imported_memories: bool,
4703 offset: i32,
4704 heap_access_oob: Label,
4705 unaligned_atomic: Label,
4706 ) -> Result<(), CompileError> {
4707 self.memory_op(
4708 addr,
4709 memarg,
4710 false,
4711 1,
4712 imported_memories,
4713 offset,
4714 heap_access_oob,
4715 unaligned_atomic,
4716 |this, addr| this.emit_maybe_unaligned_load(Size::S8, false, ret, addr),
4717 )
4718 }
4719
4720 fn i64_load_8s(
4721 &mut self,
4722 addr: Location,
4723 memarg: &MemArg,
4724 ret: Location,
4725 _need_check: bool,
4726 imported_memories: bool,
4727 offset: i32,
4728 heap_access_oob: Label,
4729 unaligned_atomic: Label,
4730 ) -> Result<(), CompileError> {
4731 self.memory_op(
4732 addr,
4733 memarg,
4734 false,
4735 1,
4736 imported_memories,
4737 offset,
4738 heap_access_oob,
4739 unaligned_atomic,
4740 |this, addr| this.emit_maybe_unaligned_load(Size::S8, true, ret, addr),
4741 )
4742 }
4743
4744 fn i64_load_32u(
4745 &mut self,
4746 addr: Location,
4747 memarg: &MemArg,
4748 ret: Location,
4749 _need_check: bool,
4750 imported_memories: bool,
4751 offset: i32,
4752 heap_access_oob: Label,
4753 unaligned_atomic: Label,
4754 ) -> Result<(), CompileError> {
4755 self.memory_op(
4756 addr,
4757 memarg,
4758 false,
4759 4,
4760 imported_memories,
4761 offset,
4762 heap_access_oob,
4763 unaligned_atomic,
4764 |this, addr| this.emit_maybe_unaligned_load(Size::S32, false, ret, addr),
4765 )
4766 }
4767
4768 fn i64_load_32s(
4769 &mut self,
4770 addr: Location,
4771 memarg: &MemArg,
4772 ret: Location,
4773 _need_check: bool,
4774 imported_memories: bool,
4775 offset: i32,
4776 heap_access_oob: Label,
4777 unaligned_atomic: Label,
4778 ) -> Result<(), CompileError> {
4779 self.memory_op(
4780 addr,
4781 memarg,
4782 false,
4783 4,
4784 imported_memories,
4785 offset,
4786 heap_access_oob,
4787 unaligned_atomic,
4788 |this, addr| this.emit_maybe_unaligned_load(Size::S32, true, ret, addr),
4789 )
4790 }
4791
4792 fn i64_load_16u(
4793 &mut self,
4794 addr: Location,
4795 memarg: &MemArg,
4796 ret: Location,
4797 _need_check: bool,
4798 imported_memories: bool,
4799 offset: i32,
4800 heap_access_oob: Label,
4801 unaligned_atomic: Label,
4802 ) -> Result<(), CompileError> {
4803 self.memory_op(
4804 addr,
4805 memarg,
4806 false,
4807 2,
4808 imported_memories,
4809 offset,
4810 heap_access_oob,
4811 unaligned_atomic,
4812 |this, addr| this.emit_maybe_unaligned_load(Size::S16, false, ret, addr),
4813 )
4814 }
4815
4816 fn i64_load_16s(
4817 &mut self,
4818 addr: Location,
4819 memarg: &MemArg,
4820 ret: Location,
4821 _need_check: bool,
4822 imported_memories: bool,
4823 offset: i32,
4824 heap_access_oob: Label,
4825 unaligned_atomic: Label,
4826 ) -> Result<(), CompileError> {
4827 self.memory_op(
4828 addr,
4829 memarg,
4830 false,
4831 2,
4832 imported_memories,
4833 offset,
4834 heap_access_oob,
4835 unaligned_atomic,
4836 |this, addr| this.emit_maybe_unaligned_load(Size::S16, true, ret, addr),
4837 )
4838 }
4839
4840 fn i64_atomic_load(
4841 &mut self,
4842 addr: Location,
4843 memarg: &MemArg,
4844 ret: Location,
4845 _need_check: bool,
4846 imported_memories: bool,
4847 offset: i32,
4848 heap_access_oob: Label,
4849 unaligned_atomic: Label,
4850 ) -> Result<(), CompileError> {
4851 self.memory_op(
4852 addr,
4853 memarg,
4854 true,
4855 8,
4856 imported_memories,
4857 offset,
4858 heap_access_oob,
4859 unaligned_atomic,
4860 |this, addr| this.emit_relaxed_load(Size::S64, true, ret, Location::Memory(addr, 0)),
4861 )
4862 }
4863
4864 fn i64_atomic_load_8u(
4865 &mut self,
4866 addr: Location,
4867 memarg: &MemArg,
4868 ret: Location,
4869 _need_check: bool,
4870 imported_memories: bool,
4871 offset: i32,
4872 heap_access_oob: Label,
4873 unaligned_atomic: Label,
4874 ) -> Result<(), CompileError> {
4875 self.memory_op(
4876 addr,
4877 memarg,
4878 true,
4879 1,
4880 imported_memories,
4881 offset,
4882 heap_access_oob,
4883 unaligned_atomic,
4884 |this, addr| this.emit_relaxed_load(Size::S8, false, ret, Location::Memory(addr, 0)),
4885 )
4886 }
4887
4888 fn i64_atomic_load_16u(
4889 &mut self,
4890 addr: Location,
4891 memarg: &MemArg,
4892 ret: Location,
4893 _need_check: bool,
4894 imported_memories: bool,
4895 offset: i32,
4896 heap_access_oob: Label,
4897 unaligned_atomic: Label,
4898 ) -> Result<(), CompileError> {
4899 self.memory_op(
4900 addr,
4901 memarg,
4902 true,
4903 2,
4904 imported_memories,
4905 offset,
4906 heap_access_oob,
4907 unaligned_atomic,
4908 |this, addr| this.emit_relaxed_load(Size::S16, false, ret, Location::Memory(addr, 0)),
4909 )
4910 }
4911
4912 fn i64_atomic_load_32u(
4913 &mut self,
4914 addr: Location,
4915 memarg: &MemArg,
4916 ret: Location,
4917 _need_check: bool,
4918 imported_memories: bool,
4919 offset: i32,
4920 heap_access_oob: Label,
4921 unaligned_atomic: Label,
4922 ) -> Result<(), CompileError> {
4923 self.memory_op(
4924 addr,
4925 memarg,
4926 true,
4927 4,
4928 imported_memories,
4929 offset,
4930 heap_access_oob,
4931 unaligned_atomic,
4932 |this, addr| this.emit_relaxed_load(Size::S32, false, ret, Location::Memory(addr, 0)),
4933 )
4934 }
4935
4936 fn i64_save(
4937 &mut self,
4938 value: Location,
4939 memarg: &MemArg,
4940 addr: Location,
4941 _need_check: bool,
4942 imported_memories: bool,
4943 offset: i32,
4944 heap_access_oob: Label,
4945 unaligned_atomic: Label,
4946 ) -> Result<(), CompileError> {
4947 self.memory_op(
4948 addr,
4949 memarg,
4950 false,
4951 8,
4952 imported_memories,
4953 offset,
4954 heap_access_oob,
4955 unaligned_atomic,
4956 |this, addr| this.emit_maybe_unaligned_store(Size::S64, value, addr),
4957 )
4958 }
4959
4960 fn i64_save_8(
4961 &mut self,
4962 value: Location,
4963 memarg: &MemArg,
4964 addr: Location,
4965 _need_check: bool,
4966 imported_memories: bool,
4967 offset: i32,
4968 heap_access_oob: Label,
4969 unaligned_atomic: Label,
4970 ) -> Result<(), CompileError> {
4971 self.memory_op(
4972 addr,
4973 memarg,
4974 false,
4975 1,
4976 imported_memories,
4977 offset,
4978 heap_access_oob,
4979 unaligned_atomic,
4980 |this, addr| this.emit_maybe_unaligned_store(Size::S8, value, addr),
4981 )
4982 }
4983
4984 fn i64_save_16(
4985 &mut self,
4986 value: Location,
4987 memarg: &MemArg,
4988 addr: Location,
4989 _need_check: bool,
4990 imported_memories: bool,
4991 offset: i32,
4992 heap_access_oob: Label,
4993 unaligned_atomic: Label,
4994 ) -> Result<(), CompileError> {
4995 self.memory_op(
4996 addr,
4997 memarg,
4998 false,
4999 2,
5000 imported_memories,
5001 offset,
5002 heap_access_oob,
5003 unaligned_atomic,
5004 |this, addr| this.emit_maybe_unaligned_store(Size::S16, value, addr),
5005 )
5006 }
5007
5008 fn i64_save_32(
5009 &mut self,
5010 value: Location,
5011 memarg: &MemArg,
5012 addr: Location,
5013 _need_check: bool,
5014 imported_memories: bool,
5015 offset: i32,
5016 heap_access_oob: Label,
5017 unaligned_atomic: Label,
5018 ) -> Result<(), CompileError> {
5019 self.memory_op(
5020 addr,
5021 memarg,
5022 false,
5023 4,
5024 imported_memories,
5025 offset,
5026 heap_access_oob,
5027 unaligned_atomic,
5028 |this, addr| this.emit_maybe_unaligned_store(Size::S32, value, addr),
5029 )
5030 }
5031
5032 fn i64_atomic_save(
5033 &mut self,
5034 value: Location,
5035 memarg: &MemArg,
5036 addr: Location,
5037 _need_check: bool,
5038 imported_memories: bool,
5039 offset: i32,
5040 heap_access_oob: Label,
5041 unaligned_atomic: Label,
5042 ) -> Result<(), CompileError> {
5043 self.memory_op(
5044 addr,
5045 memarg,
5046 true,
5047 8,
5048 imported_memories,
5049 offset,
5050 heap_access_oob,
5051 unaligned_atomic,
5052 |this, addr| this.emit_relaxed_store(Size::S64, value, Location::Memory(addr, 0)),
5053 )?;
5054 self.assembler.emit_rwfence()
5055 }
5056
5057 fn i64_atomic_save_8(
5058 &mut self,
5059 value: Location,
5060 memarg: &MemArg,
5061 addr: Location,
5062 _need_check: bool,
5063 imported_memories: bool,
5064 offset: i32,
5065 heap_access_oob: Label,
5066 unaligned_atomic: Label,
5067 ) -> Result<(), CompileError> {
5068 self.memory_op(
5069 addr,
5070 memarg,
5071 true,
5072 1,
5073 imported_memories,
5074 offset,
5075 heap_access_oob,
5076 unaligned_atomic,
5077 |this, addr| this.emit_relaxed_store(Size::S8, value, Location::Memory(addr, 0)),
5078 )?;
5079 self.assembler.emit_rwfence()
5080 }
5081
5082 fn i64_atomic_save_16(
5083 &mut self,
5084 value: Location,
5085 memarg: &MemArg,
5086 addr: Location,
5087 _need_check: bool,
5088 imported_memories: bool,
5089 offset: i32,
5090 heap_access_oob: Label,
5091 unaligned_atomic: Label,
5092 ) -> Result<(), CompileError> {
5093 self.memory_op(
5094 addr,
5095 memarg,
5096 true,
5097 2,
5098 imported_memories,
5099 offset,
5100 heap_access_oob,
5101 unaligned_atomic,
5102 |this, addr| this.emit_relaxed_store(Size::S16, value, Location::Memory(addr, 0)),
5103 )?;
5104 self.assembler.emit_rwfence()
5105 }
5106
5107 fn i64_atomic_save_32(
5108 &mut self,
5109 value: Location,
5110 memarg: &MemArg,
5111 addr: Location,
5112 _need_check: bool,
5113 imported_memories: bool,
5114 offset: i32,
5115 heap_access_oob: Label,
5116 unaligned_atomic: Label,
5117 ) -> Result<(), CompileError> {
5118 self.memory_op(
5119 addr,
5120 memarg,
5121 true,
5122 4,
5123 imported_memories,
5124 offset,
5125 heap_access_oob,
5126 unaligned_atomic,
5127 |this, addr| this.emit_relaxed_store(Size::S32, value, Location::Memory(addr, 0)),
5128 )?;
5129 self.assembler.emit_rwfence()
5130 }
5131
5132 fn i64_atomic_add(
5133 &mut self,
5134 loc: Location,
5135 target: Location,
5136 memarg: &MemArg,
5137 ret: Location,
5138 _need_check: bool,
5139 imported_memories: bool,
5140 offset: i32,
5141 heap_access_oob: Label,
5142 unaligned_atomic: Label,
5143 ) -> Result<(), CompileError> {
5144 self.memory_op(
5145 target,
5146 memarg,
5147 true,
5148 8,
5149 imported_memories,
5150 offset,
5151 heap_access_oob,
5152 unaligned_atomic,
5153 |this, addr| {
5154 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Add, Size::S64, ret, addr, loc)
5155 },
5156 )
5157 }
5158
5159 fn i64_atomic_add_8u(
5160 &mut self,
5161 loc: Location,
5162 target: Location,
5163 memarg: &MemArg,
5164 ret: Location,
5165 _need_check: bool,
5166 imported_memories: bool,
5167 offset: i32,
5168 heap_access_oob: Label,
5169 unaligned_atomic: Label,
5170 ) -> Result<(), CompileError> {
5171 self.memory_op(
5172 target,
5173 memarg,
5174 true,
5175 1,
5176 imported_memories,
5177 offset,
5178 heap_access_oob,
5179 unaligned_atomic,
5180 |this, addr| {
5181 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Add, Size::S8, ret, addr, loc)
5182 },
5183 )
5184 }
5185
5186 fn i64_atomic_add_16u(
5187 &mut self,
5188 loc: Location,
5189 target: Location,
5190 memarg: &MemArg,
5191 ret: Location,
5192 _need_check: bool,
5193 imported_memories: bool,
5194 offset: i32,
5195 heap_access_oob: Label,
5196 unaligned_atomic: Label,
5197 ) -> Result<(), CompileError> {
5198 self.memory_op(
5199 target,
5200 memarg,
5201 true,
5202 2,
5203 imported_memories,
5204 offset,
5205 heap_access_oob,
5206 unaligned_atomic,
5207 |this, addr| {
5208 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Add, Size::S16, ret, addr, loc)
5209 },
5210 )
5211 }
5212
5213 fn i64_atomic_add_32u(
5214 &mut self,
5215 loc: Location,
5216 target: Location,
5217 memarg: &MemArg,
5218 ret: Location,
5219 _need_check: bool,
5220 imported_memories: bool,
5221 offset: i32,
5222 heap_access_oob: Label,
5223 unaligned_atomic: Label,
5224 ) -> Result<(), CompileError> {
5225 self.memory_op(
5226 target,
5227 memarg,
5228 true,
5229 4,
5230 imported_memories,
5231 offset,
5232 heap_access_oob,
5233 unaligned_atomic,
5234 |this, addr| {
5235 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Add, Size::S32, ret, addr, loc)
5236 },
5237 )
5238 }
5239
5240 fn i64_atomic_sub(
5241 &mut self,
5242 loc: Location,
5243 target: Location,
5244 memarg: &MemArg,
5245 ret: Location,
5246 _need_check: bool,
5247 imported_memories: bool,
5248 offset: i32,
5249 heap_access_oob: Label,
5250 unaligned_atomic: Label,
5251 ) -> Result<(), CompileError> {
5252 self.memory_op(
5253 target,
5254 memarg,
5255 true,
5256 8,
5257 imported_memories,
5258 offset,
5259 heap_access_oob,
5260 unaligned_atomic,
5261 |this, addr| {
5262 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Sub, Size::S64, ret, addr, loc)
5263 },
5264 )
5265 }
5266
5267 fn i64_atomic_sub_8u(
5268 &mut self,
5269 loc: Location,
5270 target: Location,
5271 memarg: &MemArg,
5272 ret: Location,
5273 _need_check: bool,
5274 imported_memories: bool,
5275 offset: i32,
5276 heap_access_oob: Label,
5277 unaligned_atomic: Label,
5278 ) -> Result<(), CompileError> {
5279 self.memory_op(
5280 target,
5281 memarg,
5282 true,
5283 1,
5284 imported_memories,
5285 offset,
5286 heap_access_oob,
5287 unaligned_atomic,
5288 |this, addr| {
5289 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Sub, Size::S8, ret, addr, loc)
5290 },
5291 )
5292 }
5293
5294 fn i64_atomic_sub_16u(
5295 &mut self,
5296 loc: Location,
5297 target: Location,
5298 memarg: &MemArg,
5299 ret: Location,
5300 _need_check: bool,
5301 imported_memories: bool,
5302 offset: i32,
5303 heap_access_oob: Label,
5304 unaligned_atomic: Label,
5305 ) -> Result<(), CompileError> {
5306 self.memory_op(
5307 target,
5308 memarg,
5309 true,
5310 2,
5311 imported_memories,
5312 offset,
5313 heap_access_oob,
5314 unaligned_atomic,
5315 |this, addr| {
5316 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Sub, Size::S16, ret, addr, loc)
5317 },
5318 )
5319 }
5320
5321 fn i64_atomic_sub_32u(
5322 &mut self,
5323 loc: Location,
5324 target: Location,
5325 memarg: &MemArg,
5326 ret: Location,
5327 _need_check: bool,
5328 imported_memories: bool,
5329 offset: i32,
5330 heap_access_oob: Label,
5331 unaligned_atomic: Label,
5332 ) -> Result<(), CompileError> {
5333 self.memory_op(
5334 target,
5335 memarg,
5336 true,
5337 4,
5338 imported_memories,
5339 offset,
5340 heap_access_oob,
5341 unaligned_atomic,
5342 |this, addr| {
5343 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Sub, Size::S32, ret, addr, loc)
5344 },
5345 )
5346 }
5347
5348 fn i64_atomic_and(
5349 &mut self,
5350 loc: Location,
5351 target: Location,
5352 memarg: &MemArg,
5353 ret: Location,
5354 _need_check: bool,
5355 imported_memories: bool,
5356 offset: i32,
5357 heap_access_oob: Label,
5358 unaligned_atomic: Label,
5359 ) -> Result<(), CompileError> {
5360 self.memory_op(
5361 target,
5362 memarg,
5363 true,
5364 8,
5365 imported_memories,
5366 offset,
5367 heap_access_oob,
5368 unaligned_atomic,
5369 |this, addr| {
5370 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::And, Size::S64, ret, addr, loc)
5371 },
5372 )
5373 }
5374
5375 fn i64_atomic_and_8u(
5376 &mut self,
5377 loc: Location,
5378 target: Location,
5379 memarg: &MemArg,
5380 ret: Location,
5381 _need_check: bool,
5382 imported_memories: bool,
5383 offset: i32,
5384 heap_access_oob: Label,
5385 unaligned_atomic: Label,
5386 ) -> Result<(), CompileError> {
5387 self.memory_op(
5388 target,
5389 memarg,
5390 true,
5391 1,
5392 imported_memories,
5393 offset,
5394 heap_access_oob,
5395 unaligned_atomic,
5396 |this, addr| {
5397 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::And, Size::S8, ret, addr, loc)
5398 },
5399 )
5400 }
5401
5402 fn i64_atomic_and_16u(
5403 &mut self,
5404 loc: Location,
5405 target: Location,
5406 memarg: &MemArg,
5407 ret: Location,
5408 _need_check: bool,
5409 imported_memories: bool,
5410 offset: i32,
5411 heap_access_oob: Label,
5412 unaligned_atomic: Label,
5413 ) -> Result<(), CompileError> {
5414 self.memory_op(
5415 target,
5416 memarg,
5417 true,
5418 2,
5419 imported_memories,
5420 offset,
5421 heap_access_oob,
5422 unaligned_atomic,
5423 |this, addr| {
5424 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::And, Size::S16, ret, addr, loc)
5425 },
5426 )
5427 }
5428
5429 fn i64_atomic_and_32u(
5430 &mut self,
5431 loc: Location,
5432 target: Location,
5433 memarg: &MemArg,
5434 ret: Location,
5435 _need_check: bool,
5436 imported_memories: bool,
5437 offset: i32,
5438 heap_access_oob: Label,
5439 unaligned_atomic: Label,
5440 ) -> Result<(), CompileError> {
5441 self.memory_op(
5442 target,
5443 memarg,
5444 true,
5445 4,
5446 imported_memories,
5447 offset,
5448 heap_access_oob,
5449 unaligned_atomic,
5450 |this, addr| {
5451 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::And, Size::S32, ret, addr, loc)
5452 },
5453 )
5454 }
5455
5456 fn i64_atomic_or(
5457 &mut self,
5458 loc: Location,
5459 target: Location,
5460 memarg: &MemArg,
5461 ret: Location,
5462 _need_check: bool,
5463 imported_memories: bool,
5464 offset: i32,
5465 heap_access_oob: Label,
5466 unaligned_atomic: Label,
5467 ) -> Result<(), CompileError> {
5468 self.memory_op(
5469 target,
5470 memarg,
5471 true,
5472 8,
5473 imported_memories,
5474 offset,
5475 heap_access_oob,
5476 unaligned_atomic,
5477 |this, addr| {
5478 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Or, Size::S64, ret, addr, loc)
5479 },
5480 )
5481 }
5482
5483 fn i64_atomic_or_8u(
5484 &mut self,
5485 loc: Location,
5486 target: Location,
5487 memarg: &MemArg,
5488 ret: Location,
5489 _need_check: bool,
5490 imported_memories: bool,
5491 offset: i32,
5492 heap_access_oob: Label,
5493 unaligned_atomic: Label,
5494 ) -> Result<(), CompileError> {
5495 self.memory_op(
5496 target,
5497 memarg,
5498 true,
5499 1,
5500 imported_memories,
5501 offset,
5502 heap_access_oob,
5503 unaligned_atomic,
5504 |this, addr| {
5505 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Or, Size::S8, ret, addr, loc)
5506 },
5507 )
5508 }
5509
5510 fn i64_atomic_or_16u(
5511 &mut self,
5512 loc: Location,
5513 target: Location,
5514 memarg: &MemArg,
5515 ret: Location,
5516 _need_check: bool,
5517 imported_memories: bool,
5518 offset: i32,
5519 heap_access_oob: Label,
5520 unaligned_atomic: Label,
5521 ) -> Result<(), CompileError> {
5522 self.memory_op(
5523 target,
5524 memarg,
5525 true,
5526 2,
5527 imported_memories,
5528 offset,
5529 heap_access_oob,
5530 unaligned_atomic,
5531 |this, addr| {
5532 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Or, Size::S16, ret, addr, loc)
5533 },
5534 )
5535 }
5536
5537 fn i64_atomic_or_32u(
5538 &mut self,
5539 loc: Location,
5540 target: Location,
5541 memarg: &MemArg,
5542 ret: Location,
5543 _need_check: bool,
5544 imported_memories: bool,
5545 offset: i32,
5546 heap_access_oob: Label,
5547 unaligned_atomic: Label,
5548 ) -> Result<(), CompileError> {
5549 self.memory_op(
5550 target,
5551 memarg,
5552 true,
5553 4,
5554 imported_memories,
5555 offset,
5556 heap_access_oob,
5557 unaligned_atomic,
5558 |this, addr| {
5559 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Or, Size::S32, ret, addr, loc)
5560 },
5561 )
5562 }
5563
5564 fn i64_atomic_xor(
5565 &mut self,
5566 loc: Location,
5567 target: Location,
5568 memarg: &MemArg,
5569 ret: Location,
5570 _need_check: bool,
5571 imported_memories: bool,
5572 offset: i32,
5573 heap_access_oob: Label,
5574 unaligned_atomic: Label,
5575 ) -> Result<(), CompileError> {
5576 self.memory_op(
5577 target,
5578 memarg,
5579 true,
5580 8,
5581 imported_memories,
5582 offset,
5583 heap_access_oob,
5584 unaligned_atomic,
5585 |this, addr| {
5586 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Xor, Size::S64, ret, addr, loc)
5587 },
5588 )
5589 }
5590
5591 fn i64_atomic_xor_8u(
5592 &mut self,
5593 loc: Location,
5594 target: Location,
5595 memarg: &MemArg,
5596 ret: Location,
5597 _need_check: bool,
5598 imported_memories: bool,
5599 offset: i32,
5600 heap_access_oob: Label,
5601 unaligned_atomic: Label,
5602 ) -> Result<(), CompileError> {
5603 self.memory_op(
5604 target,
5605 memarg,
5606 true,
5607 1,
5608 imported_memories,
5609 offset,
5610 heap_access_oob,
5611 unaligned_atomic,
5612 |this, addr| {
5613 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Xor, Size::S8, ret, addr, loc)
5614 },
5615 )
5616 }
5617
5618 fn i64_atomic_xor_16u(
5619 &mut self,
5620 loc: Location,
5621 target: Location,
5622 memarg: &MemArg,
5623 ret: Location,
5624 _need_check: bool,
5625 imported_memories: bool,
5626 offset: i32,
5627 heap_access_oob: Label,
5628 unaligned_atomic: Label,
5629 ) -> Result<(), CompileError> {
5630 self.memory_op(
5631 target,
5632 memarg,
5633 true,
5634 2,
5635 imported_memories,
5636 offset,
5637 heap_access_oob,
5638 unaligned_atomic,
5639 |this, addr| {
5640 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Xor, Size::S16, ret, addr, loc)
5641 },
5642 )
5643 }
5644
5645 fn i64_atomic_xor_32u(
5646 &mut self,
5647 loc: Location,
5648 target: Location,
5649 memarg: &MemArg,
5650 ret: Location,
5651 _need_check: bool,
5652 imported_memories: bool,
5653 offset: i32,
5654 heap_access_oob: Label,
5655 unaligned_atomic: Label,
5656 ) -> Result<(), CompileError> {
5657 self.memory_op(
5658 target,
5659 memarg,
5660 true,
5661 4,
5662 imported_memories,
5663 offset,
5664 heap_access_oob,
5665 unaligned_atomic,
5666 |this, addr| {
5667 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Xor, Size::S32, ret, addr, loc)
5668 },
5669 )
5670 }
5671
5672 fn i64_atomic_xchg(
5673 &mut self,
5674 loc: Location,
5675 target: Location,
5676 memarg: &MemArg,
5677 ret: Location,
5678 _need_check: bool,
5679 imported_memories: bool,
5680 offset: i32,
5681 heap_access_oob: Label,
5682 unaligned_atomic: Label,
5683 ) -> Result<(), CompileError> {
5684 self.memory_op(
5685 target,
5686 memarg,
5687 true,
5688 8,
5689 imported_memories,
5690 offset,
5691 heap_access_oob,
5692 unaligned_atomic,
5693 |this, addr| {
5694 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Exchange, Size::S64, ret, addr, loc)
5695 },
5696 )
5697 }
5698
5699 fn i64_atomic_xchg_8u(
5700 &mut self,
5701 loc: Location,
5702 target: Location,
5703 memarg: &MemArg,
5704 ret: Location,
5705 _need_check: bool,
5706 imported_memories: bool,
5707 offset: i32,
5708 heap_access_oob: Label,
5709 unaligned_atomic: Label,
5710 ) -> Result<(), CompileError> {
5711 self.memory_op(
5712 target,
5713 memarg,
5714 true,
5715 1,
5716 imported_memories,
5717 offset,
5718 heap_access_oob,
5719 unaligned_atomic,
5720 |this, addr| {
5721 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Exchange, Size::S8, ret, addr, loc)
5722 },
5723 )
5724 }
5725
5726 fn i64_atomic_xchg_16u(
5727 &mut self,
5728 loc: Location,
5729 target: Location,
5730 memarg: &MemArg,
5731 ret: Location,
5732 _need_check: bool,
5733 imported_memories: bool,
5734 offset: i32,
5735 heap_access_oob: Label,
5736 unaligned_atomic: Label,
5737 ) -> Result<(), CompileError> {
5738 self.memory_op(
5739 target,
5740 memarg,
5741 true,
5742 2,
5743 imported_memories,
5744 offset,
5745 heap_access_oob,
5746 unaligned_atomic,
5747 |this, addr| {
5748 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Exchange, Size::S16, ret, addr, loc)
5749 },
5750 )
5751 }
5752
5753 fn i64_atomic_xchg_32u(
5754 &mut self,
5755 loc: Location,
5756 target: Location,
5757 memarg: &MemArg,
5758 ret: Location,
5759 _need_check: bool,
5760 imported_memories: bool,
5761 offset: i32,
5762 heap_access_oob: Label,
5763 unaligned_atomic: Label,
5764 ) -> Result<(), CompileError> {
5765 self.memory_op(
5766 target,
5767 memarg,
5768 true,
5769 4,
5770 imported_memories,
5771 offset,
5772 heap_access_oob,
5773 unaligned_atomic,
5774 |this, addr| {
5775 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Exchange, Size::S32, ret, addr, loc)
5776 },
5777 )
5778 }
5779
5780 fn i64_atomic_cmpxchg(
5781 &mut self,
5782 new: Location,
5783 cmp: Location,
5784 target: Location,
5785 memarg: &MemArg,
5786 ret: Location,
5787 _need_check: bool,
5788 imported_memories: bool,
5789 offset: i32,
5790 heap_access_oob: Label,
5791 unaligned_atomic: Label,
5792 ) -> Result<(), CompileError> {
5793 self.memory_op(
5794 target,
5795 memarg,
5796 true,
5797 8,
5798 imported_memories,
5799 offset,
5800 heap_access_oob,
5801 unaligned_atomic,
5802 |this, addr| this.emit_relaxed_atomic_cmpxchg(Size::S64, ret, addr, new, cmp),
5803 )
5804 }
5805
5806 fn i64_atomic_cmpxchg_8u(
5807 &mut self,
5808 new: Location,
5809 cmp: Location,
5810 target: Location,
5811 memarg: &MemArg,
5812 ret: Location,
5813 _need_check: bool,
5814 imported_memories: bool,
5815 offset: i32,
5816 heap_access_oob: Label,
5817 unaligned_atomic: Label,
5818 ) -> Result<(), CompileError> {
5819 self.memory_op(
5820 target,
5821 memarg,
5822 true,
5823 1,
5824 imported_memories,
5825 offset,
5826 heap_access_oob,
5827 unaligned_atomic,
5828 |this, addr| this.emit_relaxed_atomic_cmpxchg(Size::S8, ret, addr, new, cmp),
5829 )
5830 }
5831
5832 fn i64_atomic_cmpxchg_16u(
5833 &mut self,
5834 new: Location,
5835 cmp: Location,
5836 target: Location,
5837 memarg: &MemArg,
5838 ret: Location,
5839 _need_check: bool,
5840 imported_memories: bool,
5841 offset: i32,
5842 heap_access_oob: Label,
5843 unaligned_atomic: Label,
5844 ) -> Result<(), CompileError> {
5845 self.memory_op(
5846 target,
5847 memarg,
5848 true,
5849 2,
5850 imported_memories,
5851 offset,
5852 heap_access_oob,
5853 unaligned_atomic,
5854 |this, addr| this.emit_relaxed_atomic_cmpxchg(Size::S16, ret, addr, new, cmp),
5855 )
5856 }
5857
5858 fn i64_atomic_cmpxchg_32u(
5859 &mut self,
5860 new: Location,
5861 cmp: Location,
5862 target: Location,
5863 memarg: &MemArg,
5864 ret: Location,
5865 _need_check: bool,
5866 imported_memories: bool,
5867 offset: i32,
5868 heap_access_oob: Label,
5869 unaligned_atomic: Label,
5870 ) -> Result<(), CompileError> {
5871 self.memory_op(
5872 target,
5873 memarg,
5874 true,
5875 4,
5876 imported_memories,
5877 offset,
5878 heap_access_oob,
5879 unaligned_atomic,
5880 |this, addr| this.emit_relaxed_atomic_cmpxchg(Size::S32, ret, addr, new, cmp),
5881 )
5882 }
5883
5884 fn f32_load(
5885 &mut self,
5886 addr: Location,
5887 memarg: &MemArg,
5888 ret: Location,
5889 _need_check: bool,
5890 imported_memories: bool,
5891 offset: i32,
5892 heap_access_oob: Label,
5893 unaligned_atomic: Label,
5894 ) -> Result<(), CompileError> {
5895 self.memory_op(
5896 addr,
5897 memarg,
5898 false,
5899 4,
5900 imported_memories,
5901 offset,
5902 heap_access_oob,
5903 unaligned_atomic,
5904 |this, addr| this.emit_relaxed_load(Size::S32, false, ret, Location::Memory(addr, 0)),
5905 )
5906 }
5907
5908 fn f32_save(
5909 &mut self,
5910 value: Location,
5911 memarg: &MemArg,
5912 addr: Location,
5913 canonicalize: bool,
5914 _need_check: bool,
5915 imported_memories: bool,
5916 offset: i32,
5917 heap_access_oob: Label,
5918 unaligned_atomic: Label,
5919 ) -> Result<(), CompileError> {
5920 self.memory_op(
5921 addr,
5922 memarg,
5923 false,
5924 4,
5925 imported_memories,
5926 offset,
5927 heap_access_oob,
5928 unaligned_atomic,
5929 |this, addr| {
5930 if !canonicalize {
5931 this.emit_relaxed_store(Size::S32, value, Location::Memory(addr, 0))
5932 } else {
5933 this.canonicalize_nan(Size::S32, value, Location::Memory(addr, 0))
5934 }
5935 },
5936 )
5937 }
5938
5939 fn f64_load(
5940 &mut self,
5941 addr: Location,
5942 memarg: &MemArg,
5943 ret: Location,
5944 _need_check: bool,
5945 imported_memories: bool,
5946 offset: i32,
5947 heap_access_oob: Label,
5948 unaligned_atomic: Label,
5949 ) -> Result<(), CompileError> {
5950 self.memory_op(
5951 addr,
5952 memarg,
5953 false,
5954 8,
5955 imported_memories,
5956 offset,
5957 heap_access_oob,
5958 unaligned_atomic,
5959 |this, addr| this.emit_relaxed_load(Size::S64, false, ret, Location::Memory(addr, 0)),
5960 )
5961 }
5962
5963 fn f64_save(
5964 &mut self,
5965 value: Location,
5966 memarg: &MemArg,
5967 addr: Location,
5968 canonicalize: bool,
5969 _need_check: bool,
5970 imported_memories: bool,
5971 offset: i32,
5972 heap_access_oob: Label,
5973 unaligned_atomic: Label,
5974 ) -> Result<(), CompileError> {
5975 self.memory_op(
5976 addr,
5977 memarg,
5978 false,
5979 8,
5980 imported_memories,
5981 offset,
5982 heap_access_oob,
5983 unaligned_atomic,
5984 |this, addr| {
5985 if !canonicalize {
5986 this.emit_relaxed_store(Size::S64, value, Location::Memory(addr, 0))
5987 } else {
5988 this.canonicalize_nan(Size::S64, value, Location::Memory(addr, 0))
5989 }
5990 },
5991 )
5992 }
5993
5994 fn convert_f64_i64(
5995 &mut self,
5996 loc: Location,
5997 signed: bool,
5998 ret: Location,
5999 ) -> Result<(), CompileError> {
6000 self.convert_int_to_float(loc, Size::S64, ret, Size::S64, signed)
6001 }
6002
6003 fn convert_f64_i32(
6004 &mut self,
6005 loc: Location,
6006 signed: bool,
6007 ret: Location,
6008 ) -> Result<(), CompileError> {
6009 self.convert_int_to_float(loc, Size::S32, ret, Size::S64, signed)
6010 }
6011
6012 fn convert_f32_i64(
6013 &mut self,
6014 loc: Location,
6015 signed: bool,
6016 ret: Location,
6017 ) -> Result<(), CompileError> {
6018 self.convert_int_to_float(loc, Size::S64, ret, Size::S32, signed)
6019 }
6020
6021 fn convert_f32_i32(
6022 &mut self,
6023 loc: Location,
6024 signed: bool,
6025 ret: Location,
6026 ) -> Result<(), CompileError> {
6027 self.convert_int_to_float(loc, Size::S32, ret, Size::S32, signed)
6028 }
6029
6030 fn convert_i64_f64(
6031 &mut self,
6032 loc: Location,
6033 ret: Location,
6034 signed: bool,
6035 sat: bool,
6036 ) -> Result<(), CompileError> {
6037 self.convert_float_to_int(loc, Size::S64, ret, Size::S64, signed, sat)
6038 }
6039
6040 fn convert_i32_f64(
6041 &mut self,
6042 loc: Location,
6043 ret: Location,
6044 signed: bool,
6045 sat: bool,
6046 ) -> Result<(), CompileError> {
6047 self.convert_float_to_int(loc, Size::S64, ret, Size::S32, signed, sat)
6048 }
6049
6050 fn convert_i64_f32(
6051 &mut self,
6052 loc: Location,
6053 ret: Location,
6054 signed: bool,
6055 sat: bool,
6056 ) -> Result<(), CompileError> {
6057 self.convert_float_to_int(loc, Size::S32, ret, Size::S64, signed, sat)
6058 }
6059
6060 fn convert_i32_f32(
6061 &mut self,
6062 loc: Location,
6063 ret: Location,
6064 signed: bool,
6065 sat: bool,
6066 ) -> Result<(), CompileError> {
6067 self.convert_float_to_int(loc, Size::S32, ret, Size::S32, signed, sat)
6068 }
6069
6070 fn convert_f64_f32(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6071 self.convert_float_to_float(loc, Size::S32, ret, Size::S64)
6072 }
6073
6074 fn convert_f32_f64(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6075 self.convert_float_to_float(loc, Size::S64, ret, Size::S32)
6076 }
6077
6078 fn f64_neg(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6079 self.emit_relaxed_binop_fp(Assembler::emit_fneg, Size::S64, loc, ret, true)
6080 }
6081
6082 fn f64_abs(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6083 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
6084 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6085 })?;
6086 let mask = self.acquire_temp_gpr().ok_or_else(|| {
6087 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6088 })?;
6089
6090 self.move_location(Size::S64, loc, Location::GPR(tmp))?;
6091 self.assembler
6092 .emit_mov_imm(Location::GPR(mask), 0x7fffffffffffffffi64)?;
6093 self.assembler.emit_and(
6094 Size::S64,
6095 Location::GPR(tmp),
6096 Location::GPR(mask),
6097 Location::GPR(tmp),
6098 )?;
6099 self.move_location(Size::S64, Location::GPR(tmp), ret)?;
6100
6101 self.release_gpr(tmp);
6102 self.release_gpr(mask);
6103 Ok(())
6104 }
6105
6106 fn emit_i64_copysign(&mut self, tmp1: Self::GPR, tmp2: Self::GPR) -> Result<(), CompileError> {
6107 let mask = self.acquire_temp_gpr().ok_or_else(|| {
6108 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6109 })?;
6110
6111 self.assembler
6112 .emit_mov_imm(Location::GPR(mask), 0x7fffffffffffffffu64 as _)?;
6113 self.assembler.emit_and(
6114 Size::S64,
6115 Location::GPR(tmp1),
6116 Location::GPR(mask),
6117 Location::GPR(tmp1),
6118 )?;
6119
6120 self.assembler
6121 .emit_mov_imm(Location::GPR(mask), 0x8000000000000000u64 as _)?;
6122 self.assembler.emit_and(
6123 Size::S64,
6124 Location::GPR(tmp2),
6125 Location::GPR(mask),
6126 Location::GPR(tmp2),
6127 )?;
6128
6129 self.release_gpr(mask);
6130 self.assembler.emit_or(
6131 Size::S64,
6132 Location::GPR(tmp1),
6133 Location::GPR(tmp2),
6134 Location::GPR(tmp1),
6135 )
6136 }
6137
6138 fn f64_sqrt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6139 self.emit_relaxed_binop_fp(Assembler::emit_fsqrt, Size::S64, loc, ret, true)
6140 }
6141
6142 fn f64_trunc(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6143 self.emit_relaxed_fcvt_with_rounding(RoundingMode::Rtz, Size::S64, loc, ret)
6144 }
6145
6146 fn f64_ceil(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6147 self.emit_relaxed_fcvt_with_rounding(RoundingMode::Rup, Size::S64, loc, ret)
6148 }
6149
6150 fn f64_floor(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6151 self.emit_relaxed_fcvt_with_rounding(RoundingMode::Rdn, Size::S64, loc, ret)
6152 }
6153
6154 fn f64_nearest(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6155 self.emit_relaxed_fcvt_with_rounding(RoundingMode::Rne, Size::S64, loc, ret)
6156 }
6157
6158 fn f64_cmp_ge(
6159 &mut self,
6160 loc_a: Location,
6161 loc_b: Location,
6162 ret: Location,
6163 ) -> Result<(), CompileError> {
6164 self.emit_relaxed_fcmp(Condition::Ge, Size::S64, loc_a, loc_b, ret)
6165 }
6166
6167 fn f64_cmp_gt(
6168 &mut self,
6169 loc_a: Location,
6170 loc_b: Location,
6171 ret: Location,
6172 ) -> Result<(), CompileError> {
6173 self.emit_relaxed_fcmp(Condition::Gt, Size::S64, loc_a, loc_b, ret)
6174 }
6175
6176 fn f64_cmp_le(
6177 &mut self,
6178 loc_a: Location,
6179 loc_b: Location,
6180 ret: Location,
6181 ) -> Result<(), CompileError> {
6182 self.emit_relaxed_fcmp(Condition::Le, Size::S64, loc_a, loc_b, ret)
6183 }
6184
6185 fn f64_cmp_lt(
6186 &mut self,
6187 loc_a: Location,
6188 loc_b: Location,
6189 ret: Location,
6190 ) -> Result<(), CompileError> {
6191 self.emit_relaxed_fcmp(Condition::Lt, Size::S64, loc_a, loc_b, ret)
6192 }
6193
6194 fn f64_cmp_ne(
6195 &mut self,
6196 loc_a: Location,
6197 loc_b: Location,
6198 ret: Location,
6199 ) -> Result<(), CompileError> {
6200 self.emit_relaxed_fcmp(Condition::Ne, Size::S64, loc_a, loc_b, ret)
6201 }
6202
6203 fn f64_cmp_eq(
6204 &mut self,
6205 loc_a: Location,
6206 loc_b: Location,
6207 ret: Location,
6208 ) -> Result<(), CompileError> {
6209 self.emit_relaxed_fcmp(Condition::Eq, Size::S64, loc_a, loc_b, ret)
6210 }
6211
6212 fn f64_min(
6213 &mut self,
6214 loc_a: Location,
6215 loc_b: Location,
6216 ret: Location,
6217 ) -> Result<(), CompileError> {
6218 self.emit_relaxed_binop3_fp(
6219 Assembler::emit_fmin,
6220 Size::S64,
6221 loc_a,
6222 loc_b,
6223 ret,
6224 ImmType::None,
6225 true,
6226 )
6227 }
6228
6229 fn f64_max(
6230 &mut self,
6231 loc_a: Location,
6232 loc_b: Location,
6233 ret: Location,
6234 ) -> Result<(), CompileError> {
6235 self.emit_relaxed_binop3_fp(
6236 Assembler::emit_fmax,
6237 Size::S64,
6238 loc_a,
6239 loc_b,
6240 ret,
6241 ImmType::None,
6242 true,
6243 )
6244 }
6245
6246 fn f64_add(
6247 &mut self,
6248 loc_a: Location,
6249 loc_b: Location,
6250 ret: Location,
6251 ) -> Result<(), CompileError> {
6252 self.emit_relaxed_binop3_fp(
6253 Assembler::emit_add,
6254 Size::S64,
6255 loc_a,
6256 loc_b,
6257 ret,
6258 ImmType::None,
6259 false,
6260 )
6261 }
6262
6263 fn f64_sub(
6264 &mut self,
6265 loc_a: Location,
6266 loc_b: Location,
6267 ret: Location,
6268 ) -> Result<(), CompileError> {
6269 self.emit_relaxed_binop3_fp(
6270 Assembler::emit_sub,
6271 Size::S64,
6272 loc_a,
6273 loc_b,
6274 ret,
6275 ImmType::None,
6276 false,
6277 )
6278 }
6279
6280 fn f64_mul(
6281 &mut self,
6282 loc_a: Location,
6283 loc_b: Location,
6284 ret: Location,
6285 ) -> Result<(), CompileError> {
6286 self.emit_relaxed_binop3_fp(
6287 Assembler::emit_mul,
6288 Size::S64,
6289 loc_a,
6290 loc_b,
6291 ret,
6292 ImmType::None,
6293 false,
6294 )
6295 }
6296
6297 fn f64_div(
6298 &mut self,
6299 loc_a: Location,
6300 loc_b: Location,
6301 ret: Location,
6302 ) -> Result<(), CompileError> {
6303 self.emit_relaxed_binop3_fp(
6304 Assembler::emit_fdiv,
6305 Size::S64,
6306 loc_a,
6307 loc_b,
6308 ret,
6309 ImmType::None,
6310 false,
6311 )
6312 }
6313
6314 fn f32_neg(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6315 self.emit_relaxed_binop_fp(Assembler::emit_fneg, Size::S32, loc, ret, true)
6316 }
6317
6318 fn f32_abs(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6319 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
6320 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6321 })?;
6322 let mask = self.acquire_temp_gpr().ok_or_else(|| {
6323 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6324 })?;
6325
6326 self.move_location(Size::S32, loc, Location::GPR(tmp))?;
6327 self.assembler
6328 .emit_mov_imm(Location::GPR(mask), 0x7fffffffi64)?;
6329 self.assembler.emit_and(
6330 Size::S32,
6331 Location::GPR(tmp),
6332 Location::GPR(mask),
6333 Location::GPR(tmp),
6334 )?;
6335 self.move_location(Size::S32, Location::GPR(tmp), ret)?;
6336
6337 self.release_gpr(tmp);
6338 self.release_gpr(mask);
6339 Ok(())
6340 }
6341
6342 fn emit_i32_copysign(&mut self, tmp1: Self::GPR, tmp2: Self::GPR) -> Result<(), CompileError> {
6343 let mask = self.acquire_temp_gpr().ok_or_else(|| {
6344 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6345 })?;
6346
6347 self.assembler
6348 .emit_mov_imm(Location::GPR(mask), 0x7fffffffu32 as _)?;
6349 self.assembler.emit_and(
6350 Size::S32,
6351 Location::GPR(tmp1),
6352 Location::GPR(mask),
6353 Location::GPR(tmp1),
6354 )?;
6355
6356 self.assembler
6357 .emit_mov_imm(Location::GPR(mask), 0x80000000u32 as _)?;
6358 self.assembler.emit_and(
6359 Size::S32,
6360 Location::GPR(tmp2),
6361 Location::GPR(mask),
6362 Location::GPR(tmp2),
6363 )?;
6364
6365 self.release_gpr(mask);
6366 self.assembler.emit_or(
6367 Size::S32,
6368 Location::GPR(tmp1),
6369 Location::GPR(tmp2),
6370 Location::GPR(tmp1),
6371 )
6372 }
6373
6374 fn f32_sqrt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6375 self.emit_relaxed_binop_fp(Assembler::emit_fsqrt, Size::S32, loc, ret, true)
6376 }
6377
6378 fn f32_trunc(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6379 self.emit_relaxed_fcvt_with_rounding(RoundingMode::Rtz, Size::S32, loc, ret)
6380 }
6381
6382 fn f32_ceil(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6383 self.emit_relaxed_fcvt_with_rounding(RoundingMode::Rup, Size::S32, loc, ret)
6384 }
6385
6386 fn f32_floor(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6387 self.emit_relaxed_fcvt_with_rounding(RoundingMode::Rdn, Size::S32, loc, ret)
6388 }
6389
6390 fn f32_nearest(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6391 self.emit_relaxed_fcvt_with_rounding(RoundingMode::Rne, Size::S32, loc, ret)
6392 }
6393
6394 fn f32_cmp_ge(
6395 &mut self,
6396 loc_a: Location,
6397 loc_b: Location,
6398 ret: Location,
6399 ) -> Result<(), CompileError> {
6400 self.emit_relaxed_fcmp(Condition::Ge, Size::S32, loc_a, loc_b, ret)
6401 }
6402
6403 fn f32_cmp_gt(
6404 &mut self,
6405 loc_a: Location,
6406 loc_b: Location,
6407 ret: Location,
6408 ) -> Result<(), CompileError> {
6409 self.emit_relaxed_fcmp(Condition::Gt, Size::S32, loc_a, loc_b, ret)
6410 }
6411
6412 fn f32_cmp_le(
6413 &mut self,
6414 loc_a: Location,
6415 loc_b: Location,
6416 ret: Location,
6417 ) -> Result<(), CompileError> {
6418 self.emit_relaxed_fcmp(Condition::Le, Size::S32, loc_a, loc_b, ret)
6419 }
6420
6421 fn f32_cmp_lt(
6422 &mut self,
6423 loc_a: Location,
6424 loc_b: Location,
6425 ret: Location,
6426 ) -> Result<(), CompileError> {
6427 self.emit_relaxed_fcmp(Condition::Lt, Size::S32, loc_a, loc_b, ret)
6428 }
6429
6430 fn f32_cmp_ne(
6431 &mut self,
6432 loc_a: Location,
6433 loc_b: Location,
6434 ret: Location,
6435 ) -> Result<(), CompileError> {
6436 self.emit_relaxed_fcmp(Condition::Ne, Size::S32, loc_a, loc_b, ret)
6437 }
6438
6439 fn f32_cmp_eq(
6440 &mut self,
6441 loc_a: Location,
6442 loc_b: Location,
6443 ret: Location,
6444 ) -> Result<(), CompileError> {
6445 self.emit_relaxed_fcmp(Condition::Eq, Size::S32, loc_a, loc_b, ret)
6446 }
6447
6448 fn f32_min(
6449 &mut self,
6450 loc_a: Location,
6451 loc_b: Location,
6452 ret: Location,
6453 ) -> Result<(), CompileError> {
6454 self.emit_relaxed_binop3_fp(
6455 Assembler::emit_fmin,
6456 Size::S32,
6457 loc_a,
6458 loc_b,
6459 ret,
6460 ImmType::None,
6461 true,
6462 )
6463 }
6464
6465 fn f32_max(
6466 &mut self,
6467 loc_a: Location,
6468 loc_b: Location,
6469 ret: Location,
6470 ) -> Result<(), CompileError> {
6471 self.emit_relaxed_binop3_fp(
6472 Assembler::emit_fmax,
6473 Size::S32,
6474 loc_a,
6475 loc_b,
6476 ret,
6477 ImmType::None,
6478 true,
6479 )
6480 }
6481
6482 fn f32_add(
6483 &mut self,
6484 loc_a: Location,
6485 loc_b: Location,
6486 ret: Location,
6487 ) -> Result<(), CompileError> {
6488 self.emit_relaxed_binop3_fp(
6489 Assembler::emit_add,
6490 Size::S32,
6491 loc_a,
6492 loc_b,
6493 ret,
6494 ImmType::None,
6495 false,
6496 )
6497 }
6498
6499 fn f32_sub(
6500 &mut self,
6501 loc_a: Location,
6502 loc_b: Location,
6503 ret: Location,
6504 ) -> Result<(), CompileError> {
6505 self.emit_relaxed_binop3_fp(
6506 Assembler::emit_sub,
6507 Size::S32,
6508 loc_a,
6509 loc_b,
6510 ret,
6511 ImmType::None,
6512 false,
6513 )
6514 }
6515
6516 fn f32_mul(
6517 &mut self,
6518 loc_a: Location,
6519 loc_b: Location,
6520 ret: Location,
6521 ) -> Result<(), CompileError> {
6522 self.emit_relaxed_binop3_fp(
6523 Assembler::emit_mul,
6524 Size::S32,
6525 loc_a,
6526 loc_b,
6527 ret,
6528 ImmType::None,
6529 false,
6530 )
6531 }
6532
6533 fn f32_div(
6534 &mut self,
6535 loc_a: Location,
6536 loc_b: Location,
6537 ret: Location,
6538 ) -> Result<(), CompileError> {
6539 self.emit_relaxed_binop3_fp(
6540 Assembler::emit_fdiv,
6541 Size::S32,
6542 loc_a,
6543 loc_b,
6544 ret,
6545 ImmType::None,
6546 false,
6547 )
6548 }
6549
6550 fn gen_std_trampoline(
6551 &self,
6552 sig: &FunctionType,
6553 calling_convention: CallingConvention,
6554 ) -> Result<FunctionBody, CompileError> {
6555 gen_std_trampoline_riscv(sig, calling_convention)
6556 }
6557
6558 fn gen_std_dynamic_import_trampoline(
6559 &self,
6560 vmoffsets: &VMOffsets,
6561 sig: &FunctionType,
6562 _calling_convention: CallingConvention,
6563 ) -> Result<FunctionBody, CompileError> {
6564 gen_std_dynamic_import_trampoline_riscv(vmoffsets, sig)
6565 }
6566 fn gen_import_call_trampoline(
6569 &self,
6570 vmoffsets: &VMOffsets,
6571 index: FunctionIndex,
6572 sig: &FunctionType,
6573 calling_convention: CallingConvention,
6574 ) -> Result<CustomSection, CompileError> {
6575 gen_import_call_trampoline_riscv(vmoffsets, index, sig, calling_convention)
6576 }
6577
6578 #[cfg(feature = "unwind")]
6579 fn gen_dwarf_unwind_info(&mut self, code_len: usize) -> Option<UnwindInstructions> {
6580 let mut instructions = vec![];
6581 for &(instruction_offset, ref inst) in &self.unwind_ops {
6582 let instruction_offset = instruction_offset as u32;
6583 match *inst {
6584 UnwindOps::PushFP { up_to_sp } => {
6585 instructions.push((
6586 instruction_offset,
6587 CallFrameInstruction::CfaOffset(up_to_sp as i32),
6588 ));
6589 instructions.push((
6590 instruction_offset,
6591 CallFrameInstruction::Offset(RiscV::FP, -(up_to_sp as i32)),
6592 ));
6593 }
6594 UnwindOps::DefineNewFrame => {
6595 instructions.push((
6596 instruction_offset,
6597 CallFrameInstruction::CfaRegister(RiscV::X8),
6598 ));
6599 }
6600 UnwindOps::SaveRegister { reg, bp_neg_offset } => instructions.push((
6601 instruction_offset,
6602 CallFrameInstruction::Offset(reg.dwarf_index(), -bp_neg_offset),
6603 )),
6604 UnwindOps::SubtractFP { up_to_sp } => {
6605 instructions.push((
6606 instruction_offset,
6607 CallFrameInstruction::CfaOffset(up_to_sp as i32),
6608 ));
6609 }
6610 UnwindOps::Push2Regs { .. } => unimplemented!(),
6611 }
6612 }
6613 Some(UnwindInstructions {
6614 instructions,
6615 len: code_len as u32,
6616 })
6617 }
6618 #[cfg(not(feature = "unwind"))]
6619
6620 fn gen_dwarf_unwind_info(&mut self, _code_len: usize) -> Option<UnwindInstructions> {
6621 None
6622 }
6623
6624 fn gen_windows_unwind_info(&mut self, _code_len: usize) -> Option<Vec<u8>> {
6625 None
6626 }
6627}