1use dynasmrt::{DynasmError, VecAssembler, riscv::RiscvRelocation};
4#[cfg(feature = "unwind")]
5use gimli::{RiscV, write::CallFrameInstruction};
6
7use wasmer_compiler::{
8 CANONICAL_NAN_F32, CANONICAL_NAN_F64,
9 types::{
10 address_map::InstructionAddressMap,
11 function::FunctionBody,
12 relocation::{Relocation, RelocationKind, RelocationTarget},
13 section::CustomSection,
14 },
15 wasmparser::MemArg,
16};
17use wasmer_types::{
18 CompileError, FunctionIndex, FunctionType, SourceLoc, TrapCode, TrapInformation, VMOffsets,
19 target::{CallingConvention, Target},
20};
21
22use crate::{
23 codegen_error,
24 common_decl::*,
25 emitter_riscv::*,
26 location::{Location as AbstractLocation, Reg},
27 machine::*,
28 riscv_decl::{FPR, GPR},
29 unwind::{UnwindInstructions, UnwindOps, UnwindRegister},
30};
31
32type Assembler = VecAssembler<RiscvRelocation>;
33type Location = AbstractLocation<GPR, FPR>;
34
35use std::{
36 collections::HashMap,
37 ops::{Deref, DerefMut},
38};
39pub struct AssemblerRiscv {
41 pub inner: Assembler,
43}
44
45impl AssemblerRiscv {
46 pub fn new(base_addr: usize, _target: Option<Target>) -> Result<Self, CompileError> {
48 Ok(Self {
50 inner: Assembler::new(base_addr),
51 })
52 }
53
54 pub fn finalize(self) -> Result<Vec<u8>, DynasmError> {
56 self.inner.finalize()
57 }
58}
59
60impl Deref for AssemblerRiscv {
61 type Target = Assembler;
62 fn deref(&self) -> &Self::Target {
63 &self.inner
64 }
65}
66
67impl DerefMut for AssemblerRiscv {
68 fn deref_mut(&mut self) -> &mut Self::Target {
69 &mut self.inner
70 }
71}
72
73pub struct MachineRiscv {
75 assembler: AssemblerRiscv,
76 used_gprs: u32,
77 used_fprs: u32,
78 trap_table: TrapTable,
79 instructions_address_map: Vec<InstructionAddressMap>,
82 src_loc: u32,
84 unwind_ops: Vec<(usize, UnwindOps<GPR, FPR>)>,
86}
87
88const SCRATCH_REG: GPR = GPR::X28;
89
90impl MachineRiscv {
91 pub fn new(target: Option<Target>) -> Result<Self, CompileError> {
93 Ok(MachineRiscv {
95 assembler: AssemblerRiscv::new(0, target)?,
96 used_gprs: 0,
97 used_fprs: 0,
98 trap_table: TrapTable::default(),
99 instructions_address_map: vec![],
100 src_loc: 0,
101 unwind_ops: vec![],
102 })
103 }
104
105 fn used_gprs_contains(&self, r: &GPR) -> bool {
106 self.used_gprs & (1 << r.into_index()) != 0
107 }
108 fn used_gprs_insert(&mut self, r: GPR) {
109 self.used_gprs |= 1 << r.into_index();
110 }
111 fn used_gprs_remove(&mut self, r: &GPR) -> bool {
112 let ret = self.used_gprs_contains(r);
113 self.used_gprs &= !(1 << r.into_index());
114 ret
115 }
116
117 fn used_fp_contains(&self, r: &FPR) -> bool {
118 self.used_fprs & (1 << r.into_index()) != 0
119 }
120 fn used_fprs_insert(&mut self, r: FPR) {
121 self.used_fprs |= 1 << r.into_index();
122 }
123 fn used_fprs_remove(&mut self, r: &FPR) -> bool {
124 let ret = self.used_fp_contains(r);
125 self.used_fprs &= !(1 << r.into_index());
126 ret
127 }
128
129 fn location_to_reg(
130 &mut self,
131 sz: Size,
132 src: Location,
133 temps: &mut Vec<GPR>,
134 allow_imm: ImmType,
135 read_val: bool,
136 wanted: Option<GPR>,
137 ) -> Result<Location, CompileError> {
138 match src {
139 Location::GPR(_) | Location::SIMD(_) => Ok(src),
140 Location::Memory(reg, val) => {
141 let tmp = if let Some(wanted) = wanted {
142 wanted
143 } else {
144 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
145 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
146 })?;
147 temps.push(tmp);
148 tmp
149 };
150 if read_val {
151 if ImmType::Bits12.compatible_imm(val as _) {
152 self.assembler.emit_ld(sz, false, Location::GPR(tmp), src)?;
153 } else {
154 if reg == tmp {
155 codegen_error!("singlepass reg == tmp unreachable");
156 }
157 self.assembler.emit_mov_imm(Location::GPR(tmp), val as _)?;
158 self.assembler.emit_add(
159 Size::S64,
160 Location::GPR(reg),
161 Location::GPR(tmp),
162 Location::GPR(tmp),
163 )?;
164 self.assembler.emit_ld(
165 sz,
166 false,
167 Location::GPR(tmp),
168 Location::Memory(tmp, 0),
169 )?;
170 }
171 }
172 Ok(Location::GPR(tmp))
173 }
174 _ if src.is_imm() => {
175 let imm = src.imm_value_scalar().unwrap();
176 if imm == 0 {
177 Ok(Location::GPR(GPR::XZero))
178 } else if allow_imm.compatible_imm(imm) {
179 Ok(src)
180 } else {
181 let tmp = if let Some(wanted) = wanted {
182 wanted
183 } else {
184 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
185 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
186 })?;
187 temps.push(tmp);
188 tmp
189 };
190 self.assembler.emit_mov_imm(Location::GPR(tmp), imm as _)?;
191 Ok(Location::GPR(tmp))
192 }
193 }
194 _ => todo!("unsupported location"),
195 }
196 }
197
198 fn location_to_fpr(
199 &mut self,
200 sz: Size,
201 src: Location,
202 temps: &mut Vec<FPR>,
203 allow_imm: ImmType,
204 read_val: bool,
205 ) -> Result<Location, CompileError> {
206 match src {
207 Location::SIMD(_) => Ok(src),
208 Location::GPR(_) => {
209 let tmp = self.acquire_temp_simd().ok_or_else(|| {
210 CompileError::Codegen("singlepass cannot acquire temp fpr".to_owned())
211 })?;
212 temps.push(tmp);
213 if read_val {
214 self.assembler.emit_mov(sz, src, Location::SIMD(tmp))?;
215 }
216 Ok(Location::SIMD(tmp))
217 }
218 Location::Memory(_, _) => {
219 let tmp = self.acquire_temp_simd().ok_or_else(|| {
220 CompileError::Codegen("singlepass cannot acquire temp fpr".to_owned())
221 })?;
222 temps.push(tmp);
223 if read_val {
224 self.assembler
225 .emit_ld(sz, false, Location::SIMD(tmp), src)?;
226 }
227 Ok(Location::SIMD(tmp))
228 }
229 _ if src.is_imm() => {
230 let tmp = self.acquire_temp_simd().ok_or_else(|| {
231 CompileError::Codegen("singlepass cannot acquire temp fpr".to_owned())
232 })?;
233 temps.push(tmp);
234
235 let mut gpr_temps = vec![];
236 let dst =
237 self.location_to_reg(sz, src, &mut gpr_temps, allow_imm, read_val, None)?;
238 self.assembler.emit_mov(sz, dst, Location::SIMD(tmp))?;
239 for r in gpr_temps {
240 self.release_gpr(r);
241 }
242
243 Ok(Location::SIMD(tmp))
244 }
245 _ => todo!("unsupported location"),
246 }
247 }
248
249 fn emit_relaxed_binop(
250 &mut self,
251 op: fn(&mut Assembler, Size, Location, Location) -> Result<(), CompileError>,
252 sz: Size,
253 src: Location,
254 dst: Location,
255 ) -> Result<(), CompileError> {
256 let mut temps = vec![];
257 let src = self.location_to_reg(sz, src, &mut temps, ImmType::None, true, None)?;
258 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
259 op(&mut self.assembler, sz, src, dest)?;
260 if dst != dest {
261 self.move_location(sz, dest, dst)?;
262 }
263 for r in temps {
264 self.release_gpr(r);
265 }
266 Ok(())
267 }
268
269 fn emit_relaxed_binop_fp(
270 &mut self,
271 op: fn(&mut Assembler, Size, Location, Location) -> Result<(), CompileError>,
272 sz: Size,
273 src: Location,
274 dst: Location,
275 putback: bool,
276 ) -> Result<(), CompileError> {
277 let mut temps = vec![];
278 let src = self.location_to_fpr(sz, src, &mut temps, ImmType::None, true)?;
279 let dest = self.location_to_fpr(sz, dst, &mut temps, ImmType::None, !putback)?;
280 op(&mut self.assembler, sz, src, dest)?;
281 if dst != dest && putback {
282 self.move_location(sz, dest, dst)?;
283 }
284 for r in temps {
285 self.release_simd(r);
286 }
287 Ok(())
288 }
289
290 fn emit_relaxed_binop3(
291 &mut self,
292 op: fn(&mut Assembler, Size, Location, Location, Location) -> Result<(), CompileError>,
293 sz: Size,
294 src1: Location,
295 src2: Location,
296 dst: Location,
297 allow_imm: ImmType,
298 ) -> Result<(), CompileError> {
299 let mut temps = vec![];
300 let src1 = self.location_to_reg(sz, src1, &mut temps, ImmType::None, true, None)?;
301 let src2 = self.location_to_reg(sz, src2, &mut temps, allow_imm, true, None)?;
302 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
303 op(&mut self.assembler, sz, src1, src2, dest)?;
304 if dst != dest {
305 self.move_location(sz, dest, dst)?;
306 }
307 for r in temps {
308 self.release_gpr(r);
309 }
310 Ok(())
311 }
312
313 fn emit_relaxed_atomic_binop3(
314 &mut self,
315 op: AtomicBinaryOp,
316 sz: Size,
317 dst: Location,
318 addr: GPR,
319 src: Location,
320 ) -> Result<(), CompileError> {
321 let mut temps = vec![];
322 let source = self.location_to_reg(sz, src, &mut temps, ImmType::None, false, None)?;
323 let dest = self.location_to_reg(Size::S64, dst, &mut temps, ImmType::None, false, None)?;
324 let (Location::GPR(source), Location::GPR(dest)) = (source, dest) else {
325 panic!("emit_relaxed_atomic_binop3 expects locations in registers");
326 };
327
328 match sz {
331 Size::S32 | Size::S64 => {
332 if op == AtomicBinaryOp::Sub {
333 self.assembler.emit_neg(
334 Size::S64,
335 Location::GPR(source),
336 Location::GPR(source),
337 )?;
338 self.assembler.emit_atomic_binop(
339 AtomicBinaryOp::Add,
340 sz,
341 dest,
342 addr,
343 source,
344 )?;
345 } else {
346 self.assembler
347 .emit_atomic_binop(op, sz, dest, addr, source)?;
348 }
349 self.assembler.emit_rwfence()?;
350 }
351 Size::S8 | Size::S16 => {
352 let aligned_addr = self.acquire_temp_gpr().ok_or_else(|| {
353 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
354 })?;
355 temps.push(aligned_addr);
356 let bit_offset = self.acquire_temp_gpr().ok_or_else(|| {
357 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
358 })?;
359 temps.push(bit_offset);
360 let bit_mask = self.acquire_temp_gpr().ok_or_else(|| {
361 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
362 })?;
363 temps.push(bit_mask);
364
365 self.assembler.emit_and(
366 Size::S64,
367 Location::GPR(addr),
368 Location::Imm64(-4i64 as _),
369 Location::GPR(aligned_addr),
370 )?;
371 self.assembler.emit_and(
372 Size::S64,
373 Location::GPR(addr),
374 Location::Imm64(3),
375 Location::GPR(bit_offset),
376 )?;
377 self.assembler.emit_sll(
378 Size::S64,
379 Location::GPR(bit_offset),
380 Location::Imm64(3),
381 Location::GPR(bit_offset),
382 )?;
383 self.assembler.emit_mov_imm(
384 Location::GPR(bit_mask),
385 if sz == Size::S8 {
386 u8::MAX as _
387 } else {
388 u16::MAX as _
389 },
390 )?;
391 self.assembler.emit_and(
392 Size::S32,
393 Location::GPR(source),
394 Location::GPR(bit_mask),
395 Location::GPR(source),
396 )?;
397 self.assembler.emit_sll(
398 Size::S64,
399 Location::GPR(bit_mask),
400 Location::GPR(bit_offset),
401 Location::GPR(bit_mask),
402 )?;
403 self.assembler.emit_sll(
404 Size::S64,
405 Location::GPR(source),
406 Location::GPR(bit_offset),
407 Location::GPR(source),
408 )?;
409
410 match op {
411 AtomicBinaryOp::Add | AtomicBinaryOp::Sub | AtomicBinaryOp::Exchange => {
412 let loaded_value = self.acquire_temp_gpr().ok_or_else(|| {
413 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
414 })?;
415 temps.push(loaded_value);
416 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
417 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
418 })?;
419 temps.push(tmp);
420
421 let label_retry = self.get_label();
423 self.emit_label(label_retry)?;
424
425 self.assembler
426 .emit_reserved_ld(Size::S32, loaded_value, aligned_addr)?;
427
428 match op {
429 AtomicBinaryOp::Add => self.assembler.emit_add(
430 Size::S64,
431 Location::GPR(loaded_value),
432 Location::GPR(source),
433 Location::GPR(tmp),
434 )?,
435 AtomicBinaryOp::Sub => self.assembler.emit_sub(
436 Size::S64,
437 Location::GPR(loaded_value),
438 Location::GPR(source),
439 Location::GPR(tmp),
440 )?,
441 AtomicBinaryOp::Exchange => self.assembler.emit_mov(
442 Size::S64,
443 Location::GPR(source),
444 Location::GPR(tmp),
445 )?,
446 _ => unreachable!(),
447 }
448
449 self.assembler.emit_xor(
450 Size::S64,
451 Location::GPR(tmp),
452 Location::GPR(loaded_value),
453 Location::GPR(tmp),
454 )?;
455 self.assembler.emit_and(
456 Size::S64,
457 Location::GPR(tmp),
458 Location::GPR(bit_mask),
459 Location::GPR(tmp),
460 )?;
461 self.assembler.emit_xor(
462 Size::S64,
463 Location::GPR(tmp),
464 Location::GPR(loaded_value),
465 Location::GPR(tmp),
466 )?;
467 self.assembler
468 .emit_reserved_sd(Size::S32, tmp, aligned_addr, tmp)?;
469 let tmp2 = self.acquire_temp_gpr().ok_or_else(|| {
470 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
471 })?;
472 temps.push(tmp2);
473 self.assembler
474 .emit_on_true_label(Location::GPR(tmp), label_retry, tmp2)?;
475
476 self.assembler.emit_rwfence()?;
477
478 self.assembler.emit_and(
480 Size::S32,
481 Location::GPR(loaded_value),
482 Location::GPR(bit_mask),
483 Location::GPR(dest),
484 )?;
485 self.assembler.emit_srl(
486 Size::S32,
487 Location::GPR(dest),
488 Location::GPR(bit_offset),
489 Location::GPR(dest),
490 )?;
491 }
492 AtomicBinaryOp::Or | AtomicBinaryOp::Xor => {
493 self.assembler.emit_atomic_binop(
494 op,
495 Size::S32,
496 dest,
497 aligned_addr,
498 source,
499 )?;
500 self.assembler.emit_rwfence()?;
501 self.assembler.emit_and(
502 Size::S32,
503 Location::GPR(dest),
504 Location::GPR(bit_mask),
505 Location::GPR(dest),
506 )?;
507 self.assembler.emit_srl(
508 Size::S32,
509 Location::GPR(dest),
510 Location::GPR(bit_offset),
511 Location::GPR(dest),
512 )?;
513 }
514 AtomicBinaryOp::And => {
515 self.assembler.emit_not(
516 Size::S64,
517 Location::GPR(bit_mask),
518 Location::GPR(bit_mask),
519 )?;
520 self.assembler.emit_or(
521 Size::S64,
522 Location::GPR(bit_mask),
523 Location::GPR(source),
524 Location::GPR(source),
525 )?;
526 self.assembler.emit_not(
527 Size::S64,
528 Location::GPR(bit_mask),
529 Location::GPR(bit_mask),
530 )?;
531 self.assembler.emit_atomic_binop(
532 op,
533 Size::S32,
534 dest,
535 aligned_addr,
536 source,
537 )?;
538 self.assembler.emit_rwfence()?;
539 self.assembler.emit_and(
540 Size::S32,
541 Location::GPR(dest),
542 Location::GPR(bit_mask),
543 Location::GPR(dest),
544 )?;
545 self.assembler.emit_srl(
546 Size::S32,
547 Location::GPR(dest),
548 Location::GPR(bit_offset),
549 Location::GPR(dest),
550 )?;
551 }
552 }
553 }
554 }
555
556 if dst != Location::GPR(dest) {
557 self.move_location(sz, Location::GPR(dest), dst)?;
558 }
559
560 for r in temps {
561 self.release_gpr(r);
562 }
563 Ok(())
564 }
565
566 fn emit_relaxed_atomic_cmpxchg(
567 &mut self,
568 size: Size,
569 dst: Location,
570 addr: GPR,
571 new: Location,
572 cmp: Location,
573 ) -> Result<(), CompileError> {
574 let mut temps = vec![];
575 let cmp = self.location_to_reg(size, cmp, &mut temps, ImmType::None, true, None)?;
576 let new = self.location_to_reg(size, new, &mut temps, ImmType::None, true, None)?;
577 let (Location::GPR(cmp), Location::GPR(new)) = (cmp, new) else {
578 panic!("emit_relaxed_atomic_cmpxchg expects locations in registers");
579 };
580
581 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
582 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
583 })?;
584 temps.push(jmp_tmp);
585
586 match size {
587 Size::S32 | Size::S64 => {
588 let value = self.acquire_temp_gpr().ok_or_else(|| {
589 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
590 })?;
591 temps.push(value);
592 let cond = self.acquire_temp_gpr().ok_or_else(|| {
593 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
594 })?;
595 temps.push(cond);
596
597 let label_retry = self.get_label();
599 let label_after_retry = self.get_label();
600 self.emit_label(label_retry)?;
601
602 self.assembler.emit_reserved_ld(size, value, addr)?;
603 self.assembler.emit_cmp(
604 Condition::Eq,
605 Location::GPR(value),
606 Location::GPR(cmp),
607 Location::GPR(cond),
608 )?;
609 self.assembler.emit_on_false_label(
610 Location::GPR(cond),
611 label_after_retry,
612 jmp_tmp,
613 )?;
614 self.assembler.emit_reserved_sd(size, cond, addr, new)?;
615 self.assembler
616 .emit_on_true_label(Location::GPR(cond), label_retry, jmp_tmp)?;
617
618 self.assembler.emit_rwfence()?;
620 self.emit_label(label_after_retry)?;
621
622 self.assembler.emit_mov(size, Location::GPR(value), dst)?;
623 }
624 Size::S8 | Size::S16 => {
625 let value = self.acquire_temp_gpr().ok_or_else(|| {
626 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
627 })?;
628 temps.push(value);
629 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
630 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
631 })?;
632 temps.push(tmp);
633 let bit_offset = self.acquire_temp_gpr().ok_or_else(|| {
634 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
635 })?;
636 temps.push(bit_offset);
637 let bit_mask = self.acquire_temp_gpr().ok_or_else(|| {
638 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
639 })?;
640 temps.push(bit_mask);
641 let cond = self.acquire_temp_gpr().ok_or_else(|| {
642 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
643 })?;
644 temps.push(cond);
645
646 self.assembler.emit_and(
648 Size::S64,
649 Location::GPR(addr),
650 Location::Imm64(3),
651 Location::GPR(bit_offset),
652 )?;
653 self.assembler.emit_and(
654 Size::S64,
655 Location::GPR(addr),
656 Location::Imm64(-4i64 as _),
657 Location::GPR(addr),
658 )?;
659 self.assembler.emit_sll(
660 Size::S64,
661 Location::GPR(bit_offset),
662 Location::Imm64(3),
663 Location::GPR(bit_offset),
664 )?;
665 self.assembler.emit_mov_imm(
666 Location::GPR(bit_mask),
667 if size == Size::S8 {
668 u8::MAX as _
669 } else {
670 u16::MAX as _
671 },
672 )?;
673 self.assembler.emit_and(
674 Size::S32,
675 Location::GPR(new),
676 Location::GPR(bit_mask),
677 Location::GPR(new),
678 )?;
679 self.assembler.emit_sll(
680 Size::S64,
681 Location::GPR(bit_mask),
682 Location::GPR(bit_offset),
683 Location::GPR(bit_mask),
684 )?;
685 self.assembler.emit_sll(
686 Size::S64,
687 Location::GPR(new),
688 Location::GPR(bit_offset),
689 Location::GPR(new),
690 )?;
691 self.assembler.emit_sll(
692 Size::S64,
693 Location::GPR(cmp),
694 Location::GPR(bit_offset),
695 Location::GPR(cmp),
696 )?;
697
698 let label_retry = self.get_label();
700 let label_after_retry = self.get_label();
701 self.emit_label(label_retry)?;
702
703 self.assembler.emit_reserved_ld(Size::S32, value, addr)?;
704 self.assembler.emit_and(
705 Size::S32,
706 Location::GPR(value),
707 Location::GPR(bit_mask),
708 Location::GPR(tmp),
709 )?;
710
711 self.assembler.emit_cmp(
712 Condition::Eq,
713 Location::GPR(tmp),
714 Location::GPR(cmp),
715 Location::GPR(cond),
716 )?;
717 self.assembler.emit_on_false_label(
718 Location::GPR(cond),
719 label_after_retry,
720 jmp_tmp,
721 )?;
722
723 self.assembler.emit_xor(
725 Size::S32,
726 Location::GPR(value),
727 Location::GPR(new),
728 Location::GPR(tmp),
729 )?;
730 self.assembler.emit_and(
731 Size::S32,
732 Location::GPR(tmp),
733 Location::GPR(bit_mask),
734 Location::GPR(tmp),
735 )?;
736 self.assembler.emit_xor(
737 Size::S32,
738 Location::GPR(tmp),
739 Location::GPR(value),
740 Location::GPR(tmp),
741 )?;
742 self.assembler
743 .emit_reserved_sd(Size::S32, cond, addr, tmp)?;
744 self.assembler
745 .emit_on_true_label(Location::GPR(cond), label_retry, jmp_tmp)?;
746
747 self.assembler.emit_rwfence()?;
749 self.emit_label(label_after_retry)?;
750
751 self.assembler.emit_and(
752 Size::S32,
753 Location::GPR(value),
754 Location::GPR(bit_mask),
755 Location::GPR(tmp),
756 )?;
757 self.assembler.emit_srl(
758 Size::S32,
759 Location::GPR(tmp),
760 Location::GPR(bit_offset),
761 Location::GPR(tmp),
762 )?;
763 self.assembler
764 .emit_mov(Size::S32, Location::GPR(tmp), dst)?;
765 }
766 }
767
768 for r in temps {
769 self.release_gpr(r);
770 }
771 Ok(())
772 }
773
774 #[allow(clippy::too_many_arguments)]
775 fn emit_relaxed_binop3_fp(
776 &mut self,
777 op: fn(&mut Assembler, Size, Location, Location, Location) -> Result<(), CompileError>,
778 sz: Size,
779 src1: Location,
780 src2: Location,
781 dst: Location,
782 allow_imm: ImmType,
783 return_nan_if_present: bool,
784 ) -> Result<(), CompileError> {
785 let mut temps = vec![];
786 let mut gprs = vec![];
787 let src1 = self.location_to_fpr(sz, src1, &mut temps, ImmType::None, true)?;
788 let src2 = self.location_to_fpr(sz, src2, &mut temps, allow_imm, true)?;
789 let dest = self.location_to_fpr(sz, dst, &mut temps, ImmType::None, false)?;
790
791 let label_after = self.get_label();
792 if return_nan_if_present {
793 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
794 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
795 })?;
796 gprs.push(tmp);
797 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
798 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
799 })?;
800 gprs.push(jmp_tmp);
801
802 let canonical_nan = match sz {
804 Size::S32 => CANONICAL_NAN_F32 as u64,
805 Size::S64 => CANONICAL_NAN_F64,
806 _ => unreachable!(),
807 };
808
809 self.assembler
810 .emit_mov_imm(Location::GPR(tmp), canonical_nan as _)?;
811 self.assembler.emit_mov(sz, Location::GPR(tmp), dest)?;
812
813 self.assembler
814 .emit_fcmp(Condition::Eq, sz, src1, src1, Location::GPR(tmp))?;
815 self.assembler
816 .emit_on_false_label(Location::GPR(tmp), label_after, jmp_tmp)?;
817
818 self.assembler
819 .emit_fcmp(Condition::Eq, sz, src2, src2, Location::GPR(tmp))?;
820 self.assembler
821 .emit_on_false_label(Location::GPR(tmp), label_after, jmp_tmp)?;
822 }
823
824 op(&mut self.assembler, sz, src1, src2, dest)?;
825 self.emit_label(label_after)?;
826
827 if dst != dest {
828 self.move_location(sz, dest, dst)?;
829 }
830 for r in temps {
831 self.release_simd(r);
832 }
833 for r in gprs {
834 self.release_gpr(r);
835 }
836 Ok(())
837 }
838
839 fn emit_relaxed_cmp(
840 &mut self,
841 c: Condition,
842 loc_a: Location,
843 loc_b: Location,
844 ret: Location,
845 sz: Size,
846 signed: bool,
847 ) -> Result<(), CompileError> {
848 let mut temps = vec![];
850 let loc_a = self.location_to_reg(sz, loc_a, &mut temps, ImmType::None, true, None)?;
851 let loc_b = self.location_to_reg(sz, loc_b, &mut temps, ImmType::None, true, None)?;
852
853 if sz != Size::S64 {
854 self.assembler.emit_extend(sz, signed, loc_a, loc_a)?;
855 self.assembler.emit_extend(sz, signed, loc_b, loc_b)?;
856 }
857
858 let dest = self.location_to_reg(sz, ret, &mut temps, ImmType::None, false, None)?;
859 self.assembler.emit_cmp(c, loc_a, loc_b, dest)?;
860 if ret != dest {
861 self.move_location(sz, dest, ret)?;
862 }
863 for r in temps {
864 self.release_gpr(r);
865 }
866 Ok(())
867 }
868
869 fn emit_cmpop_i32_dynamic_b(
871 &mut self,
872 c: Condition,
873 loc_a: Location,
874 loc_b: Location,
875 ret: Location,
876 signed: bool,
877 ) -> Result<(), CompileError> {
878 self.emit_relaxed_cmp(c, loc_a, loc_b, ret, Size::S32, signed)
879 }
880
881 fn emit_cmpop_i64_dynamic_b(
883 &mut self,
884 c: Condition,
885 loc_a: Location,
886 loc_b: Location,
887 ret: Location,
888 ) -> Result<(), CompileError> {
889 self.emit_relaxed_cmp(c, loc_a, loc_b, ret, Size::S64, false)
890 }
891
892 #[allow(clippy::too_many_arguments)]
896 fn memory_op<F: FnOnce(&mut Self, GPR) -> Result<(), CompileError>>(
897 &mut self,
898 addr: Location,
899 memarg: &MemArg,
900 check_alignment: bool,
901 value_size: usize,
902 imported_memories: bool,
903 offset: i32,
904 heap_access_oob: Label,
905 unaligned_atomic: Label,
906 cb: F,
907 ) -> Result<(), CompileError> {
908 let value_size = value_size as i64;
909 let tmp_addr = self.acquire_temp_gpr().ok_or_else(|| {
910 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
911 })?;
912
913 let (base_loc, bound_loc) = if imported_memories {
915 self.emit_relaxed_binop(
917 Assembler::emit_mov,
918 Size::S64,
919 Location::Memory(self.get_vmctx_reg(), offset),
920 Location::GPR(tmp_addr),
921 )?;
922 (Location::Memory(tmp_addr, 0), Location::Memory(tmp_addr, 8))
923 } else {
924 (
925 Location::Memory(self.get_vmctx_reg(), offset),
926 Location::Memory(self.get_vmctx_reg(), offset + 8),
927 )
928 };
929
930 let tmp_base = self.acquire_temp_gpr().ok_or_else(|| {
931 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
932 })?;
933 let tmp_bound = self.acquire_temp_gpr().ok_or_else(|| {
934 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
935 })?;
936
937 self.emit_relaxed_load(Size::S64, false, Location::GPR(tmp_base), base_loc)?;
939
940 self.emit_relaxed_load(Size::S64, false, Location::GPR(tmp_bound), bound_loc)?;
942
943 self.assembler.emit_add(
947 Size::S64,
948 Location::GPR(tmp_bound),
949 Location::GPR(tmp_base),
950 Location::GPR(tmp_bound),
951 )?;
952 self.assembler.emit_sub(
953 Size::S64,
954 Location::GPR(tmp_bound),
955 Location::Imm64(value_size as _),
956 Location::GPR(tmp_bound),
957 )?;
958
959 self.move_location(Size::S32, addr, Location::GPR(tmp_addr))?;
963
964 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
965 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
966 })?;
967
968 if memarg.offset != 0 {
970 if ImmType::Bits12.compatible_imm(memarg.offset as _) {
971 self.assembler.emit_add(
972 Size::S64,
973 Location::Imm64(memarg.offset),
974 Location::GPR(tmp_addr),
975 Location::GPR(tmp_addr),
976 )?;
977 } else {
978 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
979 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
980 })?;
981 self.assembler
982 .emit_mov_imm(Location::GPR(tmp), memarg.offset as _)?;
983 self.assembler.emit_add(
984 Size::S64,
985 Location::GPR(tmp_addr),
986 Location::GPR(tmp),
987 Location::GPR(tmp_addr),
988 )?;
989 self.release_gpr(tmp);
990 }
991
992 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
995 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
996 })?;
997 self.assembler.emit_srl(
998 Size::S64,
999 Location::GPR(tmp_addr),
1000 Location::Imm64(32),
1001 Location::GPR(tmp),
1002 )?;
1003 self.assembler
1004 .emit_on_true_label_far(Location::GPR(tmp), heap_access_oob, jmp_tmp)?;
1005 self.release_gpr(tmp);
1006 }
1007
1008 self.assembler.emit_add(
1010 Size::S64,
1011 Location::GPR(tmp_base),
1012 Location::GPR(tmp_addr),
1013 Location::GPR(tmp_addr),
1014 )?;
1015
1016 let cond = tmp_base;
1018
1019 self.assembler.emit_cmp(
1021 Condition::Le,
1022 Location::GPR(tmp_addr),
1023 Location::GPR(tmp_bound),
1024 Location::GPR(cond),
1025 )?;
1026
1027 self.assembler
1029 .emit_on_false_label_far(Location::GPR(cond), heap_access_oob, jmp_tmp)?;
1030
1031 self.release_gpr(tmp_bound);
1032 self.release_gpr(cond);
1033
1034 let align = value_size as u32;
1035 if check_alignment && align != 1 {
1036 self.assembler.emit_and(
1037 Size::S64,
1038 Location::GPR(tmp_addr),
1039 Location::Imm64((align - 1) as u64),
1040 Location::GPR(cond),
1041 )?;
1042 self.assembler.emit_on_true_label_far(
1043 Location::GPR(cond),
1044 unaligned_atomic,
1045 jmp_tmp,
1046 )?;
1047 }
1048 let begin = self.assembler.get_offset().0;
1049 cb(self, tmp_addr)?;
1050 let end = self.assembler.get_offset().0;
1051 self.mark_address_range_with_trap_code(TrapCode::HeapAccessOutOfBounds, begin, end);
1052
1053 self.release_gpr(jmp_tmp);
1054 self.release_gpr(tmp_addr);
1055 Ok(())
1056 }
1057
1058 fn emit_relaxed_load(
1059 &mut self,
1060 sz: Size,
1061 signed: bool,
1062 dst: Location,
1063 src: Location,
1064 ) -> Result<(), CompileError> {
1065 let mut temps = vec![];
1066 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
1067 match src {
1068 Location::Memory(addr, offset) => {
1069 if ImmType::Bits12.compatible_imm(offset as i64) {
1070 self.assembler.emit_ld(sz, signed, dest, src)?;
1071 } else {
1072 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
1073 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1074 })?;
1075 self.assembler
1076 .emit_mov_imm(Location::GPR(tmp), offset as i64)?;
1077 self.assembler.emit_add(
1078 Size::S64,
1079 Location::GPR(addr),
1080 Location::GPR(tmp),
1081 Location::GPR(tmp),
1082 )?;
1083 self.assembler
1084 .emit_ld(sz, signed, dest, Location::Memory(tmp, 0))?;
1085 temps.push(tmp);
1086 }
1087 }
1088 _ => codegen_error!("singlepass emit_relaxed_load unreachable"),
1089 }
1090 if dst != dest {
1091 self.move_location(Size::S64, dest, dst)?;
1093 }
1094 for r in temps {
1095 self.release_gpr(r);
1096 }
1097 Ok(())
1098 }
1099
1100 fn emit_maybe_unaligned_load(
1101 &mut self,
1102 sz: Size,
1103 signed: bool,
1104 dst: Location,
1105 src: GPR,
1106 ) -> Result<(), CompileError> {
1107 if let Size::S8 = sz {
1108 return self.emit_relaxed_load(sz, signed, dst, Location::Memory(src, 0));
1109 }
1110
1111 let label_unaligned = self.get_label();
1112 let label_completed = self.get_label();
1113 let mut temps = vec![];
1114 let tmp_cond = self.acquire_temp_gpr().ok_or_else(|| {
1115 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1116 })?;
1117 temps.push(tmp_cond);
1118 let dest = self.location_to_reg(sz, dst, &mut temps, ImmType::None, false, None)?;
1119 self.assembler.emit_and(
1120 Size::S64,
1121 Location::GPR(src),
1122 Location::Imm64((sz.bytes() - 1) as u64),
1123 Location::GPR(tmp_cond),
1124 )?;
1125 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
1126 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1127 })?;
1128 temps.push(jmp_tmp);
1129 self.assembler
1130 .emit_on_true_label(Location::GPR(tmp_cond), label_unaligned, jmp_tmp)?;
1131 self.assembler
1133 .emit_ld(sz, signed, dest, Location::Memory(src, 0))?;
1134 self.jmp_unconditional(label_completed)?;
1135 self.emit_label(label_unaligned)?;
1136
1137 let tmp_value = tmp_cond;
1139
1140 self.assembler
1142 .emit_ld(Size::S8, false, dest, Location::Memory(src, 0))?;
1143 for i in 1..sz.bytes() {
1144 self.assembler.emit_ld(
1145 Size::S8,
1146 if i == sz.bytes() - 1 { signed } else { false },
1147 Location::GPR(tmp_value),
1148 Location::Memory(src, i as _),
1149 )?;
1150 self.assembler.emit_sll(
1151 Size::S64,
1152 Location::GPR(tmp_value),
1153 Location::Imm64(8 * i as u64),
1154 Location::GPR(tmp_value),
1155 )?;
1156 self.assembler
1157 .emit_or(Size::S64, dest, Location::GPR(tmp_value), dest)?;
1158 }
1159
1160 self.emit_label(label_completed)?;
1162 if dst != dest {
1163 self.move_location(Size::S64, dest, dst)?;
1165 }
1166
1167 for tmp in temps {
1168 self.release_gpr(tmp);
1169 }
1170 Ok(())
1171 }
1172
1173 fn emit_relaxed_store(
1174 &mut self,
1175 sz: Size,
1176 dst: Location,
1177 src: Location,
1178 ) -> Result<(), CompileError> {
1179 let mut temps = vec![];
1180 let dest = self.location_to_reg(Size::S64, dst, &mut temps, ImmType::None, true, None)?;
1181 match src {
1182 Location::Memory(addr, offset) => {
1183 if ImmType::Bits12.compatible_imm(offset as i64) {
1184 self.assembler.emit_sd(sz, dest, src)?;
1185 } else {
1186 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
1187 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1188 })?;
1189 self.assembler
1190 .emit_mov_imm(Location::GPR(tmp), offset as i64)?;
1191 self.assembler.emit_add(
1192 Size::S64,
1193 Location::GPR(addr),
1194 Location::GPR(tmp),
1195 Location::GPR(tmp),
1196 )?;
1197 self.assembler.emit_sd(sz, dest, Location::Memory(tmp, 0))?;
1198 temps.push(tmp);
1199 }
1200 }
1201 _ => codegen_error!("singlepass emit_relaxed_store unreachable"),
1202 }
1203 for r in temps {
1204 self.release_gpr(r);
1205 }
1206 Ok(())
1207 }
1208
1209 fn emit_maybe_unaligned_store(
1210 &mut self,
1211 sz: Size,
1212 src: Location,
1213 dst: GPR,
1214 ) -> Result<(), CompileError> {
1215 if let Size::S8 = sz {
1216 return self.emit_relaxed_store(sz, src, Location::Memory(dst, 0));
1221 }
1222
1223 let label_unaligned = self.get_label();
1224 let label_completed = self.get_label();
1225 let mut temps = vec![];
1226 let tmp_cond = self.acquire_temp_gpr().ok_or_else(|| {
1227 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1228 })?;
1229 temps.push(tmp_cond);
1230 let src_value =
1231 self.location_to_reg(Size::S64, src, &mut temps, ImmType::None, true, None)?;
1232 self.assembler.emit_and(
1233 Size::S64,
1234 Location::GPR(dst),
1235 Location::Imm64((sz.bytes() - 1) as u64),
1236 Location::GPR(tmp_cond),
1237 )?;
1238 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
1239 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1240 })?;
1241 temps.push(jmp_tmp);
1242 self.assembler
1243 .emit_on_true_label(Location::GPR(tmp_cond), label_unaligned, jmp_tmp)?;
1244 self.assembler
1246 .emit_sd(sz, src_value, Location::Memory(dst, 0))?;
1247 self.jmp_unconditional(label_completed)?;
1248 self.emit_label(label_unaligned)?;
1249
1250 let size_bytes = sz.bytes();
1252 for i in 0..size_bytes {
1254 self.assembler
1255 .emit_sd(Size::S8, src_value, Location::Memory(dst, i as _))?;
1256 if i != size_bytes - 1 {
1257 self.assembler
1258 .emit_srl(Size::S64, src_value, Location::Imm64(8), src_value)?;
1259 }
1260 }
1261
1262 self.emit_label(label_completed)?;
1264
1265 for tmp in temps {
1266 self.release_gpr(tmp);
1267 }
1268 Ok(())
1269 }
1270
1271 fn emit_rol(
1272 &mut self,
1273 sz: Size,
1274 loc_a: Location,
1275 loc_b: Location,
1276 ret: Location,
1277 allow_imm: ImmType,
1278 ) -> Result<(), CompileError> {
1279 let mut temps = vec![];
1280 let size_bits = sz.bits();
1281
1282 let src2 = if let Some(imm) = loc_b.imm_value_scalar() {
1283 Location::Imm32(size_bits - (imm as u32) % size_bits)
1284 } else {
1285 let tmp1 = self.location_to_reg(
1286 sz,
1287 Location::Imm32(size_bits),
1288 &mut temps,
1289 ImmType::None,
1290 true,
1291 None,
1292 )?;
1293 let tmp2 = self.location_to_reg(sz, loc_b, &mut temps, ImmType::None, true, None)?;
1294 self.assembler.emit_sub(sz, tmp1, tmp2, tmp1)?;
1295 tmp1
1296 };
1297
1298 self.emit_ror(sz, loc_a, src2, ret, allow_imm)?;
1299
1300 for r in temps {
1301 self.release_gpr(r);
1302 }
1303 Ok(())
1304 }
1305
1306 fn emit_ror(
1307 &mut self,
1308 sz: Size,
1309 loc_a: Location,
1310 loc_b: Location,
1311 ret: Location,
1312 allow_imm: ImmType,
1313 ) -> Result<(), CompileError> {
1314 let mut temps = vec![];
1315
1316 let imm = match sz {
1317 Size::S32 | Size::S64 => Location::Imm32(sz.bits() as u32),
1318 _ => codegen_error!("singlepass emit_ror unreachable"),
1319 };
1320 let imm = self.location_to_reg(sz, imm, &mut temps, ImmType::None, false, None)?;
1321
1322 let tmp1 = self.acquire_temp_gpr().ok_or_else(|| {
1323 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1324 })?;
1325 self.emit_relaxed_binop3(
1326 Assembler::emit_srl,
1327 sz,
1328 loc_a,
1329 loc_b,
1330 Location::GPR(tmp1),
1331 allow_imm,
1332 )?;
1333
1334 let tmp2 = self.acquire_temp_gpr().ok_or_else(|| {
1335 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1336 })?;
1337 self.emit_relaxed_binop3(
1338 Assembler::emit_sub,
1339 sz,
1340 imm,
1341 loc_b,
1342 Location::GPR(tmp2),
1343 allow_imm,
1344 )?;
1345 self.emit_relaxed_binop3(
1346 Assembler::emit_sll,
1347 sz,
1348 loc_a,
1349 Location::GPR(tmp2),
1350 Location::GPR(tmp2),
1351 ImmType::Bits12,
1352 )?;
1353 self.assembler.emit_or(
1354 sz,
1355 Location::GPR(tmp1),
1356 Location::GPR(tmp2),
1357 Location::GPR(tmp1),
1358 )?;
1359
1360 self.move_location(sz, Location::GPR(tmp1), ret)?;
1361 self.release_gpr(tmp1);
1362 self.release_gpr(tmp2);
1363 for r in temps {
1364 self.release_gpr(r);
1365 }
1366 Ok(())
1367 }
1368
1369 fn emit_popcnt(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1370 let arg = self.acquire_temp_gpr().ok_or_else(|| {
1371 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1372 })?;
1373 let cnt = self.acquire_temp_gpr().ok_or_else(|| {
1374 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1375 })?;
1376 let temp = self.acquire_temp_gpr().ok_or_else(|| {
1377 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1378 })?;
1379 self.move_location(sz, src, Location::GPR(arg))?;
1380
1381 self.move_location(sz, Location::Imm32(0), Location::GPR(cnt))?;
1382 let one_imm = match sz {
1383 Size::S32 => Location::Imm32(1),
1384 Size::S64 => Location::Imm64(1),
1385 _ => codegen_error!("singlepass emit_popcnt unreachable"),
1386 };
1387
1388 let label_loop = self.assembler.get_label();
1389 let label_exit = self.assembler.get_label();
1390
1391 self.assembler.emit_label(label_loop)?; self.assembler
1393 .emit_and(sz, Location::GPR(arg), one_imm, Location::GPR(temp))?;
1394 self.assembler.emit_add(
1395 sz,
1396 Location::GPR(cnt),
1397 Location::GPR(temp),
1398 Location::GPR(cnt),
1399 )?;
1400 self.assembler
1401 .emit_srl(sz, Location::GPR(arg), one_imm, Location::GPR(arg))?;
1402 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
1403 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1404 })?;
1405 self.assembler
1406 .emit_on_false_label(Location::GPR(arg), label_exit, jmp_tmp)?;
1407 self.release_gpr(jmp_tmp);
1408 self.jmp_unconditional(label_loop)?;
1409
1410 self.assembler.emit_label(label_exit)?; self.move_location(sz, Location::GPR(cnt), dst)?;
1413
1414 self.release_gpr(arg);
1415 self.release_gpr(cnt);
1416 self.release_gpr(temp);
1417 Ok(())
1418 }
1419
1420 fn emit_ctz(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1421 let size_bits = sz.bits();
1422 let arg = self.acquire_temp_gpr().ok_or_else(|| {
1423 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1424 })?;
1425 let cnt = self.acquire_temp_gpr().ok_or_else(|| {
1426 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1427 })?;
1428 let temp = self.acquire_temp_gpr().ok_or_else(|| {
1429 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1430 })?;
1431 self.move_location(sz, src, Location::GPR(arg))?;
1432
1433 let one_imm = match sz {
1434 Size::S32 => Location::Imm32(1),
1435 Size::S64 => Location::Imm64(1),
1436 _ => codegen_error!("singlepass emit_ctz unreachable"),
1437 };
1438
1439 let label_loop = self.assembler.get_label();
1440 let label_exit = self.assembler.get_label();
1441
1442 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
1443 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1444 })?;
1445
1446 self.move_location(sz, Location::Imm32(size_bits), Location::GPR(cnt))?;
1448 self.assembler
1449 .emit_on_false_label(Location::GPR(arg), label_exit, jmp_tmp)?;
1450
1451 self.move_location(sz, Location::Imm32(0), Location::GPR(cnt))?;
1452
1453 self.assembler.emit_label(label_loop)?; self.assembler
1455 .emit_and(sz, Location::GPR(arg), one_imm, Location::GPR(temp))?;
1456 self.assembler
1457 .emit_on_true_label(Location::GPR(temp), label_exit, jmp_tmp)?;
1458
1459 self.release_gpr(jmp_tmp);
1460
1461 self.assembler
1462 .emit_add(sz, Location::GPR(cnt), one_imm, Location::GPR(cnt))?;
1463 self.assembler
1464 .emit_srl(sz, Location::GPR(arg), one_imm, Location::GPR(arg))?;
1465 self.jmp_unconditional(label_loop)?;
1466
1467 self.assembler.emit_label(label_exit)?; self.move_location(sz, Location::GPR(cnt), dst)?;
1470
1471 self.release_gpr(arg);
1472 self.release_gpr(cnt);
1473 self.release_gpr(temp);
1474 Ok(())
1475 }
1476
1477 fn emit_clz(&mut self, sz: Size, src: Location, dst: Location) -> Result<(), CompileError> {
1478 let size_bits = sz.bits();
1479 let arg = self.acquire_temp_gpr().ok_or_else(|| {
1480 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1481 })?;
1482 let cnt = self.acquire_temp_gpr().ok_or_else(|| {
1483 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1484 })?;
1485 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
1486 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1487 })?;
1488 self.move_location(sz, src, Location::GPR(arg))?;
1489
1490 let one_imm = match sz {
1491 Size::S32 => Location::Imm32(1),
1492 Size::S64 => Location::Imm64(1),
1493 _ => codegen_error!("singlepass emit_ctz unreachable"),
1494 };
1495
1496 let label_loop = self.assembler.get_label();
1497 let label_exit = self.assembler.get_label();
1498
1499 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
1500 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1501 })?;
1502
1503 self.move_location(sz, Location::Imm32(size_bits), Location::GPR(cnt))?;
1505 self.assembler
1506 .emit_on_false_label(Location::GPR(arg), label_exit, jmp_tmp)?;
1507
1508 self.move_location(sz, Location::Imm32(0), Location::GPR(cnt))?;
1509
1510 self.assembler.emit_label(label_loop)?;
1512 self.move_location(
1514 Size::S32,
1515 Location::Imm32(size_bits - 1),
1516 Location::GPR(tmp),
1517 )?;
1518 self.assembler.emit_sub(
1519 Size::S32,
1520 Location::GPR(tmp),
1521 Location::GPR(cnt),
1522 Location::GPR(tmp),
1523 )?;
1524 self.assembler.emit_srl(
1525 sz,
1526 Location::GPR(arg),
1527 Location::GPR(tmp),
1528 Location::GPR(tmp),
1529 )?;
1530 self.assembler
1531 .emit_on_true_label(Location::GPR(tmp), label_exit, jmp_tmp)?;
1532
1533 self.assembler
1534 .emit_add(sz, Location::GPR(cnt), one_imm, Location::GPR(cnt))?;
1535 self.release_gpr(jmp_tmp);
1536 self.jmp_unconditional(label_loop)?;
1537
1538 self.assembler.emit_label(label_exit)?; self.move_location(sz, Location::GPR(cnt), dst)?;
1541
1542 self.release_gpr(arg);
1543 self.release_gpr(cnt);
1544 self.release_gpr(tmp);
1545 Ok(())
1546 }
1547
1548 fn convert_float_to_int(
1549 &mut self,
1550 loc: Location,
1551 size_in: Size,
1552 ret: Location,
1553 size_out: Size,
1554 signed: bool,
1555 sat: bool,
1556 ) -> Result<(), CompileError> {
1557 let mut gprs = vec![];
1558 let mut fprs = vec![];
1559 let src = self.location_to_fpr(size_in, loc, &mut fprs, ImmType::None, true)?;
1560 let dest = self.location_to_reg(size_out, ret, &mut gprs, ImmType::None, false, None)?;
1561
1562 if sat {
1563 let end = self.assembler.get_label();
1566 let cond = self.acquire_temp_gpr().ok_or_else(|| {
1567 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1568 })?;
1569 self.zero_location(size_out, dest)?;
1570 self.assembler
1572 .emit_fcmp(Condition::Eq, size_in, src, src, Location::GPR(cond))?;
1573 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
1574 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1575 })?;
1576 self.assembler
1577 .emit_on_false_label(Location::GPR(cond), end, jmp_tmp)?;
1578 self.release_gpr(jmp_tmp);
1579 self.release_gpr(cond);
1580
1581 self.assembler
1582 .emit_fcvt(signed, size_in, src, size_out, dest)?;
1583 self.emit_label(end)?;
1584 } else {
1585 let old_fcsr = self.acquire_temp_gpr().ok_or_else(|| {
1586 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1587 })?;
1588 self.move_location(
1589 Size::S32,
1590 Location::GPR(GPR::XZero),
1591 Location::GPR(old_fcsr),
1592 )?;
1593 self.assembler.emit_swap_fscr(old_fcsr)?;
1594 self.assembler
1595 .emit_fcvt(signed, size_in, src, size_out, dest)?;
1596 self.trap_float_convertion_errors(size_in, src, old_fcsr, &mut gprs)?;
1597 self.release_gpr(old_fcsr);
1598 }
1599
1600 if ret != dest {
1601 self.move_location(size_out, dest, ret)?;
1602 }
1603 for r in gprs {
1604 self.release_gpr(r);
1605 }
1606 for r in fprs {
1607 self.release_simd(r);
1608 }
1609 Ok(())
1610 }
1611
1612 fn convert_int_to_float(
1613 &mut self,
1614 loc: Location,
1615 size_in: Size,
1616 ret: Location,
1617 size_out: Size,
1618 signed: bool,
1619 ) -> Result<(), CompileError> {
1620 let mut gprs = vec![];
1621 let mut fprs = vec![];
1622 let src = self.location_to_reg(size_in, loc, &mut gprs, ImmType::None, true, None)?;
1623 let dest = self.location_to_fpr(size_out, ret, &mut fprs, ImmType::None, false)?;
1624 self.assembler
1625 .emit_fcvt(signed, size_in, src, size_out, dest)?;
1626 if ret != dest {
1627 self.move_location(Size::S32, dest, ret)?;
1628 }
1629 for r in gprs {
1630 self.release_gpr(r);
1631 }
1632 for r in fprs {
1633 self.release_simd(r);
1634 }
1635 Ok(())
1636 }
1637
1638 fn convert_float_to_float(
1639 &mut self,
1640 loc: Location,
1641 size_in: Size,
1642 ret: Location,
1643 size_out: Size,
1644 ) -> Result<(), CompileError> {
1645 let mut temps = vec![];
1646 let src = self.location_to_fpr(size_in, loc, &mut temps, ImmType::None, true)?;
1647 let dest = self.location_to_fpr(size_out, ret, &mut temps, ImmType::None, false)?;
1648
1649 match (size_in, size_out) {
1650 (Size::S32, Size::S64) => self
1651 .assembler
1652 .emit_fcvt(false, size_in, src, size_out, dest)?,
1653 (Size::S64, Size::S32) => self
1654 .assembler
1655 .emit_fcvt(false, size_in, src, size_out, dest)?,
1656 _ => codegen_error!("singlepass convert_float_to_float unreachable"),
1657 }
1658
1659 if ret != dest {
1660 self.move_location(size_out, dest, ret)?;
1661 }
1662 for r in temps {
1663 self.release_simd(r);
1664 }
1665 Ok(())
1666 }
1667
1668 fn trap_float_convertion_errors(
1669 &mut self,
1670 sz: Size,
1671 f: Location,
1672 old_fcsr: GPR,
1673 temps: &mut Vec<GPR>,
1674 ) -> Result<(), CompileError> {
1675 let fscr = self.acquire_temp_gpr().ok_or_else(|| {
1676 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1677 })?;
1678 self.zero_location(Size::S64, Location::GPR(fscr))?;
1679 temps.push(fscr);
1680
1681 let trap_badconv = self.assembler.get_label();
1682 let end = self.assembler.get_label();
1683
1684 self.assembler.emit_swap_fscr(fscr)?;
1685
1686 self.assembler.emit_srl(
1689 Size::S32,
1690 Location::GPR(fscr),
1691 Location::Imm32(4),
1692 Location::GPR(fscr),
1693 )?;
1694
1695 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
1696 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1697 })?;
1698 temps.push(jmp_tmp);
1699
1700 self.assembler
1701 .emit_on_false_label(Location::GPR(fscr), end, jmp_tmp)?;
1702
1703 let cond = self.acquire_temp_gpr().ok_or_else(|| {
1705 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1706 })?;
1707 temps.push(cond);
1708
1709 self.assembler
1710 .emit_fcmp(Condition::Eq, sz, f, f, Location::GPR(cond))?;
1711 self.assembler
1713 .emit_on_false_label(Location::GPR(cond), trap_badconv, jmp_tmp)?;
1714 self.emit_illegal_op_internal(TrapCode::IntegerOverflow)?;
1715 self.emit_label(trap_badconv)?;
1716 self.emit_illegal_op_internal(TrapCode::BadConversionToInteger)?;
1717
1718 self.emit_label(end)?;
1719 self.assembler.emit_swap_fscr(old_fcsr)?;
1720
1721 Ok(())
1722 }
1723
1724 fn emit_illegal_op_internal(&mut self, trap: TrapCode) -> Result<(), CompileError> {
1725 self.assembler.emit_udf(trap as u8)
1726 }
1727
1728 fn emit_relaxed_fcmp(
1729 &mut self,
1730 c: Condition,
1731 size: Size,
1732 loc_a: Location,
1733 loc_b: Location,
1734 ret: Location,
1735 ) -> Result<(), CompileError> {
1736 let mut fprs = vec![];
1738 let mut gprs = vec![];
1739
1740 let loc_a = self.location_to_fpr(size, loc_a, &mut fprs, ImmType::None, true)?;
1741 let loc_b = self.location_to_fpr(size, loc_b, &mut fprs, ImmType::None, true)?;
1742 let dest = self.location_to_reg(size, ret, &mut gprs, ImmType::None, false, None)?;
1743
1744 self.assembler.emit_fcmp(c, size, loc_a, loc_b, dest)?;
1745 if ret != dest {
1746 self.move_location(size, dest, ret)?;
1747 }
1748 for r in fprs {
1749 self.release_simd(r);
1750 }
1751 for r in gprs {
1752 self.release_gpr(r);
1753 }
1754 Ok(())
1755 }
1756
1757 fn emit_relaxed_fcvt_with_rounding(
1758 &mut self,
1759 rounding: RoundingMode,
1760 size: Size,
1761 loc: Location,
1762 ret: Location,
1763 ) -> Result<(), CompileError> {
1764 let mut fprs = vec![];
1769 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
1770 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1771 })?;
1772
1773 let loc = self.location_to_fpr(size, loc, &mut fprs, ImmType::None, true)?;
1774 let dest = self.location_to_fpr(size, ret, &mut fprs, ImmType::None, false)?;
1775
1776 let cond = self.acquire_temp_gpr().ok_or_else(|| {
1777 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1778 })?;
1779 let tmp1 = self.acquire_temp_simd().ok_or_else(|| {
1780 CompileError::Codegen("singlepass cannot acquire temp fpr".to_owned())
1781 })?;
1782 let tmp2 = self.acquire_temp_simd().ok_or_else(|| {
1783 CompileError::Codegen("singlepass cannot acquire temp fpr".to_owned())
1784 })?;
1785
1786 let label_after = self.get_label();
1787
1788 let canonical_nan = match size {
1790 Size::S32 => CANONICAL_NAN_F32 as u64,
1791 Size::S64 => CANONICAL_NAN_F64,
1792 _ => unreachable!(),
1793 };
1794
1795 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
1796 CompileError::Codegen("singlepass cannot acquire temp fpr".to_owned())
1797 })?;
1798
1799 self.assembler
1800 .emit_mov_imm(Location::GPR(tmp), canonical_nan as _)?;
1801 self.assembler.emit_mov(size, Location::GPR(tmp), dest)?;
1802 self.assembler
1803 .emit_fcmp(Condition::Eq, size, loc, loc, Location::GPR(cond))?;
1804 self.assembler
1805 .emit_on_false_label(Location::GPR(cond), label_after, jmp_tmp)?;
1806
1807 if size == Size::S64 {
1809 self.assembler
1810 .emit_mov_imm(Location::GPR(cond), 0x4330000000000000)?;
1811 self.assembler
1812 .emit_mov(Size::S64, Location::GPR(cond), Location::SIMD(tmp1))?;
1813 self.f64_abs(loc, Location::SIMD(tmp2))?;
1814 } else {
1815 assert!(size == Size::S32);
1816 self.assembler
1817 .emit_mov_imm(Location::GPR(cond), 0x4b000000)?;
1818 self.assembler
1819 .emit_mov(Size::S32, Location::GPR(cond), Location::SIMD(tmp1))?;
1820 self.f32_abs(loc, Location::SIMD(tmp2))?;
1821 }
1822 self.emit_relaxed_fcmp(
1823 Condition::Lt,
1824 size,
1825 Location::SIMD(tmp2),
1826 Location::SIMD(tmp1),
1827 Location::GPR(cond),
1828 )?;
1829
1830 self.assembler.emit_mov(size, loc, dest)?;
1831 self.assembler
1832 .emit_on_false_label(Location::GPR(cond), label_after, jmp_tmp)?;
1833
1834 self.assembler
1836 .emit_fcvt_with_rounding(rounding, size, loc, dest, cond)?;
1837 self.emit_label(label_after)?;
1838
1839 if ret != dest {
1840 self.move_location(size, dest, ret)?;
1841 }
1842
1843 for r in fprs {
1844 self.release_simd(r);
1845 }
1846 self.release_gpr(jmp_tmp);
1847 self.release_gpr(tmp);
1848 self.release_gpr(cond);
1849 self.release_simd(tmp1);
1850 self.release_simd(tmp2);
1851 Ok(())
1852 }
1853
1854 fn emit_unwind_op(&mut self, op: UnwindOps<GPR, FPR>) {
1855 self.unwind_ops.push((self.get_offset().0, op));
1856 }
1857}
1858
1859pub(crate) const RISCV_RETURN_VALUE_REGISTERS: [GPR; 7] = [
1864 GPR::X10,
1865 GPR::X11,
1866 GPR::X12,
1867 GPR::X13,
1868 GPR::X14,
1869 GPR::X15,
1870 GPR::X16,
1871];
1872
1873#[allow(dead_code)]
1874#[derive(PartialEq, Copy, Clone)]
1875pub(crate) enum ImmType {
1876 None,
1877 Bits12,
1878 Bits12Subtraction,
1881 Shift32,
1882 Shift64,
1883}
1884
1885impl ImmType {
1886 pub(crate) fn compatible_imm(&self, imm: i64) -> bool {
1887 match self {
1888 ImmType::None => false,
1889 ImmType::Bits12 => (-0x800..0x800).contains(&imm),
1890 ImmType::Bits12Subtraction => (-0x801..0x801).contains(&imm),
1891 ImmType::Shift32 => (0..32).contains(&imm),
1892 ImmType::Shift64 => (0..64).contains(&imm),
1893 }
1894 }
1895}
1896
1897impl Machine for MachineRiscv {
1898 type GPR = GPR;
1899 type SIMD = FPR;
1900 fn assembler_get_offset(&self) -> Offset {
1901 self.assembler.get_offset()
1902 }
1903 fn get_vmctx_reg(&self) -> Self::GPR {
1904 GPR::X27
1906 }
1907 fn pick_gpr(&self) -> Option<Self::GPR> {
1908 use GPR::*;
1909 static REGS: &[GPR] = &[X5, X6, X7, X29, X30, X31];
1911 for r in REGS {
1912 if !self.used_gprs_contains(r) {
1913 return Some(*r);
1914 }
1915 }
1916 None
1917 }
1918
1919 fn pick_temp_gpr(&self) -> Option<GPR> {
1921 use GPR::*;
1922 static REGS: &[GPR] = &[X17, X16, X15, X14, X13, X12, X11, X10];
1926 for r in REGS {
1927 if !self.used_gprs_contains(r) {
1928 return Some(*r);
1929 }
1930 }
1931 None
1932 }
1933
1934 fn get_used_gprs(&self) -> Vec<Self::GPR> {
1935 GPR::iterator()
1936 .filter(|x| self.used_gprs & (1 << x.into_index()) != 0)
1937 .cloned()
1938 .collect()
1939 }
1940 fn get_used_simd(&self) -> Vec<Self::SIMD> {
1941 FPR::iterator()
1942 .filter(|x| self.used_fprs & (1 << x.into_index()) != 0)
1943 .cloned()
1944 .collect()
1945 }
1946 fn acquire_temp_gpr(&mut self) -> Option<Self::GPR> {
1947 let gpr = self.pick_temp_gpr();
1948 if let Some(x) = gpr {
1949 self.used_gprs_insert(x);
1950 }
1951 gpr
1952 }
1953 fn release_gpr(&mut self, gpr: Self::GPR) {
1954 assert!(self.used_gprs_remove(&gpr));
1955 }
1956 fn reserve_unused_temp_gpr(&mut self, gpr: Self::GPR) -> Self::GPR {
1957 assert!(!self.used_gprs_contains(&gpr));
1958 self.used_gprs_insert(gpr);
1959 gpr
1960 }
1961 fn reserve_gpr(&mut self, gpr: Self::GPR) {
1962 self.used_gprs_insert(gpr);
1963 }
1964 fn push_used_gpr(&mut self, used_gprs: &[Self::GPR]) -> Result<usize, CompileError> {
1965 for r in used_gprs.iter() {
1966 self.assembler.emit_push(Size::S64, Location::GPR(*r))?;
1967 }
1968 Ok(used_gprs.len() * 16)
1969 }
1970 fn pop_used_gpr(&mut self, used_gprs: &[Self::GPR]) -> Result<(), CompileError> {
1971 for r in used_gprs.iter().rev() {
1972 self.emit_pop(Size::S64, Location::GPR(*r))?;
1973 }
1974 Ok(())
1975 }
1976 fn pick_simd(&self) -> Option<Self::SIMD> {
1977 use FPR::*;
1978 static REGS: &[FPR] = &[F0, F1, F2, F3, F4, F5, F6, F7];
1979 for r in REGS {
1980 if !self.used_fp_contains(r) {
1981 return Some(*r);
1982 }
1983 }
1984 None
1985 }
1986
1987 fn pick_temp_simd(&self) -> Option<FPR> {
1989 use FPR::*;
1990 static REGS: &[FPR] = &[F28, F29, F31];
1991 for r in REGS {
1992 if !self.used_fp_contains(r) {
1993 return Some(*r);
1994 }
1995 }
1996 None
1997 }
1998
1999 fn acquire_temp_simd(&mut self) -> Option<Self::SIMD> {
2000 let fpr = self.pick_temp_simd();
2001 if let Some(x) = fpr {
2002 self.used_fprs_insert(x);
2003 }
2004 fpr
2005 }
2006 fn reserve_simd(&mut self, fpr: Self::SIMD) {
2007 self.used_fprs_insert(fpr);
2008 }
2009 fn release_simd(&mut self, fpr: Self::SIMD) {
2010 assert!(self.used_fprs_remove(&fpr));
2011 }
2012
2013 fn push_used_simd(&mut self, used_neons: &[Self::SIMD]) -> Result<usize, CompileError> {
2014 let stack_adjust = (used_neons.len() * 8) as u32;
2015 self.extend_stack(stack_adjust)?;
2016
2017 for (i, r) in used_neons.iter().enumerate() {
2018 self.assembler.emit_sd(
2019 Size::S64,
2020 Location::SIMD(*r),
2021 Location::Memory(GPR::Sp, (i * 8) as i32),
2022 )?;
2023 }
2024 Ok(stack_adjust as usize)
2025 }
2026 fn pop_used_simd(&mut self, used_neons: &[Self::SIMD]) -> Result<(), CompileError> {
2027 for (i, r) in used_neons.iter().enumerate() {
2028 self.assembler.emit_ld(
2029 Size::S64,
2030 false,
2031 Location::SIMD(*r),
2032 Location::Memory(GPR::Sp, (i * 8) as i32),
2033 )?;
2034 }
2035 let stack_adjust = (used_neons.len() * 8) as u32;
2036 self.assembler.emit_add(
2037 Size::S64,
2038 Location::GPR(GPR::Sp),
2039 Location::Imm64(stack_adjust as _),
2040 Location::GPR(GPR::Sp),
2041 )
2042 }
2043
2044 fn round_stack_adjust(&self, value: usize) -> usize {
2046 if value & 0xf != 0 {
2047 ((value >> 4) + 1) << 4
2048 } else {
2049 value
2050 }
2051 }
2052
2053 fn set_srcloc(&mut self, offset: u32) {
2055 self.src_loc = offset;
2056 }
2057
2058 fn mark_address_range_with_trap_code(&mut self, code: TrapCode, begin: usize, end: usize) {
2059 for i in begin..end {
2060 self.trap_table.offset_to_code.insert(i, code);
2061 }
2062 self.mark_instruction_address_end(begin);
2063 }
2064 fn mark_address_with_trap_code(&mut self, code: TrapCode) {
2065 let offset = self.assembler.get_offset().0;
2066 self.trap_table.offset_to_code.insert(offset, code);
2067 self.mark_instruction_address_end(offset);
2068 }
2069 fn mark_instruction_with_trap_code(&mut self, code: TrapCode) -> usize {
2071 let offset = self.assembler.get_offset().0;
2072 self.trap_table.offset_to_code.insert(offset, code);
2073 offset
2074 }
2075 fn mark_instruction_address_end(&mut self, begin: usize) {
2076 self.instructions_address_map.push(InstructionAddressMap {
2077 srcloc: SourceLoc::new(self.src_loc),
2078 code_offset: begin,
2079 code_len: self.assembler.get_offset().0 - begin,
2080 });
2081 }
2082 fn insert_stackoverflow(&mut self) {
2084 let offset = 0;
2085 self.trap_table
2086 .offset_to_code
2087 .insert(offset, TrapCode::StackOverflow);
2088 self.mark_instruction_address_end(offset);
2089 }
2090
2091 fn collect_trap_information(&self) -> Vec<TrapInformation> {
2093 self.trap_table
2094 .offset_to_code
2095 .clone()
2096 .into_iter()
2097 .map(|(offset, code)| TrapInformation {
2098 code_offset: offset as u32,
2099 trap_code: code,
2100 })
2101 .collect()
2102 }
2103
2104 fn instructions_address_map(&self) -> Vec<InstructionAddressMap> {
2105 self.instructions_address_map.clone()
2106 }
2107 fn local_on_stack(&mut self, stack_offset: i32) -> Location {
2108 Location::Memory(GPR::Fp, -stack_offset)
2109 }
2110 fn extend_stack(&mut self, delta_stack_offset: u32) -> Result<(), CompileError> {
2111 let delta = if ImmType::Bits12Subtraction.compatible_imm(delta_stack_offset as _) {
2112 Location::Imm64(delta_stack_offset as _)
2113 } else {
2114 self.assembler
2115 .emit_mov_imm(Location::GPR(SCRATCH_REG), delta_stack_offset as _)?;
2116 Location::GPR(SCRATCH_REG)
2117 };
2118 self.assembler.emit_sub(
2119 Size::S64,
2120 Location::GPR(GPR::Sp),
2121 delta,
2122 Location::GPR(GPR::Sp),
2123 )
2124 }
2125 fn truncate_stack(&mut self, delta_stack_offset: u32) -> Result<(), CompileError> {
2126 let delta = if ImmType::Bits12.compatible_imm(delta_stack_offset as _) {
2127 Location::Imm64(delta_stack_offset as _)
2128 } else {
2129 self.assembler
2130 .emit_mov_imm(Location::GPR(SCRATCH_REG), delta_stack_offset as _)?;
2131 Location::GPR(SCRATCH_REG)
2132 };
2133 self.assembler.emit_add(
2134 Size::S64,
2135 Location::GPR(GPR::Sp),
2136 delta,
2137 Location::GPR(GPR::Sp),
2138 )
2139 }
2140 fn zero_location(&mut self, size: Size, location: Location) -> Result<(), CompileError> {
2141 self.move_location(size, Location::GPR(GPR::XZero), location)
2142 }
2143 fn local_pointer(&self) -> Self::GPR {
2144 GPR::Fp
2145 }
2146 fn move_location_for_native(
2148 &mut self,
2149 _size: Size,
2150 loc: Location,
2151 dest: Location,
2152 ) -> Result<(), CompileError> {
2153 match loc {
2154 Location::Imm64(_)
2155 | Location::Imm32(_)
2156 | Location::Imm8(_)
2157 | Location::Memory(_, _) => {
2158 self.move_location(Size::S64, loc, Location::GPR(SCRATCH_REG))?;
2159 self.move_location(Size::S64, Location::GPR(SCRATCH_REG), dest)
2160 }
2161 _ => self.move_location(Size::S64, loc, dest),
2162 }
2163 }
2164
2165 fn is_local_on_stack(&self, idx: usize) -> bool {
2166 idx > 9
2167 }
2168
2169 fn get_local_location(&self, idx: usize, callee_saved_regs_size: usize) -> Location {
2171 match idx {
2173 0 => Location::GPR(GPR::X9),
2174 1 => Location::GPR(GPR::X18),
2175 2 => Location::GPR(GPR::X19),
2176 3 => Location::GPR(GPR::X20),
2177 4 => Location::GPR(GPR::X21),
2178 5 => Location::GPR(GPR::X22),
2179 6 => Location::GPR(GPR::X23),
2180 7 => Location::GPR(GPR::X24),
2181 8 => Location::GPR(GPR::X25),
2182 9 => Location::GPR(GPR::X26),
2183 _ => Location::Memory(GPR::Fp, -(((idx - 9) * 8 + callee_saved_regs_size) as i32)),
2184 }
2185 }
2186
2187 fn move_local(&mut self, stack_offset: i32, location: Location) -> Result<(), CompileError> {
2189 self.move_location(
2190 Size::S64,
2191 location,
2192 Location::Memory(GPR::Fp, -stack_offset),
2193 )?;
2194
2195 match location {
2196 Location::GPR(x) => self.emit_unwind_op(UnwindOps::SaveRegister {
2197 reg: UnwindRegister::GPR(x),
2198 bp_neg_offset: stack_offset,
2199 }),
2200 Location::SIMD(x) => self.emit_unwind_op(UnwindOps::SaveRegister {
2201 reg: UnwindRegister::FPR(x),
2202 bp_neg_offset: stack_offset,
2203 }),
2204 _ => (),
2205 }
2206 Ok(())
2207 }
2208 fn list_to_save(&self, _calling_convention: CallingConvention) -> Vec<Location> {
2209 vec![]
2210 }
2211
2212 fn get_param_registers(&self, _calling_convention: CallingConvention) -> &'static [Self::GPR] {
2214 &[
2215 GPR::X10,
2216 GPR::X11,
2217 GPR::X12,
2218 GPR::X13,
2219 GPR::X14,
2220 GPR::X15,
2221 GPR::X16,
2222 GPR::X17,
2223 ]
2224 }
2225
2226 fn get_param_location(
2227 &self,
2228 idx: usize,
2229 _sz: Size,
2230 stack_args: &mut usize,
2231 calling_convention: CallingConvention,
2232 ) -> Location {
2233 let register_params: &[GPR] = self.get_param_registers(calling_convention);
2234 if let Some(reg) = register_params.get(idx) {
2235 Location::GPR(*reg)
2236 } else {
2237 let loc = Location::Memory(GPR::Sp, *stack_args as i32);
2238 *stack_args += 8;
2239 loc
2240 }
2241 }
2242 fn get_call_param_location(
2244 &self,
2245 return_slots: usize,
2246 idx: usize,
2247 _sz: Size,
2248 stack_args: &mut usize,
2249 calling_convention: CallingConvention,
2250 ) -> Location {
2251 let return_values_memory_size =
2252 8 * return_slots.saturating_sub(RISCV_RETURN_VALUE_REGISTERS.len()) as i32;
2253 self.get_param_registers(calling_convention)
2254 .get(idx)
2255 .map_or_else(
2256 || {
2257 let loc = Location::Memory(
2258 GPR::Fp,
2259 16 + return_values_memory_size + *stack_args as i32,
2260 );
2261 *stack_args += 8;
2262 loc
2263 },
2264 |reg| Location::GPR(*reg),
2265 )
2266 }
2267 fn get_simple_param_location(
2268 &self,
2269 idx: usize,
2270 calling_convention: CallingConvention,
2271 ) -> Self::GPR {
2272 self.get_param_registers(calling_convention)[idx]
2273 }
2274
2275 fn get_return_value_location(
2276 &self,
2277 idx: usize,
2278 stack_location: &mut usize,
2279 _calling_convention: CallingConvention,
2280 ) -> AbstractLocation<Self::GPR, Self::SIMD> {
2281 RISCV_RETURN_VALUE_REGISTERS.get(idx).map_or_else(
2282 || {
2283 let loc = Location::Memory(GPR::Sp, *stack_location as i32);
2284 *stack_location += 8;
2285 loc
2286 },
2287 |reg| Location::GPR(*reg),
2288 )
2289 }
2290
2291 fn get_call_return_value_location(
2292 &self,
2293 idx: usize,
2294 _calling_convention: CallingConvention,
2295 ) -> AbstractLocation<Self::GPR, Self::SIMD> {
2296 RISCV_RETURN_VALUE_REGISTERS.get(idx).map_or_else(
2297 || {
2298 Location::Memory(
2299 GPR::Fp,
2300 (16 + (idx - RISCV_RETURN_VALUE_REGISTERS.len()) * 8) as i32,
2301 )
2302 },
2303 |reg| Location::GPR(*reg),
2304 )
2305 }
2306
2307 fn move_location(
2309 &mut self,
2310 size: Size,
2311 source: Location,
2312 dest: Location,
2313 ) -> Result<(), CompileError> {
2314 match (source, dest) {
2315 (Location::GPR(_), Location::GPR(_)) => self.assembler.emit_mov(size, source, dest),
2316 (Location::Imm32(_) | Location::Imm64(_), Location::GPR(_)) => self
2317 .assembler
2318 .emit_mov_imm(dest, source.imm_value_scalar().unwrap()),
2319 (Location::GPR(_), Location::Memory(addr, offset)) => {
2320 let addr = if ImmType::Bits12.compatible_imm(offset as _) {
2321 dest
2322 } else {
2323 self.assembler
2324 .emit_mov_imm(Location::GPR(SCRATCH_REG), offset as _)?;
2325 self.assembler.emit_add(
2326 Size::S64,
2327 Location::GPR(addr),
2328 Location::GPR(SCRATCH_REG),
2329 Location::GPR(SCRATCH_REG),
2330 )?;
2331 Location::Memory(SCRATCH_REG, 0)
2332 };
2333 self.assembler.emit_sd(size, source, addr)
2334 }
2335 (Location::SIMD(_), Location::Memory(..)) => {
2336 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2337 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2338 })?;
2339 self.move_location(size, source, Location::GPR(tmp))?;
2340 self.move_location(size, Location::GPR(tmp), dest)?;
2341 self.release_gpr(tmp);
2342 Ok(())
2343 }
2344 (Location::Memory(addr, offset), Location::GPR(_)) => {
2345 let addr = if ImmType::Bits12.compatible_imm(offset as _) {
2346 source
2347 } else {
2348 self.assembler
2349 .emit_mov_imm(Location::GPR(SCRATCH_REG), offset as _)?;
2350 self.assembler.emit_add(
2351 Size::S64,
2352 Location::GPR(addr),
2353 Location::GPR(SCRATCH_REG),
2354 Location::GPR(SCRATCH_REG),
2355 )?;
2356 Location::Memory(SCRATCH_REG, 0)
2357 };
2358 self.assembler.emit_ld(size, false, dest, addr)
2359 }
2360 (Location::GPR(_), Location::SIMD(_)) => self.assembler.emit_mov(size, source, dest),
2361 (Location::SIMD(_), Location::GPR(_)) => self.assembler.emit_mov(size, source, dest),
2362 (Location::SIMD(_), Location::SIMD(_)) => self.assembler.emit_mov(size, source, dest),
2363 _ => todo!("unsupported move: {size:?} {source:?} {dest:?}"),
2364 }
2365 }
2366
2367 fn move_location_extend(
2369 &mut self,
2370 size_val: Size,
2371 signed: bool,
2372 source: Location,
2373 size_op: Size,
2374 dest: Location,
2375 ) -> Result<(), CompileError> {
2376 if size_op != Size::S64 {
2377 codegen_error!("singlepass move_location_extend unreachable");
2378 }
2379 let mut temps = vec![];
2380 let dst = self.location_to_reg(size_op, dest, &mut temps, ImmType::None, false, None)?;
2381 let src = match (size_val, signed, source) {
2382 (Size::S64, _, _) => source,
2383 (_, _, Location::GPR(_)) => {
2384 self.assembler.emit_extend(size_val, signed, source, dst)?;
2385 dst
2386 }
2387 (_, _, Location::Memory(_, _)) => {
2388 self.assembler.emit_ld(size_val, signed, dst, source)?;
2389 dst
2390 }
2391 _ => codegen_error!(
2392 "singlepass can't emit move_location_extend {:?} {:?} {:?} => {:?} {:?}",
2393 size_val,
2394 signed,
2395 source,
2396 size_op,
2397 dest
2398 ),
2399 };
2400 if src != dst {
2401 self.move_location(size_op, src, dst)?;
2402 }
2403 if dst != dest {
2404 self.move_location(size_op, dst, dest)?;
2405 }
2406 for r in temps {
2407 self.release_gpr(r);
2408 }
2409 Ok(())
2410 }
2411
2412 fn init_stack_loc(
2414 &mut self,
2415 init_stack_loc_cnt: u64,
2416 last_stack_loc: Location,
2417 ) -> Result<(), CompileError> {
2418 let label = self.assembler.get_label();
2419 let mut temps = vec![];
2420 let dest = self.acquire_temp_gpr().ok_or_else(|| {
2421 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2422 })?;
2423 temps.push(dest);
2424 let cnt = self.location_to_reg(
2425 Size::S64,
2426 Location::Imm64(init_stack_loc_cnt),
2427 &mut temps,
2428 ImmType::None,
2429 true,
2430 None,
2431 )?;
2432 match last_stack_loc {
2433 Location::GPR(_) => codegen_error!("singlepass init_stack_loc unreachable"),
2434 Location::SIMD(_) => codegen_error!("singlepass init_stack_loc unreachable"),
2435 Location::Memory(reg, offset) => {
2436 if ImmType::Bits12.compatible_imm(offset as _) {
2437 self.assembler.emit_add(
2438 Size::S64,
2439 Location::GPR(reg),
2440 Location::Imm64(offset as _),
2441 Location::GPR(dest),
2442 )?;
2443 } else {
2444 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2445 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2446 })?;
2447 self.assembler
2448 .emit_mov_imm(Location::GPR(tmp), offset as _)?;
2449 self.assembler.emit_add(
2450 Size::S64,
2451 Location::GPR(reg),
2452 Location::GPR(tmp),
2453 Location::GPR(dest),
2454 )?;
2455 temps.push(tmp);
2456 }
2457 }
2458 _ => codegen_error!("singlepass can't emit init_stack_loc {:?}", last_stack_loc),
2459 };
2460 self.assembler.emit_label(label)?;
2461 self.assembler.emit_sd(
2462 Size::S64,
2463 Location::GPR(GPR::XZero),
2464 Location::Memory(dest, 0),
2465 )?;
2466 self.assembler
2467 .emit_sub(Size::S64, cnt, Location::Imm64(1), cnt)?;
2468 self.assembler.emit_add(
2469 Size::S64,
2470 Location::GPR(dest),
2471 Location::Imm64(8),
2472 Location::GPR(dest),
2473 )?;
2474 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
2475 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2476 })?;
2477 temps.push(jmp_tmp);
2478 self.assembler.emit_on_true_label(cnt, label, jmp_tmp)?;
2479 for r in temps {
2480 self.release_gpr(r);
2481 }
2482 Ok(())
2483 }
2484
2485 fn restore_saved_area(&mut self, saved_area_offset: i32) -> Result<(), CompileError> {
2487 self.assembler.emit_sub(
2488 Size::S64,
2489 Location::GPR(GPR::Fp),
2490 Location::Imm64(saved_area_offset as _),
2491 Location::GPR(GPR::Sp),
2492 )
2493 }
2494
2495 fn pop_location(&mut self, location: Location) -> Result<(), CompileError> {
2497 self.emit_pop(Size::S64, location)
2498 }
2499 fn assembler_finalize(
2500 self,
2501 assembly_comments: HashMap<usize, AssemblyComment>,
2502 ) -> Result<FinalizedAssembly, CompileError> {
2503 Ok(FinalizedAssembly {
2504 body: self.assembler.finalize().map_err(|e| {
2505 CompileError::Codegen(format!("Assembler failed finalization with: {e:?}"))
2506 })?,
2507 assembly_comments,
2508 })
2509 }
2510 fn get_offset(&self) -> Offset {
2511 self.assembler.get_offset()
2512 }
2513 fn finalize_function(&mut self) -> Result<(), CompileError> {
2514 self.assembler.finalize_function()?;
2515 Ok(())
2516 }
2517
2518 fn emit_function_prolog(&mut self) -> Result<(), CompileError> {
2519 self.assembler.emit_sub(
2520 Size::S64,
2521 Location::GPR(GPR::Sp),
2522 Location::Imm64(16),
2523 Location::GPR(GPR::Sp),
2524 )?;
2525 self.emit_unwind_op(UnwindOps::SubtractFP { up_to_sp: 16 });
2526
2527 self.assembler.emit_sd(
2528 Size::S64,
2529 Location::GPR(GPR::X1), Location::Memory(GPR::Sp, 8),
2531 )?;
2532 self.assembler.emit_sd(
2533 Size::S64,
2534 Location::GPR(GPR::Fp),
2535 Location::Memory(GPR::Sp, 0),
2536 )?;
2537 self.emit_unwind_op(UnwindOps::SaveRegister {
2538 reg: UnwindRegister::GPR(GPR::X1),
2539 bp_neg_offset: 8,
2540 });
2541 self.emit_unwind_op(UnwindOps::SaveRegister {
2542 reg: UnwindRegister::GPR(GPR::Fp),
2543 bp_neg_offset: 16,
2544 });
2545
2546 self.assembler
2547 .emit_mov(Size::S64, Location::GPR(GPR::Sp), Location::GPR(GPR::Fp))?;
2548 self.emit_unwind_op(UnwindOps::DefineNewFrame);
2549 Ok(())
2550 }
2551
2552 fn emit_function_epilog(&mut self) -> Result<(), CompileError> {
2553 self.assembler
2554 .emit_mov(Size::S64, Location::GPR(GPR::Fp), Location::GPR(GPR::Sp))?;
2555 self.assembler.emit_ld(
2556 Size::S64,
2557 false,
2558 Location::GPR(GPR::X1), Location::Memory(GPR::Sp, 8),
2560 )?;
2561 self.assembler.emit_ld(
2562 Size::S64,
2563 false,
2564 Location::GPR(GPR::Fp),
2565 Location::Memory(GPR::Sp, 0),
2566 )?;
2567 self.assembler.emit_add(
2568 Size::S64,
2569 Location::GPR(GPR::Sp),
2570 Location::Imm64(16),
2571 Location::GPR(GPR::Sp),
2572 )?;
2573
2574 Ok(())
2575 }
2576 fn emit_function_return_float(&mut self) -> Result<(), CompileError> {
2577 self.assembler
2578 .emit_mov(Size::S64, Location::GPR(GPR::X10), Location::SIMD(FPR::F10))
2579 }
2580 fn canonicalize_nan(
2581 &mut self,
2582 sz: Size,
2583 input: Location,
2584 output: Location,
2585 ) -> Result<(), CompileError> {
2586 let mut temps = vec![];
2587 match (sz, input, output) {
2589 (Size::S32, Location::SIMD(_), Location::SIMD(_)) => {
2590 self.assembler.emit_fmax(sz, input, input, output)?;
2591 }
2592 (Size::S64, Location::SIMD(_), Location::SIMD(_)) => {
2593 self.assembler.emit_fmax(sz, input, input, output)?;
2594 }
2595 (Size::S32, Location::SIMD(_), _) | (Size::S64, Location::SIMD(_), _) => {
2596 let tmp = self.location_to_fpr(sz, output, &mut temps, ImmType::None, false)?;
2597 self.assembler.emit_fmax(sz, input, input, tmp)?;
2598 self.move_location(sz, tmp, output)?;
2599 }
2600 (Size::S32, Location::Memory(_, _), _) | (Size::S64, Location::Memory(_, _), _) => {
2601 let src = self.location_to_fpr(sz, input, &mut temps, ImmType::None, true)?;
2602 let tmp = self.location_to_fpr(sz, output, &mut temps, ImmType::None, false)?;
2603 self.assembler.emit_fmax(sz, src, src, tmp)?;
2604 if tmp != output {
2605 self.move_location(sz, tmp, output)?;
2606 }
2607 }
2608 _ => codegen_error!(
2609 "singlepass can't emit canonicalize_nan {:?} {:?} {:?}",
2610 sz,
2611 input,
2612 output
2613 ),
2614 }
2615
2616 for r in temps {
2617 self.release_simd(r);
2618 }
2619 Ok(())
2620 }
2621 fn emit_illegal_op(&mut self, trap: TrapCode) -> Result<(), CompileError> {
2622 let offset = self.assembler.get_offset().0;
2623 self.assembler.emit_udf(trap as u8)?;
2624 self.mark_instruction_address_end(offset);
2625 Ok(())
2626 }
2627 fn get_label(&mut self) -> Label {
2628 self.assembler.new_dynamic_label()
2629 }
2630 fn emit_label(&mut self, label: Label) -> Result<(), CompileError> {
2631 self.assembler.emit_label(label)
2632 }
2633 fn get_gpr_for_call(&self) -> Self::GPR {
2634 GPR::X1
2635 }
2636 fn emit_call_register(&mut self, register: Self::GPR) -> Result<(), CompileError> {
2637 self.assembler.emit_call_register(register)
2638 }
2639 fn emit_call_label(&mut self, label: Label) -> Result<(), CompileError> {
2640 self.assembler.emit_call_label(label)
2641 }
2642 fn arch_emit_indirect_call_with_trampoline(
2643 &mut self,
2644 _location: Location,
2645 ) -> Result<(), CompileError> {
2646 codegen_error!("singlepass arch_emit_indirect_call_with_trampoline unimplemented")
2647 }
2648 fn emit_call_location(&mut self, location: Location) -> Result<(), CompileError> {
2649 let mut temps = vec![];
2650 let loc = self.location_to_reg(
2651 Size::S64,
2652 location,
2653 &mut temps,
2654 ImmType::None,
2655 true,
2656 Some(self.get_gpr_for_call()),
2657 )?;
2658 match loc {
2659 Location::GPR(reg) => self.assembler.emit_call_register(reg),
2660 _ => codegen_error!("singlepass can't emit CALL Location"),
2661 }?;
2662 for r in temps {
2663 self.release_gpr(r);
2664 }
2665 Ok(())
2666 }
2667 fn emit_debug_breakpoint(&mut self) -> Result<(), CompileError> {
2668 self.assembler.emit_brk()
2669 }
2670 fn location_add(
2671 &mut self,
2672 size: Size,
2673 source: Location,
2674 dest: Location,
2675 _flags: bool,
2676 ) -> Result<(), CompileError> {
2677 let mut temps = vec![];
2678 let src = self.location_to_reg(size, source, &mut temps, ImmType::Bits12, true, None)?;
2679 let dst = self.location_to_reg(size, dest, &mut temps, ImmType::None, true, None)?;
2680 self.assembler.emit_add(size, dst, src, dst)?;
2681 if dst != dest {
2682 self.move_location(size, dst, dest)?;
2683 }
2684 for r in temps {
2685 self.release_gpr(r);
2686 }
2687 Ok(())
2688 }
2689 fn location_cmp(
2690 &mut self,
2691 _size: Size,
2692 _source: Location,
2693 _dest: Location,
2694 ) -> Result<(), CompileError> {
2695 codegen_error!("singlepass location_cmp not implemented")
2696 }
2697 fn jmp_unconditional(&mut self, label: Label) -> Result<(), CompileError> {
2698 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2699 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2700 })?;
2701 self.assembler.emit_j_label(label, Some(tmp))?;
2702 self.release_gpr(tmp);
2703 Ok(())
2704 }
2705 fn jmp_on_condition(
2706 &mut self,
2707 cond: UnsignedCondition,
2708 size: Size,
2709 loc_a: AbstractLocation<Self::GPR, Self::SIMD>,
2710 loc_b: AbstractLocation<Self::GPR, Self::SIMD>,
2711 label: Label,
2712 ) -> Result<(), CompileError> {
2713 let c = match cond {
2714 UnsignedCondition::Equal => Condition::Eq,
2715 UnsignedCondition::NotEqual => Condition::Ne,
2716 UnsignedCondition::Above => Condition::Gtu,
2717 UnsignedCondition::AboveEqual => Condition::Geu,
2718 UnsignedCondition::Below => Condition::Ltu,
2719 UnsignedCondition::BelowEqual => Condition::Leu,
2720 };
2721
2722 let mut temps = vec![];
2723 let loc_a = self.location_to_reg(size, loc_a, &mut temps, ImmType::None, true, None)?;
2724 let loc_b = self.location_to_reg(size, loc_b, &mut temps, ImmType::None, true, None)?;
2725
2726 if size != Size::S64 {
2727 self.assembler.emit_extend(size, false, loc_a, loc_a)?;
2728 self.assembler.emit_extend(size, false, loc_b, loc_b)?;
2729 }
2730
2731 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2732 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2733 })?;
2734 temps.push(tmp);
2735 self.assembler
2736 .emit_jmp_on_condition(c, loc_a, loc_b, label, tmp)?;
2737
2738 for r in temps {
2739 self.release_gpr(r);
2740 }
2741 Ok(())
2742 }
2743
2744 fn emit_jmp_to_jumptable(&mut self, label: Label, cond: Location) -> Result<(), CompileError> {
2746 let tmp1 = self.acquire_temp_gpr().ok_or_else(|| {
2747 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2748 })?;
2749 let tmp2 = self.acquire_temp_gpr().ok_or_else(|| {
2750 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2751 })?;
2752
2753 self.assembler.emit_load_label(tmp1, label)?;
2754 self.move_location(Size::S32, cond, Location::GPR(tmp2))?;
2755
2756 self.assembler.emit_sll(
2759 Size::S32,
2760 Location::GPR(tmp2),
2761 Location::Imm32(3),
2762 Location::GPR(tmp2),
2763 )?;
2764 self.assembler.emit_add(
2765 Size::S64,
2766 Location::GPR(tmp1),
2767 Location::GPR(tmp2),
2768 Location::GPR(tmp2),
2769 )?;
2770 self.assembler.emit_j_register(tmp2)?;
2771 self.release_gpr(tmp2);
2772 self.release_gpr(tmp1);
2773 Ok(())
2774 }
2775
2776 fn align_for_loop(&mut self) -> Result<(), CompileError> {
2777 Ok(())
2779 }
2780 fn emit_ret(&mut self) -> Result<(), CompileError> {
2781 self.assembler.emit_ret()
2782 }
2783 fn emit_push(&mut self, size: Size, loc: Location) -> Result<(), CompileError> {
2784 self.assembler.emit_push(size, loc)
2785 }
2786 fn emit_pop(&mut self, size: Size, loc: Location) -> Result<(), CompileError> {
2787 self.assembler.emit_pop(size, loc)
2788 }
2789 fn emit_relaxed_mov(
2791 &mut self,
2792 sz: Size,
2793 src: Location,
2794 dst: Location,
2795 ) -> Result<(), CompileError> {
2796 self.emit_relaxed_binop(Assembler::emit_mov, sz, src, dst)
2797 }
2798 fn emit_relaxed_cmp(
2799 &mut self,
2800 _sz: Size,
2801 _src: Location,
2802 _dst: Location,
2803 ) -> Result<(), CompileError> {
2804 todo!();
2805 }
2806 fn emit_memory_fence(&mut self) -> Result<(), CompileError> {
2807 self.assembler.emit_rwfence()
2808 }
2809 fn emit_relaxed_sign_extension(
2810 &mut self,
2811 sz_src: Size,
2812 src: Location,
2813 sz_dst: Size,
2814 dst: Location,
2815 ) -> Result<(), CompileError> {
2816 let mut temps = vec![];
2817
2818 match (src, dst) {
2819 (Location::Memory(reg, offset), Location::GPR(_)) => {
2820 let src = if ImmType::Bits12.compatible_imm(offset as _) {
2821 src
2822 } else {
2823 let tmp =
2824 self.location_to_reg(sz_src, src, &mut temps, ImmType::None, true, None)?;
2825 self.assembler.emit_mov_imm(tmp, offset as _)?;
2826 self.assembler
2827 .emit_add(Size::S64, Location::GPR(reg), tmp, tmp)?;
2828 let Location::GPR(tmp) = tmp else {
2829 unreachable!()
2830 };
2831 Location::Memory(tmp, 0)
2832 };
2833 self.assembler.emit_ld(sz_src, true, dst, src)?;
2834 }
2835 _ => {
2836 let src =
2837 self.location_to_reg(sz_src, src, &mut temps, ImmType::None, true, None)?;
2838 let dest =
2839 self.location_to_reg(sz_dst, dst, &mut temps, ImmType::None, false, None)?;
2840 self.assembler.emit_extend(sz_src, true, src, dest)?;
2841 if dst != dest {
2842 self.move_location(sz_dst, dest, dst)?;
2843 }
2844 }
2845 }
2846
2847 for r in temps {
2848 self.release_gpr(r);
2849 }
2850
2851 Ok(())
2852 }
2853 fn emit_imul_imm32(
2854 &mut self,
2855 size: Size,
2856 imm32: u32,
2857 gpr: Self::GPR,
2858 ) -> Result<(), CompileError> {
2859 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
2860 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2861 })?;
2862 self.assembler
2863 .emit_mov_imm(Location::GPR(tmp), imm32 as _)?;
2864 self.assembler.emit_mul(
2865 size,
2866 Location::GPR(gpr),
2867 Location::GPR(tmp),
2868 Location::GPR(gpr),
2869 )?;
2870 self.release_gpr(tmp);
2871 Ok(())
2872 }
2873 fn emit_binop_add32(
2874 &mut self,
2875 loc_a: Location,
2876 loc_b: Location,
2877 ret: Location,
2878 ) -> Result<(), CompileError> {
2879 self.emit_relaxed_binop3(
2880 Assembler::emit_add,
2881 Size::S32,
2882 loc_a,
2883 loc_b,
2884 ret,
2885 ImmType::Bits12,
2886 )
2887 }
2888 fn emit_binop_sub32(
2889 &mut self,
2890 loc_a: Location,
2891 loc_b: Location,
2892 ret: Location,
2893 ) -> Result<(), CompileError> {
2894 self.emit_relaxed_binop3(
2895 Assembler::emit_sub,
2896 Size::S32,
2897 loc_a,
2898 loc_b,
2899 ret,
2900 ImmType::Bits12Subtraction,
2901 )
2902 }
2903 fn emit_binop_mul32(
2904 &mut self,
2905 loc_a: Location,
2906 loc_b: Location,
2907 ret: Location,
2908 ) -> Result<(), CompileError> {
2909 self.emit_relaxed_binop3(
2910 Assembler::emit_mul,
2911 Size::S32,
2912 loc_a,
2913 loc_b,
2914 ret,
2915 ImmType::None,
2916 )
2917 }
2918 fn emit_binop_udiv32(
2919 &mut self,
2920 loc_a: Location,
2921 loc_b: Location,
2922 ret: Location,
2923 integer_division_by_zero: Label,
2924 ) -> Result<usize, CompileError> {
2925 let mut temps = vec![];
2926 let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
2927 let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
2928 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
2929
2930 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
2931 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2932 })?;
2933 temps.push(jmp_tmp);
2934
2935 self.assembler
2936 .emit_on_false_label_far(src2, integer_division_by_zero, jmp_tmp)?;
2937 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
2938 self.assembler.emit_udiv(Size::S32, src1, src2, dest)?;
2939 if ret != dest {
2940 self.move_location(Size::S32, dest, ret)?;
2941 }
2942 for r in temps {
2943 self.release_gpr(r);
2944 }
2945 Ok(offset)
2946 }
2947 fn emit_binop_sdiv32(
2948 &mut self,
2949 loc_a: Location,
2950 loc_b: Location,
2951 ret: Location,
2952 integer_division_by_zero: Label,
2953 integer_overflow: Label,
2954 ) -> Result<usize, CompileError> {
2955 let mut temps = vec![];
2956 let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
2957 let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
2958 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
2959
2960 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
2961 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2962 })?;
2963 temps.push(jmp_tmp);
2964
2965 self.assembler
2966 .emit_on_false_label_far(src2, integer_division_by_zero, jmp_tmp)?;
2967 let label_nooverflow = self.assembler.get_label();
2968 let tmp = self.location_to_reg(
2969 Size::S32,
2970 Location::Imm32(i32::MIN as u32),
2971 &mut temps,
2972 ImmType::None,
2973 true,
2974 None,
2975 )?;
2976 self.assembler.emit_cmp(Condition::Ne, tmp, src1, tmp)?;
2977 self.assembler
2978 .emit_on_true_label(tmp, label_nooverflow, jmp_tmp)?;
2979 self.move_location(Size::S32, Location::Imm32(-1i32 as _), tmp)?;
2980 self.assembler.emit_cmp(Condition::Eq, tmp, src2, tmp)?;
2981 self.assembler
2982 .emit_on_true_label_far(tmp, integer_overflow, jmp_tmp)?;
2983 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
2984 self.assembler.emit_label(label_nooverflow)?;
2985 self.assembler.emit_sdiv(Size::S32, src1, src2, dest)?;
2986 if ret != dest {
2987 self.move_location(Size::S32, dest, ret)?;
2988 }
2989 for r in temps {
2990 self.release_gpr(r);
2991 }
2992 Ok(offset)
2993 }
2994 fn emit_binop_urem32(
2995 &mut self,
2996 loc_a: Location,
2997 loc_b: Location,
2998 ret: Location,
2999 integer_division_by_zero: Label,
3000 ) -> Result<usize, CompileError> {
3001 let mut temps = vec![];
3002 let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
3003 let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
3004 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3005
3006 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
3007 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3008 })?;
3009 temps.push(jmp_tmp);
3010
3011 self.assembler
3012 .emit_on_false_label_far(src2, integer_division_by_zero, jmp_tmp)?;
3013 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
3014 self.assembler.emit_urem(Size::S32, src1, src2, dest)?;
3015 if ret != dest {
3016 self.move_location(Size::S32, dest, ret)?;
3017 }
3018 for r in temps {
3019 self.release_gpr(r);
3020 }
3021 Ok(offset)
3022 }
3023 fn emit_binop_srem32(
3024 &mut self,
3025 loc_a: Location,
3026 loc_b: Location,
3027 ret: Location,
3028 integer_division_by_zero: Label,
3029 ) -> Result<usize, CompileError> {
3030 let mut temps = vec![];
3031 let src1 = self.location_to_reg(Size::S32, loc_a, &mut temps, ImmType::None, true, None)?;
3032 let src2 = self.location_to_reg(Size::S32, loc_b, &mut temps, ImmType::None, true, None)?;
3033 let dest = self.location_to_reg(Size::S32, ret, &mut temps, ImmType::None, false, None)?;
3034
3035 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
3036 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3037 })?;
3038 temps.push(jmp_tmp);
3039
3040 self.assembler
3041 .emit_on_false_label_far(src2, integer_division_by_zero, jmp_tmp)?;
3042 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
3043 self.assembler.emit_srem(Size::S32, src1, src2, dest)?;
3044 if ret != dest {
3045 self.move_location(Size::S32, dest, ret)?;
3046 }
3047 for r in temps {
3048 self.release_gpr(r);
3049 }
3050 Ok(offset)
3051 }
3052 fn emit_binop_and32(
3053 &mut self,
3054 loc_a: Location,
3055 loc_b: Location,
3056 ret: Location,
3057 ) -> Result<(), CompileError> {
3058 self.emit_relaxed_binop3(
3059 Assembler::emit_and,
3060 Size::S32,
3061 loc_a,
3062 loc_b,
3063 ret,
3064 ImmType::Bits12,
3065 )
3066 }
3067 fn emit_binop_or32(
3068 &mut self,
3069 loc_a: Location,
3070 loc_b: Location,
3071 ret: Location,
3072 ) -> Result<(), CompileError> {
3073 self.emit_relaxed_binop3(
3074 Assembler::emit_or,
3075 Size::S32,
3076 loc_a,
3077 loc_b,
3078 ret,
3079 ImmType::Bits12,
3080 )
3081 }
3082 fn emit_binop_xor32(
3083 &mut self,
3084 loc_a: Location,
3085 loc_b: Location,
3086 ret: Location,
3087 ) -> Result<(), CompileError> {
3088 self.emit_relaxed_binop3(
3089 Assembler::emit_xor,
3090 Size::S32,
3091 loc_a,
3092 loc_b,
3093 ret,
3094 ImmType::Bits12,
3095 )
3096 }
3097 fn i32_cmp_ge_s(
3098 &mut self,
3099 loc_a: Location,
3100 loc_b: Location,
3101 ret: Location,
3102 ) -> Result<(), CompileError> {
3103 self.emit_cmpop_i32_dynamic_b(Condition::Ge, loc_a, loc_b, ret, true)
3104 }
3105 fn i32_cmp_gt_s(
3106 &mut self,
3107 loc_a: Location,
3108 loc_b: Location,
3109 ret: Location,
3110 ) -> Result<(), CompileError> {
3111 self.emit_cmpop_i32_dynamic_b(Condition::Gt, loc_a, loc_b, ret, true)
3112 }
3113 fn i32_cmp_le_s(
3114 &mut self,
3115 loc_a: Location,
3116 loc_b: Location,
3117 ret: Location,
3118 ) -> Result<(), CompileError> {
3119 self.emit_cmpop_i32_dynamic_b(Condition::Le, loc_a, loc_b, ret, true)
3120 }
3121 fn i32_cmp_lt_s(
3122 &mut self,
3123 loc_a: Location,
3124 loc_b: Location,
3125 ret: Location,
3126 ) -> Result<(), CompileError> {
3127 self.emit_cmpop_i32_dynamic_b(Condition::Lt, loc_a, loc_b, ret, true)
3128 }
3129 fn i32_cmp_ge_u(
3130 &mut self,
3131 loc_a: Location,
3132 loc_b: Location,
3133 ret: Location,
3134 ) -> Result<(), CompileError> {
3135 self.emit_cmpop_i32_dynamic_b(Condition::Geu, loc_a, loc_b, ret, false)
3136 }
3137 fn i32_cmp_gt_u(
3138 &mut self,
3139 loc_a: Location,
3140 loc_b: Location,
3141 ret: Location,
3142 ) -> Result<(), CompileError> {
3143 self.emit_cmpop_i32_dynamic_b(Condition::Gtu, loc_a, loc_b, ret, false)
3144 }
3145 fn i32_cmp_le_u(
3146 &mut self,
3147 loc_a: Location,
3148 loc_b: Location,
3149 ret: Location,
3150 ) -> Result<(), CompileError> {
3151 self.emit_cmpop_i32_dynamic_b(Condition::Leu, loc_a, loc_b, ret, false)
3152 }
3153 fn i32_cmp_lt_u(
3154 &mut self,
3155 loc_a: Location,
3156 loc_b: Location,
3157 ret: Location,
3158 ) -> Result<(), CompileError> {
3159 self.emit_cmpop_i32_dynamic_b(Condition::Ltu, loc_a, loc_b, ret, false)
3160 }
3161 fn i32_cmp_ne(
3162 &mut self,
3163 loc_a: Location,
3164 loc_b: Location,
3165 ret: Location,
3166 ) -> Result<(), CompileError> {
3167 self.emit_cmpop_i32_dynamic_b(Condition::Ne, loc_a, loc_b, ret, true)
3168 }
3169 fn i32_cmp_eq(
3170 &mut self,
3171 loc_a: Location,
3172 loc_b: Location,
3173 ret: Location,
3174 ) -> Result<(), CompileError> {
3175 self.emit_cmpop_i32_dynamic_b(Condition::Eq, loc_a, loc_b, ret, true)
3176 }
3177 fn i32_clz(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
3178 self.emit_clz(Size::S32, loc, ret)
3179 }
3180 fn i32_ctz(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
3181 self.emit_ctz(Size::S32, loc, ret)
3182 }
3183 fn i32_popcnt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
3184 self.emit_popcnt(Size::S32, loc, ret)
3185 }
3186 fn i32_shl(
3187 &mut self,
3188 loc_a: Location,
3189 loc_b: Location,
3190 ret: Location,
3191 ) -> Result<(), CompileError> {
3192 self.emit_relaxed_binop3(
3193 Assembler::emit_sll,
3194 Size::S32,
3195 loc_a,
3196 loc_b,
3197 ret,
3198 ImmType::Shift32,
3199 )
3200 }
3201 fn i32_shr(
3202 &mut self,
3203 loc_a: Location,
3204 loc_b: Location,
3205 ret: Location,
3206 ) -> Result<(), CompileError> {
3207 self.emit_relaxed_binop3(
3208 Assembler::emit_srl,
3209 Size::S32,
3210 loc_a,
3211 loc_b,
3212 ret,
3213 ImmType::Shift32,
3214 )
3215 }
3216 fn i32_sar(
3217 &mut self,
3218 loc_a: Location,
3219 loc_b: Location,
3220 ret: Location,
3221 ) -> Result<(), CompileError> {
3222 self.emit_relaxed_binop3(
3223 Assembler::emit_sra,
3224 Size::S32,
3225 loc_a,
3226 loc_b,
3227 ret,
3228 ImmType::Shift32,
3229 )
3230 }
3231 fn i32_rol(
3232 &mut self,
3233 loc_a: Location,
3234 loc_b: Location,
3235 ret: Location,
3236 ) -> Result<(), CompileError> {
3237 self.emit_rol(Size::S32, loc_a, loc_b, ret, ImmType::Shift32)
3238 }
3239 fn i32_ror(
3240 &mut self,
3241 loc_a: Location,
3242 loc_b: Location,
3243 ret: Location,
3244 ) -> Result<(), CompileError> {
3245 self.emit_ror(Size::S32, loc_a, loc_b, ret, ImmType::Shift32)
3246 }
3247 fn i32_load(
3248 &mut self,
3249 addr: Location,
3250 memarg: &MemArg,
3251 ret: Location,
3252 _need_check: bool,
3253 imported_memories: bool,
3254 offset: i32,
3255 heap_access_oob: Label,
3256 unaligned_atomic: Label,
3257 ) -> Result<(), CompileError> {
3258 self.memory_op(
3259 addr,
3260 memarg,
3261 false,
3262 4,
3263 imported_memories,
3264 offset,
3265 heap_access_oob,
3266 unaligned_atomic,
3267 |this, addr| this.emit_maybe_unaligned_load(Size::S32, true, ret, addr),
3268 )
3269 }
3270 fn i32_load_8u(
3271 &mut self,
3272 addr: Location,
3273 memarg: &MemArg,
3274 ret: Location,
3275 _need_check: bool,
3276 imported_memories: bool,
3277 offset: i32,
3278 heap_access_oob: Label,
3279 unaligned_atomic: Label,
3280 ) -> Result<(), CompileError> {
3281 self.memory_op(
3282 addr,
3283 memarg,
3284 false,
3285 1,
3286 imported_memories,
3287 offset,
3288 heap_access_oob,
3289 unaligned_atomic,
3290 |this, addr| this.emit_maybe_unaligned_load(Size::S8, false, ret, addr),
3291 )
3292 }
3293 fn i32_load_8s(
3294 &mut self,
3295 addr: Location,
3296 memarg: &MemArg,
3297 ret: Location,
3298 _need_check: bool,
3299 imported_memories: bool,
3300 offset: i32,
3301 heap_access_oob: Label,
3302 unaligned_atomic: Label,
3303 ) -> Result<(), CompileError> {
3304 self.memory_op(
3305 addr,
3306 memarg,
3307 false,
3308 1,
3309 imported_memories,
3310 offset,
3311 heap_access_oob,
3312 unaligned_atomic,
3313 |this, addr| this.emit_maybe_unaligned_load(Size::S8, true, ret, addr),
3314 )
3315 }
3316 fn i32_load_16u(
3317 &mut self,
3318 addr: Location,
3319 memarg: &MemArg,
3320 ret: Location,
3321 _need_check: bool,
3322 imported_memories: bool,
3323 offset: i32,
3324 heap_access_oob: Label,
3325 unaligned_atomic: Label,
3326 ) -> Result<(), CompileError> {
3327 self.memory_op(
3328 addr,
3329 memarg,
3330 false,
3331 2,
3332 imported_memories,
3333 offset,
3334 heap_access_oob,
3335 unaligned_atomic,
3336 |this, addr| this.emit_maybe_unaligned_load(Size::S16, false, ret, addr),
3337 )
3338 }
3339 fn i32_load_16s(
3340 &mut self,
3341 addr: Location,
3342 memarg: &MemArg,
3343 ret: Location,
3344 _need_check: bool,
3345 imported_memories: bool,
3346 offset: i32,
3347 heap_access_oob: Label,
3348 unaligned_atomic: Label,
3349 ) -> Result<(), CompileError> {
3350 self.memory_op(
3351 addr,
3352 memarg,
3353 false,
3354 2,
3355 imported_memories,
3356 offset,
3357 heap_access_oob,
3358 unaligned_atomic,
3359 |this, addr| this.emit_maybe_unaligned_load(Size::S16, true, ret, addr),
3360 )
3361 }
3362 fn i32_atomic_load(
3363 &mut self,
3364 addr: Location,
3365 memarg: &MemArg,
3366 ret: Location,
3367 _need_check: bool,
3368 imported_memories: bool,
3369 offset: i32,
3370 heap_access_oob: Label,
3371 unaligned_atomic: Label,
3372 ) -> Result<(), CompileError> {
3373 self.memory_op(
3374 addr,
3375 memarg,
3376 true,
3377 4,
3378 imported_memories,
3379 offset,
3380 heap_access_oob,
3381 unaligned_atomic,
3382 |this, addr| this.emit_relaxed_load(Size::S32, true, ret, Location::Memory(addr, 0)),
3383 )
3384 }
3385 fn i32_atomic_load_8u(
3386 &mut self,
3387 addr: Location,
3388 memarg: &MemArg,
3389 ret: Location,
3390 _need_check: bool,
3391 imported_memories: bool,
3392 offset: i32,
3393 heap_access_oob: Label,
3394 unaligned_atomic: Label,
3395 ) -> Result<(), CompileError> {
3396 self.memory_op(
3397 addr,
3398 memarg,
3399 true,
3400 1,
3401 imported_memories,
3402 offset,
3403 heap_access_oob,
3404 unaligned_atomic,
3405 |this, addr| this.emit_relaxed_load(Size::S8, true, ret, Location::Memory(addr, 0)),
3406 )
3407 }
3408 fn i32_atomic_load_16u(
3409 &mut self,
3410 addr: Location,
3411 memarg: &MemArg,
3412 ret: Location,
3413 _need_check: bool,
3414 imported_memories: bool,
3415 offset: i32,
3416 heap_access_oob: Label,
3417 unaligned_atomic: Label,
3418 ) -> Result<(), CompileError> {
3419 self.memory_op(
3420 addr,
3421 memarg,
3422 true,
3423 2,
3424 imported_memories,
3425 offset,
3426 heap_access_oob,
3427 unaligned_atomic,
3428 |this, addr| this.emit_relaxed_load(Size::S16, true, ret, Location::Memory(addr, 0)),
3429 )
3430 }
3431 fn i32_save(
3432 &mut self,
3433 value: Location,
3434 memarg: &MemArg,
3435 addr: Location,
3436 _need_check: bool,
3437 imported_memories: bool,
3438 offset: i32,
3439 heap_access_oob: Label,
3440 unaligned_atomic: Label,
3441 ) -> Result<(), CompileError> {
3442 self.memory_op(
3443 addr,
3444 memarg,
3445 false,
3446 4,
3447 imported_memories,
3448 offset,
3449 heap_access_oob,
3450 unaligned_atomic,
3451 |this, addr| this.emit_maybe_unaligned_store(Size::S32, value, addr),
3452 )
3453 }
3454 fn i32_save_8(
3455 &mut self,
3456 value: Location,
3457 memarg: &MemArg,
3458 addr: Location,
3459 _need_check: bool,
3460 imported_memories: bool,
3461 offset: i32,
3462 heap_access_oob: Label,
3463 unaligned_atomic: Label,
3464 ) -> Result<(), CompileError> {
3465 self.memory_op(
3466 addr,
3467 memarg,
3468 false,
3469 1,
3470 imported_memories,
3471 offset,
3472 heap_access_oob,
3473 unaligned_atomic,
3474 |this, addr| this.emit_maybe_unaligned_store(Size::S8, value, addr),
3475 )
3476 }
3477 fn i32_save_16(
3478 &mut self,
3479 value: Location,
3480 memarg: &MemArg,
3481 addr: Location,
3482 _need_check: bool,
3483 imported_memories: bool,
3484 offset: i32,
3485 heap_access_oob: Label,
3486 unaligned_atomic: Label,
3487 ) -> Result<(), CompileError> {
3488 self.memory_op(
3489 addr,
3490 memarg,
3491 false,
3492 2,
3493 imported_memories,
3494 offset,
3495 heap_access_oob,
3496 unaligned_atomic,
3497 |this, addr| this.emit_maybe_unaligned_store(Size::S16, value, addr),
3498 )
3499 }
3500 fn i32_atomic_save(
3501 &mut self,
3502 value: Location,
3503 memarg: &MemArg,
3504 addr: Location,
3505 _need_check: bool,
3506 imported_memories: bool,
3507 offset: i32,
3508 heap_access_oob: Label,
3509 unaligned_atomic: Label,
3510 ) -> Result<(), CompileError> {
3511 self.memory_op(
3512 addr,
3513 memarg,
3514 true,
3515 4,
3516 imported_memories,
3517 offset,
3518 heap_access_oob,
3519 unaligned_atomic,
3520 |this, addr| this.emit_relaxed_store(Size::S32, value, Location::Memory(addr, 0)),
3521 )?;
3522 self.assembler.emit_rwfence()
3523 }
3524 fn i32_atomic_save_8(
3525 &mut self,
3526 value: Location,
3527 memarg: &MemArg,
3528 addr: Location,
3529 _need_check: bool,
3530 imported_memories: bool,
3531 offset: i32,
3532 heap_access_oob: Label,
3533 unaligned_atomic: Label,
3534 ) -> Result<(), CompileError> {
3535 self.memory_op(
3536 addr,
3537 memarg,
3538 true,
3539 1,
3540 imported_memories,
3541 offset,
3542 heap_access_oob,
3543 unaligned_atomic,
3544 |this, addr| this.emit_relaxed_store(Size::S8, value, Location::Memory(addr, 0)),
3545 )?;
3546 self.assembler.emit_rwfence()
3547 }
3548 fn i32_atomic_save_16(
3549 &mut self,
3550 value: Location,
3551 memarg: &MemArg,
3552 addr: Location,
3553 _need_check: bool,
3554 imported_memories: bool,
3555 offset: i32,
3556 heap_access_oob: Label,
3557 unaligned_atomic: Label,
3558 ) -> Result<(), CompileError> {
3559 self.memory_op(
3560 addr,
3561 memarg,
3562 true,
3563 2,
3564 imported_memories,
3565 offset,
3566 heap_access_oob,
3567 unaligned_atomic,
3568 |this, addr| this.emit_relaxed_store(Size::S16, value, Location::Memory(addr, 0)),
3569 )?;
3570 self.assembler.emit_rwfence()
3571 }
3572 fn i32_atomic_add(
3573 &mut self,
3574 loc: Location,
3575 target: Location,
3576 memarg: &MemArg,
3577 ret: Location,
3578 _need_check: bool,
3579 imported_memories: bool,
3580 offset: i32,
3581 heap_access_oob: Label,
3582 unaligned_atomic: Label,
3583 ) -> Result<(), CompileError> {
3584 self.memory_op(
3585 target,
3586 memarg,
3587 true,
3588 4,
3589 imported_memories,
3590 offset,
3591 heap_access_oob,
3592 unaligned_atomic,
3593 |this, addr| {
3594 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Add, Size::S32, ret, addr, loc)
3595 },
3596 )
3597 }
3598 fn i32_atomic_add_8u(
3599 &mut self,
3600 loc: Location,
3601 target: Location,
3602 memarg: &MemArg,
3603 ret: Location,
3604 _need_check: bool,
3605 imported_memories: bool,
3606 offset: i32,
3607 heap_access_oob: Label,
3608 unaligned_atomic: Label,
3609 ) -> Result<(), CompileError> {
3610 self.memory_op(
3611 target,
3612 memarg,
3613 true,
3614 1,
3615 imported_memories,
3616 offset,
3617 heap_access_oob,
3618 unaligned_atomic,
3619 |this, addr| {
3620 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Add, Size::S8, ret, addr, loc)
3621 },
3622 )
3623 }
3624 fn i32_atomic_add_16u(
3625 &mut self,
3626 loc: Location,
3627 target: Location,
3628 memarg: &MemArg,
3629 ret: Location,
3630 _need_check: bool,
3631 imported_memories: bool,
3632 offset: i32,
3633 heap_access_oob: Label,
3634 unaligned_atomic: Label,
3635 ) -> Result<(), CompileError> {
3636 self.memory_op(
3637 target,
3638 memarg,
3639 true,
3640 2,
3641 imported_memories,
3642 offset,
3643 heap_access_oob,
3644 unaligned_atomic,
3645 |this, addr| {
3646 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Add, Size::S16, ret, addr, loc)
3647 },
3648 )
3649 }
3650 fn i32_atomic_sub(
3651 &mut self,
3652 loc: Location,
3653 target: Location,
3654 memarg: &MemArg,
3655 ret: Location,
3656 _need_check: bool,
3657 imported_memories: bool,
3658 offset: i32,
3659 heap_access_oob: Label,
3660 unaligned_atomic: Label,
3661 ) -> Result<(), CompileError> {
3662 self.memory_op(
3663 target,
3664 memarg,
3665 true,
3666 4,
3667 imported_memories,
3668 offset,
3669 heap_access_oob,
3670 unaligned_atomic,
3671 |this, addr| {
3672 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Sub, Size::S32, ret, addr, loc)
3673 },
3674 )
3675 }
3676 fn i32_atomic_sub_8u(
3677 &mut self,
3678 loc: Location,
3679 target: Location,
3680 memarg: &MemArg,
3681 ret: Location,
3682 _need_check: bool,
3683 imported_memories: bool,
3684 offset: i32,
3685 heap_access_oob: Label,
3686 unaligned_atomic: Label,
3687 ) -> Result<(), CompileError> {
3688 self.memory_op(
3689 target,
3690 memarg,
3691 true,
3692 1,
3693 imported_memories,
3694 offset,
3695 heap_access_oob,
3696 unaligned_atomic,
3697 |this, addr| {
3698 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Sub, Size::S8, ret, addr, loc)
3699 },
3700 )
3701 }
3702 fn i32_atomic_sub_16u(
3703 &mut self,
3704 loc: Location,
3705 target: Location,
3706 memarg: &MemArg,
3707 ret: Location,
3708 _need_check: bool,
3709 imported_memories: bool,
3710 offset: i32,
3711 heap_access_oob: Label,
3712 unaligned_atomic: Label,
3713 ) -> Result<(), CompileError> {
3714 self.memory_op(
3715 target,
3716 memarg,
3717 true,
3718 2,
3719 imported_memories,
3720 offset,
3721 heap_access_oob,
3722 unaligned_atomic,
3723 |this, addr| {
3724 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Sub, Size::S16, ret, addr, loc)
3725 },
3726 )
3727 }
3728 fn i32_atomic_and(
3729 &mut self,
3730 loc: Location,
3731 target: Location,
3732 memarg: &MemArg,
3733 ret: Location,
3734 _need_check: bool,
3735 imported_memories: bool,
3736 offset: i32,
3737 heap_access_oob: Label,
3738 unaligned_atomic: Label,
3739 ) -> Result<(), CompileError> {
3740 self.memory_op(
3741 target,
3742 memarg,
3743 true,
3744 4,
3745 imported_memories,
3746 offset,
3747 heap_access_oob,
3748 unaligned_atomic,
3749 |this, addr| {
3750 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::And, Size::S32, ret, addr, loc)
3751 },
3752 )
3753 }
3754 fn i32_atomic_and_8u(
3755 &mut self,
3756 loc: Location,
3757 target: Location,
3758 memarg: &MemArg,
3759 ret: Location,
3760 _need_check: bool,
3761 imported_memories: bool,
3762 offset: i32,
3763 heap_access_oob: Label,
3764 unaligned_atomic: Label,
3765 ) -> Result<(), CompileError> {
3766 self.memory_op(
3767 target,
3768 memarg,
3769 true,
3770 1,
3771 imported_memories,
3772 offset,
3773 heap_access_oob,
3774 unaligned_atomic,
3775 |this, addr| {
3776 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::And, Size::S8, ret, addr, loc)
3777 },
3778 )
3779 }
3780 fn i32_atomic_and_16u(
3781 &mut self,
3782 loc: Location,
3783 target: Location,
3784 memarg: &MemArg,
3785 ret: Location,
3786 _need_check: bool,
3787 imported_memories: bool,
3788 offset: i32,
3789 heap_access_oob: Label,
3790 unaligned_atomic: Label,
3791 ) -> Result<(), CompileError> {
3792 self.memory_op(
3793 target,
3794 memarg,
3795 true,
3796 2,
3797 imported_memories,
3798 offset,
3799 heap_access_oob,
3800 unaligned_atomic,
3801 |this, addr| {
3802 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::And, Size::S16, ret, addr, loc)
3803 },
3804 )
3805 }
3806 fn i32_atomic_or(
3807 &mut self,
3808 loc: Location,
3809 target: Location,
3810 memarg: &MemArg,
3811 ret: Location,
3812 _need_check: bool,
3813 imported_memories: bool,
3814 offset: i32,
3815 heap_access_oob: Label,
3816 unaligned_atomic: Label,
3817 ) -> Result<(), CompileError> {
3818 self.memory_op(
3819 target,
3820 memarg,
3821 true,
3822 4,
3823 imported_memories,
3824 offset,
3825 heap_access_oob,
3826 unaligned_atomic,
3827 |this, addr| {
3828 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Or, Size::S32, ret, addr, loc)
3829 },
3830 )
3831 }
3832 fn i32_atomic_or_8u(
3833 &mut self,
3834 loc: Location,
3835 target: Location,
3836 memarg: &MemArg,
3837 ret: Location,
3838 _need_check: bool,
3839 imported_memories: bool,
3840 offset: i32,
3841 heap_access_oob: Label,
3842 unaligned_atomic: Label,
3843 ) -> Result<(), CompileError> {
3844 self.memory_op(
3845 target,
3846 memarg,
3847 true,
3848 1,
3849 imported_memories,
3850 offset,
3851 heap_access_oob,
3852 unaligned_atomic,
3853 |this, addr| {
3854 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Or, Size::S8, ret, addr, loc)
3855 },
3856 )
3857 }
3858 fn i32_atomic_or_16u(
3859 &mut self,
3860 loc: Location,
3861 target: Location,
3862 memarg: &MemArg,
3863 ret: Location,
3864 _need_check: bool,
3865 imported_memories: bool,
3866 offset: i32,
3867 heap_access_oob: Label,
3868 unaligned_atomic: Label,
3869 ) -> Result<(), CompileError> {
3870 self.memory_op(
3871 target,
3872 memarg,
3873 true,
3874 2,
3875 imported_memories,
3876 offset,
3877 heap_access_oob,
3878 unaligned_atomic,
3879 |this, addr| {
3880 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Or, Size::S16, ret, addr, loc)
3881 },
3882 )
3883 }
3884 fn i32_atomic_xor(
3885 &mut self,
3886 loc: Location,
3887 target: Location,
3888 memarg: &MemArg,
3889 ret: Location,
3890 _need_check: bool,
3891 imported_memories: bool,
3892 offset: i32,
3893 heap_access_oob: Label,
3894 unaligned_atomic: Label,
3895 ) -> Result<(), CompileError> {
3896 self.memory_op(
3897 target,
3898 memarg,
3899 true,
3900 4,
3901 imported_memories,
3902 offset,
3903 heap_access_oob,
3904 unaligned_atomic,
3905 |this, addr| {
3906 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Xor, Size::S32, ret, addr, loc)
3907 },
3908 )
3909 }
3910 fn i32_atomic_xor_8u(
3911 &mut self,
3912 loc: Location,
3913 target: Location,
3914 memarg: &MemArg,
3915 ret: Location,
3916 _need_check: bool,
3917 imported_memories: bool,
3918 offset: i32,
3919 heap_access_oob: Label,
3920 unaligned_atomic: Label,
3921 ) -> Result<(), CompileError> {
3922 self.memory_op(
3923 target,
3924 memarg,
3925 true,
3926 1,
3927 imported_memories,
3928 offset,
3929 heap_access_oob,
3930 unaligned_atomic,
3931 |this, addr| {
3932 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Xor, Size::S8, ret, addr, loc)
3933 },
3934 )
3935 }
3936 fn i32_atomic_xor_16u(
3937 &mut self,
3938 loc: Location,
3939 target: Location,
3940 memarg: &MemArg,
3941 ret: Location,
3942 _need_check: bool,
3943 imported_memories: bool,
3944 offset: i32,
3945 heap_access_oob: Label,
3946 unaligned_atomic: Label,
3947 ) -> Result<(), CompileError> {
3948 self.memory_op(
3949 target,
3950 memarg,
3951 true,
3952 2,
3953 imported_memories,
3954 offset,
3955 heap_access_oob,
3956 unaligned_atomic,
3957 |this, addr| {
3958 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Xor, Size::S16, ret, addr, loc)
3959 },
3960 )
3961 }
3962 fn i32_atomic_xchg(
3963 &mut self,
3964 loc: Location,
3965 target: Location,
3966 memarg: &MemArg,
3967 ret: Location,
3968 _need_check: bool,
3969 imported_memories: bool,
3970 offset: i32,
3971 heap_access_oob: Label,
3972 unaligned_atomic: Label,
3973 ) -> Result<(), CompileError> {
3974 self.memory_op(
3975 target,
3976 memarg,
3977 true,
3978 4,
3979 imported_memories,
3980 offset,
3981 heap_access_oob,
3982 unaligned_atomic,
3983 |this, addr| {
3984 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Exchange, Size::S32, ret, addr, loc)
3985 },
3986 )
3987 }
3988 fn i32_atomic_xchg_8u(
3989 &mut self,
3990 loc: Location,
3991 target: Location,
3992 memarg: &MemArg,
3993 ret: Location,
3994 _need_check: bool,
3995 imported_memories: bool,
3996 offset: i32,
3997 heap_access_oob: Label,
3998 unaligned_atomic: Label,
3999 ) -> Result<(), CompileError> {
4000 self.memory_op(
4001 target,
4002 memarg,
4003 true,
4004 1,
4005 imported_memories,
4006 offset,
4007 heap_access_oob,
4008 unaligned_atomic,
4009 |this, addr| {
4010 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Exchange, Size::S8, ret, addr, loc)
4011 },
4012 )
4013 }
4014 fn i32_atomic_xchg_16u(
4015 &mut self,
4016 loc: Location,
4017 target: Location,
4018 memarg: &MemArg,
4019 ret: Location,
4020 _need_check: bool,
4021 imported_memories: bool,
4022 offset: i32,
4023 heap_access_oob: Label,
4024 unaligned_atomic: Label,
4025 ) -> Result<(), CompileError> {
4026 self.memory_op(
4027 target,
4028 memarg,
4029 true,
4030 2,
4031 imported_memories,
4032 offset,
4033 heap_access_oob,
4034 unaligned_atomic,
4035 |this, addr| {
4036 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Exchange, Size::S16, ret, addr, loc)
4037 },
4038 )
4039 }
4040 fn i32_atomic_cmpxchg(
4041 &mut self,
4042 new: Location,
4043 cmp: Location,
4044 target: Location,
4045 memarg: &MemArg,
4046 ret: Location,
4047 _need_check: bool,
4048 imported_memories: bool,
4049 offset: i32,
4050 heap_access_oob: Label,
4051 unaligned_atomic: Label,
4052 ) -> Result<(), CompileError> {
4053 self.memory_op(
4054 target,
4055 memarg,
4056 true,
4057 4,
4058 imported_memories,
4059 offset,
4060 heap_access_oob,
4061 unaligned_atomic,
4062 |this, addr| this.emit_relaxed_atomic_cmpxchg(Size::S32, ret, addr, new, cmp),
4063 )
4064 }
4065 fn i32_atomic_cmpxchg_8u(
4066 &mut self,
4067 new: Location,
4068 cmp: Location,
4069 target: Location,
4070 memarg: &MemArg,
4071 ret: Location,
4072 _need_check: bool,
4073 imported_memories: bool,
4074 offset: i32,
4075 heap_access_oob: Label,
4076 unaligned_atomic: Label,
4077 ) -> Result<(), CompileError> {
4078 self.memory_op(
4079 target,
4080 memarg,
4081 true,
4082 1,
4083 imported_memories,
4084 offset,
4085 heap_access_oob,
4086 unaligned_atomic,
4087 |this, addr| this.emit_relaxed_atomic_cmpxchg(Size::S8, ret, addr, new, cmp),
4088 )
4089 }
4090 fn i32_atomic_cmpxchg_16u(
4091 &mut self,
4092 new: Location,
4093 cmp: Location,
4094 target: Location,
4095 memarg: &MemArg,
4096 ret: Location,
4097 _need_check: bool,
4098 imported_memories: bool,
4099 offset: i32,
4100 heap_access_oob: Label,
4101 unaligned_atomic: Label,
4102 ) -> Result<(), CompileError> {
4103 self.memory_op(
4104 target,
4105 memarg,
4106 true,
4107 2,
4108 imported_memories,
4109 offset,
4110 heap_access_oob,
4111 unaligned_atomic,
4112 |this, addr| this.emit_relaxed_atomic_cmpxchg(Size::S16, ret, addr, new, cmp),
4113 )
4114 }
4115 fn emit_call_with_reloc(
4116 &mut self,
4117 _calling_convention: CallingConvention,
4118 reloc_target: RelocationTarget,
4119 ) -> Result<Vec<Relocation>, CompileError> {
4120 let mut relocations = vec![];
4121 let next = self.get_label();
4122 let reloc_at = self.assembler.get_offset().0;
4123 self.emit_label(next)?; self.assembler.emit_call_label(next)?;
4125 relocations.push(Relocation {
4126 kind: RelocationKind::RiscvCall,
4127 reloc_target,
4128 offset: reloc_at as u32,
4129 addend: 0,
4130 });
4131 Ok(relocations)
4132 }
4133 fn emit_binop_add64(
4134 &mut self,
4135 loc_a: Location,
4136 loc_b: Location,
4137 ret: Location,
4138 ) -> Result<(), CompileError> {
4139 self.emit_relaxed_binop3(
4140 Assembler::emit_add,
4141 Size::S64,
4142 loc_a,
4143 loc_b,
4144 ret,
4145 ImmType::Bits12,
4146 )
4147 }
4148 fn emit_binop_sub64(
4149 &mut self,
4150 loc_a: Location,
4151 loc_b: Location,
4152 ret: Location,
4153 ) -> Result<(), CompileError> {
4154 self.emit_relaxed_binop3(
4155 Assembler::emit_sub,
4156 Size::S64,
4157 loc_a,
4158 loc_b,
4159 ret,
4160 ImmType::Bits12Subtraction,
4161 )
4162 }
4163 fn emit_binop_mul64(
4164 &mut self,
4165 loc_a: Location,
4166 loc_b: Location,
4167 ret: Location,
4168 ) -> Result<(), CompileError> {
4169 self.emit_relaxed_binop3(
4170 Assembler::emit_mul,
4171 Size::S64,
4172 loc_a,
4173 loc_b,
4174 ret,
4175 ImmType::None,
4176 )
4177 }
4178 fn emit_binop_udiv64(
4179 &mut self,
4180 loc_a: Location,
4181 loc_b: Location,
4182 ret: Location,
4183 integer_division_by_zero: Label,
4184 ) -> Result<usize, CompileError> {
4185 let mut temps = vec![];
4186 let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4187 let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4188 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4189
4190 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
4191 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4192 })?;
4193 temps.push(jmp_tmp);
4194
4195 self.assembler
4196 .emit_on_false_label_far(src2, integer_division_by_zero, jmp_tmp)?;
4197 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
4198 self.assembler.emit_udiv(Size::S64, src1, src2, dest)?;
4199 if ret != dest {
4200 self.move_location(Size::S64, dest, ret)?;
4201 }
4202 for r in temps {
4203 self.release_gpr(r);
4204 }
4205 Ok(offset)
4206 }
4207 fn emit_binop_sdiv64(
4208 &mut self,
4209 loc_a: Location,
4210 loc_b: Location,
4211 ret: Location,
4212 integer_division_by_zero: Label,
4213 integer_overflow: Label,
4214 ) -> Result<usize, CompileError> {
4215 let mut temps = vec![];
4216 let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4217 let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4218 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4219
4220 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
4221 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4222 })?;
4223 temps.push(jmp_tmp);
4224
4225 self.assembler
4226 .emit_on_false_label_far(src2, integer_division_by_zero, jmp_tmp)?;
4227 let label_nooverflow = self.assembler.get_label();
4228 let tmp = self.location_to_reg(
4229 Size::S64,
4230 Location::Imm64(i64::MIN as u64),
4231 &mut temps,
4232 ImmType::None,
4233 true,
4234 None,
4235 )?;
4236
4237 self.assembler.emit_cmp(Condition::Ne, tmp, src1, tmp)?;
4238 self.assembler
4239 .emit_on_true_label(tmp, label_nooverflow, jmp_tmp)?;
4240 self.move_location(Size::S64, Location::Imm64(-1i64 as _), tmp)?;
4241 self.assembler.emit_cmp(Condition::Eq, tmp, src2, tmp)?;
4242 self.assembler
4243 .emit_on_true_label_far(tmp, integer_overflow, jmp_tmp)?;
4244 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
4245 self.assembler.emit_label(label_nooverflow)?;
4246 self.assembler.emit_sdiv(Size::S64, src1, src2, dest)?;
4247 if ret != dest {
4248 self.move_location(Size::S64, dest, ret)?;
4249 }
4250 for r in temps {
4251 self.release_gpr(r);
4252 }
4253 Ok(offset)
4254 }
4255 fn emit_binop_urem64(
4256 &mut self,
4257 loc_a: Location,
4258 loc_b: Location,
4259 ret: Location,
4260 integer_division_by_zero: Label,
4261 ) -> Result<usize, CompileError> {
4262 let mut temps = vec![];
4263 let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4264 let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4265 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4266
4267 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
4268 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4269 })?;
4270 temps.push(jmp_tmp);
4271
4272 self.assembler
4273 .emit_on_false_label_far(src2, integer_division_by_zero, jmp_tmp)?;
4274 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
4275 self.assembler.emit_urem(Size::S64, src1, src2, dest)?;
4276 if ret != dest {
4277 self.move_location(Size::S64, dest, ret)?;
4278 }
4279 for r in temps {
4280 self.release_gpr(r);
4281 }
4282 Ok(offset)
4283 }
4284 fn emit_binop_srem64(
4285 &mut self,
4286 loc_a: Location,
4287 loc_b: Location,
4288 ret: Location,
4289 integer_division_by_zero: Label,
4290 ) -> Result<usize, CompileError> {
4291 let mut temps = vec![];
4292 let src1 = self.location_to_reg(Size::S64, loc_a, &mut temps, ImmType::None, true, None)?;
4293 let src2 = self.location_to_reg(Size::S64, loc_b, &mut temps, ImmType::None, true, None)?;
4294 let dest = self.location_to_reg(Size::S64, ret, &mut temps, ImmType::None, false, None)?;
4295
4296 let jmp_tmp = self.acquire_temp_gpr().ok_or_else(|| {
4297 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4298 })?;
4299 temps.push(jmp_tmp);
4300
4301 self.assembler
4302 .emit_on_false_label_far(src2, integer_division_by_zero, jmp_tmp)?;
4303 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
4304 self.assembler.emit_srem(Size::S64, src1, src2, dest)?;
4305 if ret != dest {
4306 self.move_location(Size::S64, dest, ret)?;
4307 }
4308 for r in temps {
4309 self.release_gpr(r);
4310 }
4311 Ok(offset)
4312 }
4313 fn emit_binop_and64(
4314 &mut self,
4315 loc_a: Location,
4316 loc_b: Location,
4317 ret: Location,
4318 ) -> Result<(), CompileError> {
4319 self.emit_relaxed_binop3(
4320 Assembler::emit_and,
4321 Size::S64,
4322 loc_a,
4323 loc_b,
4324 ret,
4325 ImmType::Bits12,
4326 )
4327 }
4328 fn emit_binop_or64(
4329 &mut self,
4330 loc_a: Location,
4331 loc_b: Location,
4332 ret: Location,
4333 ) -> Result<(), CompileError> {
4334 self.emit_relaxed_binop3(
4335 Assembler::emit_or,
4336 Size::S64,
4337 loc_a,
4338 loc_b,
4339 ret,
4340 ImmType::Bits12,
4341 )
4342 }
4343 fn emit_binop_xor64(
4344 &mut self,
4345 loc_a: Location,
4346 loc_b: Location,
4347 ret: Location,
4348 ) -> Result<(), CompileError> {
4349 self.emit_relaxed_binop3(
4350 Assembler::emit_xor,
4351 Size::S64,
4352 loc_a,
4353 loc_b,
4354 ret,
4355 ImmType::Bits12,
4356 )
4357 }
4358 fn i64_cmp_ge_s(
4359 &mut self,
4360 loc_a: Location,
4361 loc_b: Location,
4362 ret: Location,
4363 ) -> Result<(), CompileError> {
4364 self.emit_cmpop_i64_dynamic_b(Condition::Ge, loc_a, loc_b, ret)
4365 }
4366 fn i64_cmp_gt_s(
4367 &mut self,
4368 loc_a: Location,
4369 loc_b: Location,
4370 ret: Location,
4371 ) -> Result<(), CompileError> {
4372 self.emit_cmpop_i64_dynamic_b(Condition::Gt, loc_a, loc_b, ret)
4373 }
4374 fn i64_cmp_le_s(
4375 &mut self,
4376 loc_a: Location,
4377 loc_b: Location,
4378 ret: Location,
4379 ) -> Result<(), CompileError> {
4380 self.emit_cmpop_i64_dynamic_b(Condition::Le, loc_a, loc_b, ret)
4381 }
4382 fn i64_cmp_lt_s(
4383 &mut self,
4384 loc_a: Location,
4385 loc_b: Location,
4386 ret: Location,
4387 ) -> Result<(), CompileError> {
4388 self.emit_cmpop_i64_dynamic_b(Condition::Lt, loc_a, loc_b, ret)
4389 }
4390 fn i64_cmp_ge_u(
4391 &mut self,
4392 loc_a: Location,
4393 loc_b: Location,
4394 ret: Location,
4395 ) -> Result<(), CompileError> {
4396 self.emit_cmpop_i64_dynamic_b(Condition::Geu, loc_a, loc_b, ret)
4397 }
4398 fn i64_cmp_gt_u(
4399 &mut self,
4400 loc_a: Location,
4401 loc_b: Location,
4402 ret: Location,
4403 ) -> Result<(), CompileError> {
4404 self.emit_cmpop_i64_dynamic_b(Condition::Gtu, loc_a, loc_b, ret)
4405 }
4406 fn i64_cmp_le_u(
4407 &mut self,
4408 loc_a: Location,
4409 loc_b: Location,
4410 ret: Location,
4411 ) -> Result<(), CompileError> {
4412 self.emit_cmpop_i64_dynamic_b(Condition::Leu, loc_a, loc_b, ret)
4413 }
4414 fn i64_cmp_lt_u(
4415 &mut self,
4416 loc_a: Location,
4417 loc_b: Location,
4418 ret: Location,
4419 ) -> Result<(), CompileError> {
4420 self.emit_cmpop_i64_dynamic_b(Condition::Ltu, loc_a, loc_b, ret)
4421 }
4422 fn i64_cmp_ne(
4423 &mut self,
4424 loc_a: Location,
4425 loc_b: Location,
4426 ret: Location,
4427 ) -> Result<(), CompileError> {
4428 self.emit_cmpop_i64_dynamic_b(Condition::Ne, loc_a, loc_b, ret)
4429 }
4430 fn i64_cmp_eq(
4431 &mut self,
4432 loc_a: Location,
4433 loc_b: Location,
4434 ret: Location,
4435 ) -> Result<(), CompileError> {
4436 self.emit_cmpop_i64_dynamic_b(Condition::Eq, loc_a, loc_b, ret)
4437 }
4438 fn i64_clz(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
4439 self.emit_clz(Size::S64, loc, ret)
4440 }
4441 fn i64_ctz(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
4442 self.emit_ctz(Size::S64, loc, ret)
4443 }
4444 fn i64_popcnt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
4445 self.emit_popcnt(Size::S64, loc, ret)
4446 }
4447 fn i64_shl(
4448 &mut self,
4449 loc_a: Location,
4450 loc_b: Location,
4451 ret: Location,
4452 ) -> Result<(), CompileError> {
4453 self.emit_relaxed_binop3(
4454 Assembler::emit_sll,
4455 Size::S64,
4456 loc_a,
4457 loc_b,
4458 ret,
4459 ImmType::Shift64,
4460 )
4461 }
4462 fn i64_shr(
4463 &mut self,
4464 loc_a: Location,
4465 loc_b: Location,
4466 ret: Location,
4467 ) -> Result<(), CompileError> {
4468 self.emit_relaxed_binop3(
4469 Assembler::emit_srl,
4470 Size::S64,
4471 loc_a,
4472 loc_b,
4473 ret,
4474 ImmType::Shift64,
4475 )
4476 }
4477 fn i64_sar(
4478 &mut self,
4479 loc_a: Location,
4480 loc_b: Location,
4481 ret: Location,
4482 ) -> Result<(), CompileError> {
4483 self.emit_relaxed_binop3(
4484 Assembler::emit_sra,
4485 Size::S64,
4486 loc_a,
4487 loc_b,
4488 ret,
4489 ImmType::Shift64,
4490 )
4491 }
4492 fn i64_rol(
4493 &mut self,
4494 loc_a: Location,
4495 loc_b: Location,
4496 ret: Location,
4497 ) -> Result<(), CompileError> {
4498 self.emit_rol(Size::S64, loc_a, loc_b, ret, ImmType::Shift64)
4499 }
4500 fn i64_ror(
4501 &mut self,
4502 loc_a: Location,
4503 loc_b: Location,
4504 ret: Location,
4505 ) -> Result<(), CompileError> {
4506 self.emit_ror(Size::S64, loc_a, loc_b, ret, ImmType::Shift64)
4507 }
4508 fn i64_load(
4509 &mut self,
4510 addr: Location,
4511 memarg: &MemArg,
4512 ret: Location,
4513 _need_check: bool,
4514 imported_memories: bool,
4515 offset: i32,
4516 heap_access_oob: Label,
4517 unaligned_atomic: Label,
4518 ) -> Result<(), CompileError> {
4519 self.memory_op(
4520 addr,
4521 memarg,
4522 false,
4523 8,
4524 imported_memories,
4525 offset,
4526 heap_access_oob,
4527 unaligned_atomic,
4528 |this, addr| this.emit_maybe_unaligned_load(Size::S64, true, ret, addr),
4529 )
4530 }
4531 fn i64_load_8u(
4532 &mut self,
4533 addr: Location,
4534 memarg: &MemArg,
4535 ret: Location,
4536 _need_check: bool,
4537 imported_memories: bool,
4538 offset: i32,
4539 heap_access_oob: Label,
4540 unaligned_atomic: Label,
4541 ) -> Result<(), CompileError> {
4542 self.memory_op(
4543 addr,
4544 memarg,
4545 false,
4546 1,
4547 imported_memories,
4548 offset,
4549 heap_access_oob,
4550 unaligned_atomic,
4551 |this, addr| this.emit_maybe_unaligned_load(Size::S8, false, ret, addr),
4552 )
4553 }
4554 fn i64_load_8s(
4555 &mut self,
4556 addr: Location,
4557 memarg: &MemArg,
4558 ret: Location,
4559 _need_check: bool,
4560 imported_memories: bool,
4561 offset: i32,
4562 heap_access_oob: Label,
4563 unaligned_atomic: Label,
4564 ) -> Result<(), CompileError> {
4565 self.memory_op(
4566 addr,
4567 memarg,
4568 false,
4569 1,
4570 imported_memories,
4571 offset,
4572 heap_access_oob,
4573 unaligned_atomic,
4574 |this, addr| this.emit_maybe_unaligned_load(Size::S8, true, ret, addr),
4575 )
4576 }
4577 fn i64_load_32u(
4578 &mut self,
4579 addr: Location,
4580 memarg: &MemArg,
4581 ret: Location,
4582 _need_check: bool,
4583 imported_memories: bool,
4584 offset: i32,
4585 heap_access_oob: Label,
4586 unaligned_atomic: Label,
4587 ) -> Result<(), CompileError> {
4588 self.memory_op(
4589 addr,
4590 memarg,
4591 false,
4592 4,
4593 imported_memories,
4594 offset,
4595 heap_access_oob,
4596 unaligned_atomic,
4597 |this, addr| this.emit_maybe_unaligned_load(Size::S32, false, ret, addr),
4598 )
4599 }
4600 fn i64_load_32s(
4601 &mut self,
4602 addr: Location,
4603 memarg: &MemArg,
4604 ret: Location,
4605 _need_check: bool,
4606 imported_memories: bool,
4607 offset: i32,
4608 heap_access_oob: Label,
4609 unaligned_atomic: Label,
4610 ) -> Result<(), CompileError> {
4611 self.memory_op(
4612 addr,
4613 memarg,
4614 false,
4615 4,
4616 imported_memories,
4617 offset,
4618 heap_access_oob,
4619 unaligned_atomic,
4620 |this, addr| this.emit_maybe_unaligned_load(Size::S32, true, ret, addr),
4621 )
4622 }
4623 fn i64_load_16u(
4624 &mut self,
4625 addr: Location,
4626 memarg: &MemArg,
4627 ret: Location,
4628 _need_check: bool,
4629 imported_memories: bool,
4630 offset: i32,
4631 heap_access_oob: Label,
4632 unaligned_atomic: Label,
4633 ) -> Result<(), CompileError> {
4634 self.memory_op(
4635 addr,
4636 memarg,
4637 false,
4638 2,
4639 imported_memories,
4640 offset,
4641 heap_access_oob,
4642 unaligned_atomic,
4643 |this, addr| this.emit_maybe_unaligned_load(Size::S16, false, ret, addr),
4644 )
4645 }
4646 fn i64_load_16s(
4647 &mut self,
4648 addr: Location,
4649 memarg: &MemArg,
4650 ret: Location,
4651 _need_check: bool,
4652 imported_memories: bool,
4653 offset: i32,
4654 heap_access_oob: Label,
4655 unaligned_atomic: Label,
4656 ) -> Result<(), CompileError> {
4657 self.memory_op(
4658 addr,
4659 memarg,
4660 false,
4661 2,
4662 imported_memories,
4663 offset,
4664 heap_access_oob,
4665 unaligned_atomic,
4666 |this, addr| this.emit_maybe_unaligned_load(Size::S16, true, ret, addr),
4667 )
4668 }
4669 fn i64_atomic_load(
4670 &mut self,
4671 addr: Location,
4672 memarg: &MemArg,
4673 ret: Location,
4674 _need_check: bool,
4675 imported_memories: bool,
4676 offset: i32,
4677 heap_access_oob: Label,
4678 unaligned_atomic: Label,
4679 ) -> Result<(), CompileError> {
4680 self.memory_op(
4681 addr,
4682 memarg,
4683 true,
4684 8,
4685 imported_memories,
4686 offset,
4687 heap_access_oob,
4688 unaligned_atomic,
4689 |this, addr| this.emit_relaxed_load(Size::S64, true, ret, Location::Memory(addr, 0)),
4690 )
4691 }
4692 fn i64_atomic_load_8u(
4693 &mut self,
4694 addr: Location,
4695 memarg: &MemArg,
4696 ret: Location,
4697 _need_check: bool,
4698 imported_memories: bool,
4699 offset: i32,
4700 heap_access_oob: Label,
4701 unaligned_atomic: Label,
4702 ) -> Result<(), CompileError> {
4703 self.memory_op(
4704 addr,
4705 memarg,
4706 true,
4707 1,
4708 imported_memories,
4709 offset,
4710 heap_access_oob,
4711 unaligned_atomic,
4712 |this, addr| this.emit_relaxed_load(Size::S8, false, ret, Location::Memory(addr, 0)),
4713 )
4714 }
4715 fn i64_atomic_load_16u(
4716 &mut self,
4717 addr: Location,
4718 memarg: &MemArg,
4719 ret: Location,
4720 _need_check: bool,
4721 imported_memories: bool,
4722 offset: i32,
4723 heap_access_oob: Label,
4724 unaligned_atomic: Label,
4725 ) -> Result<(), CompileError> {
4726 self.memory_op(
4727 addr,
4728 memarg,
4729 true,
4730 2,
4731 imported_memories,
4732 offset,
4733 heap_access_oob,
4734 unaligned_atomic,
4735 |this, addr| this.emit_relaxed_load(Size::S16, false, ret, Location::Memory(addr, 0)),
4736 )
4737 }
4738 fn i64_atomic_load_32u(
4739 &mut self,
4740 addr: Location,
4741 memarg: &MemArg,
4742 ret: Location,
4743 _need_check: bool,
4744 imported_memories: bool,
4745 offset: i32,
4746 heap_access_oob: Label,
4747 unaligned_atomic: Label,
4748 ) -> Result<(), CompileError> {
4749 self.memory_op(
4750 addr,
4751 memarg,
4752 true,
4753 4,
4754 imported_memories,
4755 offset,
4756 heap_access_oob,
4757 unaligned_atomic,
4758 |this, addr| this.emit_relaxed_load(Size::S32, false, ret, Location::Memory(addr, 0)),
4759 )
4760 }
4761 fn i64_save(
4762 &mut self,
4763 value: Location,
4764 memarg: &MemArg,
4765 addr: Location,
4766 _need_check: bool,
4767 imported_memories: bool,
4768 offset: i32,
4769 heap_access_oob: Label,
4770 unaligned_atomic: Label,
4771 ) -> Result<(), CompileError> {
4772 self.memory_op(
4773 addr,
4774 memarg,
4775 false,
4776 8,
4777 imported_memories,
4778 offset,
4779 heap_access_oob,
4780 unaligned_atomic,
4781 |this, addr| this.emit_maybe_unaligned_store(Size::S64, value, addr),
4782 )
4783 }
4784 fn i64_save_8(
4785 &mut self,
4786 value: Location,
4787 memarg: &MemArg,
4788 addr: Location,
4789 _need_check: bool,
4790 imported_memories: bool,
4791 offset: i32,
4792 heap_access_oob: Label,
4793 unaligned_atomic: Label,
4794 ) -> Result<(), CompileError> {
4795 self.memory_op(
4796 addr,
4797 memarg,
4798 false,
4799 1,
4800 imported_memories,
4801 offset,
4802 heap_access_oob,
4803 unaligned_atomic,
4804 |this, addr| this.emit_maybe_unaligned_store(Size::S8, value, addr),
4805 )
4806 }
4807 fn i64_save_16(
4808 &mut self,
4809 value: Location,
4810 memarg: &MemArg,
4811 addr: Location,
4812 _need_check: bool,
4813 imported_memories: bool,
4814 offset: i32,
4815 heap_access_oob: Label,
4816 unaligned_atomic: Label,
4817 ) -> Result<(), CompileError> {
4818 self.memory_op(
4819 addr,
4820 memarg,
4821 false,
4822 2,
4823 imported_memories,
4824 offset,
4825 heap_access_oob,
4826 unaligned_atomic,
4827 |this, addr| this.emit_maybe_unaligned_store(Size::S16, value, addr),
4828 )
4829 }
4830 fn i64_save_32(
4831 &mut self,
4832 value: Location,
4833 memarg: &MemArg,
4834 addr: Location,
4835 _need_check: bool,
4836 imported_memories: bool,
4837 offset: i32,
4838 heap_access_oob: Label,
4839 unaligned_atomic: Label,
4840 ) -> Result<(), CompileError> {
4841 self.memory_op(
4842 addr,
4843 memarg,
4844 false,
4845 4,
4846 imported_memories,
4847 offset,
4848 heap_access_oob,
4849 unaligned_atomic,
4850 |this, addr| this.emit_maybe_unaligned_store(Size::S32, value, addr),
4851 )
4852 }
4853 fn i64_atomic_save(
4854 &mut self,
4855 value: Location,
4856 memarg: &MemArg,
4857 addr: Location,
4858 _need_check: bool,
4859 imported_memories: bool,
4860 offset: i32,
4861 heap_access_oob: Label,
4862 unaligned_atomic: Label,
4863 ) -> Result<(), CompileError> {
4864 self.memory_op(
4865 addr,
4866 memarg,
4867 true,
4868 8,
4869 imported_memories,
4870 offset,
4871 heap_access_oob,
4872 unaligned_atomic,
4873 |this, addr| this.emit_relaxed_store(Size::S64, value, Location::Memory(addr, 0)),
4874 )?;
4875 self.assembler.emit_rwfence()
4876 }
4877 fn i64_atomic_save_8(
4878 &mut self,
4879 value: Location,
4880 memarg: &MemArg,
4881 addr: Location,
4882 _need_check: bool,
4883 imported_memories: bool,
4884 offset: i32,
4885 heap_access_oob: Label,
4886 unaligned_atomic: Label,
4887 ) -> Result<(), CompileError> {
4888 self.memory_op(
4889 addr,
4890 memarg,
4891 true,
4892 1,
4893 imported_memories,
4894 offset,
4895 heap_access_oob,
4896 unaligned_atomic,
4897 |this, addr| this.emit_relaxed_store(Size::S8, value, Location::Memory(addr, 0)),
4898 )?;
4899 self.assembler.emit_rwfence()
4900 }
4901 fn i64_atomic_save_16(
4902 &mut self,
4903 value: Location,
4904 memarg: &MemArg,
4905 addr: Location,
4906 _need_check: bool,
4907 imported_memories: bool,
4908 offset: i32,
4909 heap_access_oob: Label,
4910 unaligned_atomic: Label,
4911 ) -> Result<(), CompileError> {
4912 self.memory_op(
4913 addr,
4914 memarg,
4915 true,
4916 2,
4917 imported_memories,
4918 offset,
4919 heap_access_oob,
4920 unaligned_atomic,
4921 |this, addr| this.emit_relaxed_store(Size::S16, value, Location::Memory(addr, 0)),
4922 )?;
4923 self.assembler.emit_rwfence()
4924 }
4925 fn i64_atomic_save_32(
4926 &mut self,
4927 value: Location,
4928 memarg: &MemArg,
4929 addr: Location,
4930 _need_check: bool,
4931 imported_memories: bool,
4932 offset: i32,
4933 heap_access_oob: Label,
4934 unaligned_atomic: Label,
4935 ) -> Result<(), CompileError> {
4936 self.memory_op(
4937 addr,
4938 memarg,
4939 true,
4940 4,
4941 imported_memories,
4942 offset,
4943 heap_access_oob,
4944 unaligned_atomic,
4945 |this, addr| this.emit_relaxed_store(Size::S32, value, Location::Memory(addr, 0)),
4946 )?;
4947 self.assembler.emit_rwfence()
4948 }
4949 fn i64_atomic_add(
4950 &mut self,
4951 loc: Location,
4952 target: Location,
4953 memarg: &MemArg,
4954 ret: Location,
4955 _need_check: bool,
4956 imported_memories: bool,
4957 offset: i32,
4958 heap_access_oob: Label,
4959 unaligned_atomic: Label,
4960 ) -> Result<(), CompileError> {
4961 self.memory_op(
4962 target,
4963 memarg,
4964 true,
4965 8,
4966 imported_memories,
4967 offset,
4968 heap_access_oob,
4969 unaligned_atomic,
4970 |this, addr| {
4971 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Add, Size::S64, ret, addr, loc)
4972 },
4973 )
4974 }
4975 fn i64_atomic_add_8u(
4976 &mut self,
4977 loc: Location,
4978 target: Location,
4979 memarg: &MemArg,
4980 ret: Location,
4981 _need_check: bool,
4982 imported_memories: bool,
4983 offset: i32,
4984 heap_access_oob: Label,
4985 unaligned_atomic: Label,
4986 ) -> Result<(), CompileError> {
4987 self.memory_op(
4988 target,
4989 memarg,
4990 true,
4991 1,
4992 imported_memories,
4993 offset,
4994 heap_access_oob,
4995 unaligned_atomic,
4996 |this, addr| {
4997 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Add, Size::S8, ret, addr, loc)
4998 },
4999 )
5000 }
5001 fn i64_atomic_add_16u(
5002 &mut self,
5003 loc: Location,
5004 target: Location,
5005 memarg: &MemArg,
5006 ret: Location,
5007 _need_check: bool,
5008 imported_memories: bool,
5009 offset: i32,
5010 heap_access_oob: Label,
5011 unaligned_atomic: Label,
5012 ) -> Result<(), CompileError> {
5013 self.memory_op(
5014 target,
5015 memarg,
5016 true,
5017 2,
5018 imported_memories,
5019 offset,
5020 heap_access_oob,
5021 unaligned_atomic,
5022 |this, addr| {
5023 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Add, Size::S16, ret, addr, loc)
5024 },
5025 )
5026 }
5027 fn i64_atomic_add_32u(
5028 &mut self,
5029 loc: Location,
5030 target: Location,
5031 memarg: &MemArg,
5032 ret: Location,
5033 _need_check: bool,
5034 imported_memories: bool,
5035 offset: i32,
5036 heap_access_oob: Label,
5037 unaligned_atomic: Label,
5038 ) -> Result<(), CompileError> {
5039 self.memory_op(
5040 target,
5041 memarg,
5042 true,
5043 4,
5044 imported_memories,
5045 offset,
5046 heap_access_oob,
5047 unaligned_atomic,
5048 |this, addr| {
5049 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Add, Size::S32, ret, addr, loc)
5050 },
5051 )
5052 }
5053 fn i64_atomic_sub(
5054 &mut self,
5055 loc: Location,
5056 target: Location,
5057 memarg: &MemArg,
5058 ret: Location,
5059 _need_check: bool,
5060 imported_memories: bool,
5061 offset: i32,
5062 heap_access_oob: Label,
5063 unaligned_atomic: Label,
5064 ) -> Result<(), CompileError> {
5065 self.memory_op(
5066 target,
5067 memarg,
5068 true,
5069 8,
5070 imported_memories,
5071 offset,
5072 heap_access_oob,
5073 unaligned_atomic,
5074 |this, addr| {
5075 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Sub, Size::S64, ret, addr, loc)
5076 },
5077 )
5078 }
5079 fn i64_atomic_sub_8u(
5080 &mut self,
5081 loc: Location,
5082 target: Location,
5083 memarg: &MemArg,
5084 ret: Location,
5085 _need_check: bool,
5086 imported_memories: bool,
5087 offset: i32,
5088 heap_access_oob: Label,
5089 unaligned_atomic: Label,
5090 ) -> Result<(), CompileError> {
5091 self.memory_op(
5092 target,
5093 memarg,
5094 true,
5095 1,
5096 imported_memories,
5097 offset,
5098 heap_access_oob,
5099 unaligned_atomic,
5100 |this, addr| {
5101 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Sub, Size::S8, ret, addr, loc)
5102 },
5103 )
5104 }
5105 fn i64_atomic_sub_16u(
5106 &mut self,
5107 loc: Location,
5108 target: Location,
5109 memarg: &MemArg,
5110 ret: Location,
5111 _need_check: bool,
5112 imported_memories: bool,
5113 offset: i32,
5114 heap_access_oob: Label,
5115 unaligned_atomic: Label,
5116 ) -> Result<(), CompileError> {
5117 self.memory_op(
5118 target,
5119 memarg,
5120 true,
5121 2,
5122 imported_memories,
5123 offset,
5124 heap_access_oob,
5125 unaligned_atomic,
5126 |this, addr| {
5127 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Sub, Size::S16, ret, addr, loc)
5128 },
5129 )
5130 }
5131 fn i64_atomic_sub_32u(
5132 &mut self,
5133 loc: Location,
5134 target: Location,
5135 memarg: &MemArg,
5136 ret: Location,
5137 _need_check: bool,
5138 imported_memories: bool,
5139 offset: i32,
5140 heap_access_oob: Label,
5141 unaligned_atomic: Label,
5142 ) -> Result<(), CompileError> {
5143 self.memory_op(
5144 target,
5145 memarg,
5146 true,
5147 4,
5148 imported_memories,
5149 offset,
5150 heap_access_oob,
5151 unaligned_atomic,
5152 |this, addr| {
5153 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Sub, Size::S32, ret, addr, loc)
5154 },
5155 )
5156 }
5157 fn i64_atomic_and(
5158 &mut self,
5159 loc: Location,
5160 target: Location,
5161 memarg: &MemArg,
5162 ret: Location,
5163 _need_check: bool,
5164 imported_memories: bool,
5165 offset: i32,
5166 heap_access_oob: Label,
5167 unaligned_atomic: Label,
5168 ) -> Result<(), CompileError> {
5169 self.memory_op(
5170 target,
5171 memarg,
5172 true,
5173 8,
5174 imported_memories,
5175 offset,
5176 heap_access_oob,
5177 unaligned_atomic,
5178 |this, addr| {
5179 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::And, Size::S64, ret, addr, loc)
5180 },
5181 )
5182 }
5183 fn i64_atomic_and_8u(
5184 &mut self,
5185 loc: Location,
5186 target: Location,
5187 memarg: &MemArg,
5188 ret: Location,
5189 _need_check: bool,
5190 imported_memories: bool,
5191 offset: i32,
5192 heap_access_oob: Label,
5193 unaligned_atomic: Label,
5194 ) -> Result<(), CompileError> {
5195 self.memory_op(
5196 target,
5197 memarg,
5198 true,
5199 1,
5200 imported_memories,
5201 offset,
5202 heap_access_oob,
5203 unaligned_atomic,
5204 |this, addr| {
5205 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::And, Size::S8, ret, addr, loc)
5206 },
5207 )
5208 }
5209 fn i64_atomic_and_16u(
5210 &mut self,
5211 loc: Location,
5212 target: Location,
5213 memarg: &MemArg,
5214 ret: Location,
5215 _need_check: bool,
5216 imported_memories: bool,
5217 offset: i32,
5218 heap_access_oob: Label,
5219 unaligned_atomic: Label,
5220 ) -> Result<(), CompileError> {
5221 self.memory_op(
5222 target,
5223 memarg,
5224 true,
5225 2,
5226 imported_memories,
5227 offset,
5228 heap_access_oob,
5229 unaligned_atomic,
5230 |this, addr| {
5231 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::And, Size::S16, ret, addr, loc)
5232 },
5233 )
5234 }
5235 fn i64_atomic_and_32u(
5236 &mut self,
5237 loc: Location,
5238 target: Location,
5239 memarg: &MemArg,
5240 ret: Location,
5241 _need_check: bool,
5242 imported_memories: bool,
5243 offset: i32,
5244 heap_access_oob: Label,
5245 unaligned_atomic: Label,
5246 ) -> Result<(), CompileError> {
5247 self.memory_op(
5248 target,
5249 memarg,
5250 true,
5251 4,
5252 imported_memories,
5253 offset,
5254 heap_access_oob,
5255 unaligned_atomic,
5256 |this, addr| {
5257 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::And, Size::S32, ret, addr, loc)
5258 },
5259 )
5260 }
5261 fn i64_atomic_or(
5262 &mut self,
5263 loc: Location,
5264 target: Location,
5265 memarg: &MemArg,
5266 ret: Location,
5267 _need_check: bool,
5268 imported_memories: bool,
5269 offset: i32,
5270 heap_access_oob: Label,
5271 unaligned_atomic: Label,
5272 ) -> Result<(), CompileError> {
5273 self.memory_op(
5274 target,
5275 memarg,
5276 true,
5277 8,
5278 imported_memories,
5279 offset,
5280 heap_access_oob,
5281 unaligned_atomic,
5282 |this, addr| {
5283 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Or, Size::S64, ret, addr, loc)
5284 },
5285 )
5286 }
5287 fn i64_atomic_or_8u(
5288 &mut self,
5289 loc: Location,
5290 target: Location,
5291 memarg: &MemArg,
5292 ret: Location,
5293 _need_check: bool,
5294 imported_memories: bool,
5295 offset: i32,
5296 heap_access_oob: Label,
5297 unaligned_atomic: Label,
5298 ) -> Result<(), CompileError> {
5299 self.memory_op(
5300 target,
5301 memarg,
5302 true,
5303 1,
5304 imported_memories,
5305 offset,
5306 heap_access_oob,
5307 unaligned_atomic,
5308 |this, addr| {
5309 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Or, Size::S8, ret, addr, loc)
5310 },
5311 )
5312 }
5313 fn i64_atomic_or_16u(
5314 &mut self,
5315 loc: Location,
5316 target: Location,
5317 memarg: &MemArg,
5318 ret: Location,
5319 _need_check: bool,
5320 imported_memories: bool,
5321 offset: i32,
5322 heap_access_oob: Label,
5323 unaligned_atomic: Label,
5324 ) -> Result<(), CompileError> {
5325 self.memory_op(
5326 target,
5327 memarg,
5328 true,
5329 2,
5330 imported_memories,
5331 offset,
5332 heap_access_oob,
5333 unaligned_atomic,
5334 |this, addr| {
5335 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Or, Size::S16, ret, addr, loc)
5336 },
5337 )
5338 }
5339 fn i64_atomic_or_32u(
5340 &mut self,
5341 loc: Location,
5342 target: Location,
5343 memarg: &MemArg,
5344 ret: Location,
5345 _need_check: bool,
5346 imported_memories: bool,
5347 offset: i32,
5348 heap_access_oob: Label,
5349 unaligned_atomic: Label,
5350 ) -> Result<(), CompileError> {
5351 self.memory_op(
5352 target,
5353 memarg,
5354 true,
5355 4,
5356 imported_memories,
5357 offset,
5358 heap_access_oob,
5359 unaligned_atomic,
5360 |this, addr| {
5361 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Or, Size::S32, ret, addr, loc)
5362 },
5363 )
5364 }
5365 fn i64_atomic_xor(
5366 &mut self,
5367 loc: Location,
5368 target: Location,
5369 memarg: &MemArg,
5370 ret: Location,
5371 _need_check: bool,
5372 imported_memories: bool,
5373 offset: i32,
5374 heap_access_oob: Label,
5375 unaligned_atomic: Label,
5376 ) -> Result<(), CompileError> {
5377 self.memory_op(
5378 target,
5379 memarg,
5380 true,
5381 8,
5382 imported_memories,
5383 offset,
5384 heap_access_oob,
5385 unaligned_atomic,
5386 |this, addr| {
5387 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Xor, Size::S64, ret, addr, loc)
5388 },
5389 )
5390 }
5391 fn i64_atomic_xor_8u(
5392 &mut self,
5393 loc: Location,
5394 target: Location,
5395 memarg: &MemArg,
5396 ret: Location,
5397 _need_check: bool,
5398 imported_memories: bool,
5399 offset: i32,
5400 heap_access_oob: Label,
5401 unaligned_atomic: Label,
5402 ) -> Result<(), CompileError> {
5403 self.memory_op(
5404 target,
5405 memarg,
5406 true,
5407 1,
5408 imported_memories,
5409 offset,
5410 heap_access_oob,
5411 unaligned_atomic,
5412 |this, addr| {
5413 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Xor, Size::S8, ret, addr, loc)
5414 },
5415 )
5416 }
5417 fn i64_atomic_xor_16u(
5418 &mut self,
5419 loc: Location,
5420 target: Location,
5421 memarg: &MemArg,
5422 ret: Location,
5423 _need_check: bool,
5424 imported_memories: bool,
5425 offset: i32,
5426 heap_access_oob: Label,
5427 unaligned_atomic: Label,
5428 ) -> Result<(), CompileError> {
5429 self.memory_op(
5430 target,
5431 memarg,
5432 true,
5433 2,
5434 imported_memories,
5435 offset,
5436 heap_access_oob,
5437 unaligned_atomic,
5438 |this, addr| {
5439 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Xor, Size::S16, ret, addr, loc)
5440 },
5441 )
5442 }
5443 fn i64_atomic_xor_32u(
5444 &mut self,
5445 loc: Location,
5446 target: Location,
5447 memarg: &MemArg,
5448 ret: Location,
5449 _need_check: bool,
5450 imported_memories: bool,
5451 offset: i32,
5452 heap_access_oob: Label,
5453 unaligned_atomic: Label,
5454 ) -> Result<(), CompileError> {
5455 self.memory_op(
5456 target,
5457 memarg,
5458 true,
5459 4,
5460 imported_memories,
5461 offset,
5462 heap_access_oob,
5463 unaligned_atomic,
5464 |this, addr| {
5465 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Xor, Size::S32, ret, addr, loc)
5466 },
5467 )
5468 }
5469 fn i64_atomic_xchg(
5470 &mut self,
5471 loc: Location,
5472 target: Location,
5473 memarg: &MemArg,
5474 ret: Location,
5475 _need_check: bool,
5476 imported_memories: bool,
5477 offset: i32,
5478 heap_access_oob: Label,
5479 unaligned_atomic: Label,
5480 ) -> Result<(), CompileError> {
5481 self.memory_op(
5482 target,
5483 memarg,
5484 true,
5485 8,
5486 imported_memories,
5487 offset,
5488 heap_access_oob,
5489 unaligned_atomic,
5490 |this, addr| {
5491 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Exchange, Size::S64, ret, addr, loc)
5492 },
5493 )
5494 }
5495 fn i64_atomic_xchg_8u(
5496 &mut self,
5497 loc: Location,
5498 target: Location,
5499 memarg: &MemArg,
5500 ret: Location,
5501 _need_check: bool,
5502 imported_memories: bool,
5503 offset: i32,
5504 heap_access_oob: Label,
5505 unaligned_atomic: Label,
5506 ) -> Result<(), CompileError> {
5507 self.memory_op(
5508 target,
5509 memarg,
5510 true,
5511 1,
5512 imported_memories,
5513 offset,
5514 heap_access_oob,
5515 unaligned_atomic,
5516 |this, addr| {
5517 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Exchange, Size::S8, ret, addr, loc)
5518 },
5519 )
5520 }
5521 fn i64_atomic_xchg_16u(
5522 &mut self,
5523 loc: Location,
5524 target: Location,
5525 memarg: &MemArg,
5526 ret: Location,
5527 _need_check: bool,
5528 imported_memories: bool,
5529 offset: i32,
5530 heap_access_oob: Label,
5531 unaligned_atomic: Label,
5532 ) -> Result<(), CompileError> {
5533 self.memory_op(
5534 target,
5535 memarg,
5536 true,
5537 2,
5538 imported_memories,
5539 offset,
5540 heap_access_oob,
5541 unaligned_atomic,
5542 |this, addr| {
5543 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Exchange, Size::S16, ret, addr, loc)
5544 },
5545 )
5546 }
5547 fn i64_atomic_xchg_32u(
5548 &mut self,
5549 loc: Location,
5550 target: Location,
5551 memarg: &MemArg,
5552 ret: Location,
5553 _need_check: bool,
5554 imported_memories: bool,
5555 offset: i32,
5556 heap_access_oob: Label,
5557 unaligned_atomic: Label,
5558 ) -> Result<(), CompileError> {
5559 self.memory_op(
5560 target,
5561 memarg,
5562 true,
5563 4,
5564 imported_memories,
5565 offset,
5566 heap_access_oob,
5567 unaligned_atomic,
5568 |this, addr| {
5569 this.emit_relaxed_atomic_binop3(AtomicBinaryOp::Exchange, Size::S32, ret, addr, loc)
5570 },
5571 )
5572 }
5573 fn i64_atomic_cmpxchg(
5574 &mut self,
5575 new: Location,
5576 cmp: Location,
5577 target: Location,
5578 memarg: &MemArg,
5579 ret: Location,
5580 _need_check: bool,
5581 imported_memories: bool,
5582 offset: i32,
5583 heap_access_oob: Label,
5584 unaligned_atomic: Label,
5585 ) -> Result<(), CompileError> {
5586 self.memory_op(
5587 target,
5588 memarg,
5589 true,
5590 8,
5591 imported_memories,
5592 offset,
5593 heap_access_oob,
5594 unaligned_atomic,
5595 |this, addr| this.emit_relaxed_atomic_cmpxchg(Size::S64, ret, addr, new, cmp),
5596 )
5597 }
5598 fn i64_atomic_cmpxchg_8u(
5599 &mut self,
5600 new: Location,
5601 cmp: Location,
5602 target: Location,
5603 memarg: &MemArg,
5604 ret: Location,
5605 _need_check: bool,
5606 imported_memories: bool,
5607 offset: i32,
5608 heap_access_oob: Label,
5609 unaligned_atomic: Label,
5610 ) -> Result<(), CompileError> {
5611 self.memory_op(
5612 target,
5613 memarg,
5614 true,
5615 1,
5616 imported_memories,
5617 offset,
5618 heap_access_oob,
5619 unaligned_atomic,
5620 |this, addr| this.emit_relaxed_atomic_cmpxchg(Size::S8, ret, addr, new, cmp),
5621 )
5622 }
5623 fn i64_atomic_cmpxchg_16u(
5624 &mut self,
5625 new: Location,
5626 cmp: Location,
5627 target: Location,
5628 memarg: &MemArg,
5629 ret: Location,
5630 _need_check: bool,
5631 imported_memories: bool,
5632 offset: i32,
5633 heap_access_oob: Label,
5634 unaligned_atomic: Label,
5635 ) -> Result<(), CompileError> {
5636 self.memory_op(
5637 target,
5638 memarg,
5639 true,
5640 2,
5641 imported_memories,
5642 offset,
5643 heap_access_oob,
5644 unaligned_atomic,
5645 |this, addr| this.emit_relaxed_atomic_cmpxchg(Size::S16, ret, addr, new, cmp),
5646 )
5647 }
5648 fn i64_atomic_cmpxchg_32u(
5649 &mut self,
5650 new: Location,
5651 cmp: Location,
5652 target: Location,
5653 memarg: &MemArg,
5654 ret: Location,
5655 _need_check: bool,
5656 imported_memories: bool,
5657 offset: i32,
5658 heap_access_oob: Label,
5659 unaligned_atomic: Label,
5660 ) -> Result<(), CompileError> {
5661 self.memory_op(
5662 target,
5663 memarg,
5664 true,
5665 4,
5666 imported_memories,
5667 offset,
5668 heap_access_oob,
5669 unaligned_atomic,
5670 |this, addr| this.emit_relaxed_atomic_cmpxchg(Size::S32, ret, addr, new, cmp),
5671 )
5672 }
5673 fn f32_load(
5674 &mut self,
5675 addr: Location,
5676 memarg: &MemArg,
5677 ret: Location,
5678 _need_check: bool,
5679 imported_memories: bool,
5680 offset: i32,
5681 heap_access_oob: Label,
5682 unaligned_atomic: Label,
5683 ) -> Result<(), CompileError> {
5684 self.memory_op(
5685 addr,
5686 memarg,
5687 false,
5688 4,
5689 imported_memories,
5690 offset,
5691 heap_access_oob,
5692 unaligned_atomic,
5693 |this, addr| this.emit_relaxed_load(Size::S32, false, ret, Location::Memory(addr, 0)),
5694 )
5695 }
5696 fn f32_save(
5697 &mut self,
5698 value: Location,
5699 memarg: &MemArg,
5700 addr: Location,
5701 canonicalize: bool,
5702 _need_check: bool,
5703 imported_memories: bool,
5704 offset: i32,
5705 heap_access_oob: Label,
5706 unaligned_atomic: Label,
5707 ) -> Result<(), CompileError> {
5708 self.memory_op(
5709 addr,
5710 memarg,
5711 false,
5712 4,
5713 imported_memories,
5714 offset,
5715 heap_access_oob,
5716 unaligned_atomic,
5717 |this, addr| {
5718 if !canonicalize {
5719 this.emit_relaxed_store(Size::S32, value, Location::Memory(addr, 0))
5720 } else {
5721 this.canonicalize_nan(Size::S32, value, Location::Memory(addr, 0))
5722 }
5723 },
5724 )
5725 }
5726 fn f64_load(
5727 &mut self,
5728 addr: Location,
5729 memarg: &MemArg,
5730 ret: Location,
5731 _need_check: bool,
5732 imported_memories: bool,
5733 offset: i32,
5734 heap_access_oob: Label,
5735 unaligned_atomic: Label,
5736 ) -> Result<(), CompileError> {
5737 self.memory_op(
5738 addr,
5739 memarg,
5740 false,
5741 8,
5742 imported_memories,
5743 offset,
5744 heap_access_oob,
5745 unaligned_atomic,
5746 |this, addr| this.emit_relaxed_load(Size::S64, false, ret, Location::Memory(addr, 0)),
5747 )
5748 }
5749 fn f64_save(
5750 &mut self,
5751 value: Location,
5752 memarg: &MemArg,
5753 addr: Location,
5754 canonicalize: bool,
5755 _need_check: bool,
5756 imported_memories: bool,
5757 offset: i32,
5758 heap_access_oob: Label,
5759 unaligned_atomic: Label,
5760 ) -> Result<(), CompileError> {
5761 self.memory_op(
5762 addr,
5763 memarg,
5764 false,
5765 8,
5766 imported_memories,
5767 offset,
5768 heap_access_oob,
5769 unaligned_atomic,
5770 |this, addr| {
5771 if !canonicalize {
5772 this.emit_relaxed_store(Size::S64, value, Location::Memory(addr, 0))
5773 } else {
5774 this.canonicalize_nan(Size::S64, value, Location::Memory(addr, 0))
5775 }
5776 },
5777 )
5778 }
5779 fn convert_f64_i64(
5780 &mut self,
5781 loc: Location,
5782 signed: bool,
5783 ret: Location,
5784 ) -> Result<(), CompileError> {
5785 self.convert_int_to_float(loc, Size::S64, ret, Size::S64, signed)
5786 }
5787 fn convert_f64_i32(
5788 &mut self,
5789 loc: Location,
5790 signed: bool,
5791 ret: Location,
5792 ) -> Result<(), CompileError> {
5793 self.convert_int_to_float(loc, Size::S32, ret, Size::S64, signed)
5794 }
5795 fn convert_f32_i64(
5796 &mut self,
5797 loc: Location,
5798 signed: bool,
5799 ret: Location,
5800 ) -> Result<(), CompileError> {
5801 self.convert_int_to_float(loc, Size::S64, ret, Size::S32, signed)
5802 }
5803 fn convert_f32_i32(
5804 &mut self,
5805 loc: Location,
5806 signed: bool,
5807 ret: Location,
5808 ) -> Result<(), CompileError> {
5809 self.convert_int_to_float(loc, Size::S32, ret, Size::S32, signed)
5810 }
5811 fn convert_i64_f64(
5812 &mut self,
5813 loc: Location,
5814 ret: Location,
5815 signed: bool,
5816 sat: bool,
5817 ) -> Result<(), CompileError> {
5818 self.convert_float_to_int(loc, Size::S64, ret, Size::S64, signed, sat)
5819 }
5820 fn convert_i32_f64(
5821 &mut self,
5822 loc: Location,
5823 ret: Location,
5824 signed: bool,
5825 sat: bool,
5826 ) -> Result<(), CompileError> {
5827 self.convert_float_to_int(loc, Size::S64, ret, Size::S32, signed, sat)
5828 }
5829 fn convert_i64_f32(
5830 &mut self,
5831 loc: Location,
5832 ret: Location,
5833 signed: bool,
5834 sat: bool,
5835 ) -> Result<(), CompileError> {
5836 self.convert_float_to_int(loc, Size::S32, ret, Size::S64, signed, sat)
5837 }
5838 fn convert_i32_f32(
5839 &mut self,
5840 loc: Location,
5841 ret: Location,
5842 signed: bool,
5843 sat: bool,
5844 ) -> Result<(), CompileError> {
5845 self.convert_float_to_int(loc, Size::S32, ret, Size::S32, signed, sat)
5846 }
5847 fn convert_f64_f32(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
5848 self.convert_float_to_float(loc, Size::S32, ret, Size::S64)
5849 }
5850 fn convert_f32_f64(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
5851 self.convert_float_to_float(loc, Size::S64, ret, Size::S32)
5852 }
5853 fn f64_neg(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
5854 self.emit_relaxed_binop_fp(Assembler::emit_fneg, Size::S64, loc, ret, true)
5855 }
5856 fn f64_abs(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
5857 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
5858 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5859 })?;
5860 let mask = self.acquire_temp_gpr().ok_or_else(|| {
5861 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5862 })?;
5863
5864 self.move_location(Size::S64, loc, Location::GPR(tmp))?;
5865 self.assembler
5866 .emit_mov_imm(Location::GPR(mask), 0x7fffffffffffffffi64)?;
5867 self.assembler.emit_and(
5868 Size::S64,
5869 Location::GPR(tmp),
5870 Location::GPR(mask),
5871 Location::GPR(tmp),
5872 )?;
5873 self.move_location(Size::S64, Location::GPR(tmp), ret)?;
5874
5875 self.release_gpr(tmp);
5876 self.release_gpr(mask);
5877 Ok(())
5878 }
5879 fn emit_i64_copysign(&mut self, tmp1: Self::GPR, tmp2: Self::GPR) -> Result<(), CompileError> {
5880 let mask = self.acquire_temp_gpr().ok_or_else(|| {
5881 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5882 })?;
5883
5884 self.assembler
5885 .emit_mov_imm(Location::GPR(mask), 0x7fffffffffffffffu64 as _)?;
5886 self.assembler.emit_and(
5887 Size::S64,
5888 Location::GPR(tmp1),
5889 Location::GPR(mask),
5890 Location::GPR(tmp1),
5891 )?;
5892
5893 self.assembler
5894 .emit_mov_imm(Location::GPR(mask), 0x8000000000000000u64 as _)?;
5895 self.assembler.emit_and(
5896 Size::S64,
5897 Location::GPR(tmp2),
5898 Location::GPR(mask),
5899 Location::GPR(tmp2),
5900 )?;
5901
5902 self.release_gpr(mask);
5903 self.assembler.emit_or(
5904 Size::S64,
5905 Location::GPR(tmp1),
5906 Location::GPR(tmp2),
5907 Location::GPR(tmp1),
5908 )
5909 }
5910 fn f64_sqrt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
5911 self.emit_relaxed_binop_fp(Assembler::emit_fsqrt, Size::S64, loc, ret, true)
5912 }
5913 fn f64_trunc(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
5914 self.emit_relaxed_fcvt_with_rounding(RoundingMode::Rtz, Size::S64, loc, ret)
5915 }
5916 fn f64_ceil(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
5917 self.emit_relaxed_fcvt_with_rounding(RoundingMode::Rup, Size::S64, loc, ret)
5918 }
5919 fn f64_floor(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
5920 self.emit_relaxed_fcvt_with_rounding(RoundingMode::Rdn, Size::S64, loc, ret)
5921 }
5922 fn f64_nearest(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
5923 self.emit_relaxed_fcvt_with_rounding(RoundingMode::Rne, Size::S64, loc, ret)
5924 }
5925 fn f64_cmp_ge(
5926 &mut self,
5927 loc_a: Location,
5928 loc_b: Location,
5929 ret: Location,
5930 ) -> Result<(), CompileError> {
5931 self.emit_relaxed_fcmp(Condition::Ge, Size::S64, loc_a, loc_b, ret)
5932 }
5933 fn f64_cmp_gt(
5934 &mut self,
5935 loc_a: Location,
5936 loc_b: Location,
5937 ret: Location,
5938 ) -> Result<(), CompileError> {
5939 self.emit_relaxed_fcmp(Condition::Gt, Size::S64, loc_a, loc_b, ret)
5940 }
5941 fn f64_cmp_le(
5942 &mut self,
5943 loc_a: Location,
5944 loc_b: Location,
5945 ret: Location,
5946 ) -> Result<(), CompileError> {
5947 self.emit_relaxed_fcmp(Condition::Le, Size::S64, loc_a, loc_b, ret)
5948 }
5949 fn f64_cmp_lt(
5950 &mut self,
5951 loc_a: Location,
5952 loc_b: Location,
5953 ret: Location,
5954 ) -> Result<(), CompileError> {
5955 self.emit_relaxed_fcmp(Condition::Lt, Size::S64, loc_a, loc_b, ret)
5956 }
5957 fn f64_cmp_ne(
5958 &mut self,
5959 loc_a: Location,
5960 loc_b: Location,
5961 ret: Location,
5962 ) -> Result<(), CompileError> {
5963 self.emit_relaxed_fcmp(Condition::Ne, Size::S64, loc_a, loc_b, ret)
5964 }
5965 fn f64_cmp_eq(
5966 &mut self,
5967 loc_a: Location,
5968 loc_b: Location,
5969 ret: Location,
5970 ) -> Result<(), CompileError> {
5971 self.emit_relaxed_fcmp(Condition::Eq, Size::S64, loc_a, loc_b, ret)
5972 }
5973 fn f64_min(
5974 &mut self,
5975 loc_a: Location,
5976 loc_b: Location,
5977 ret: Location,
5978 ) -> Result<(), CompileError> {
5979 self.emit_relaxed_binop3_fp(
5980 Assembler::emit_fmin,
5981 Size::S64,
5982 loc_a,
5983 loc_b,
5984 ret,
5985 ImmType::None,
5986 true,
5987 )
5988 }
5989 fn f64_max(
5990 &mut self,
5991 loc_a: Location,
5992 loc_b: Location,
5993 ret: Location,
5994 ) -> Result<(), CompileError> {
5995 self.emit_relaxed_binop3_fp(
5996 Assembler::emit_fmax,
5997 Size::S64,
5998 loc_a,
5999 loc_b,
6000 ret,
6001 ImmType::None,
6002 true,
6003 )
6004 }
6005 fn f64_add(
6006 &mut self,
6007 loc_a: Location,
6008 loc_b: Location,
6009 ret: Location,
6010 ) -> Result<(), CompileError> {
6011 self.emit_relaxed_binop3_fp(
6012 Assembler::emit_add,
6013 Size::S64,
6014 loc_a,
6015 loc_b,
6016 ret,
6017 ImmType::None,
6018 false,
6019 )
6020 }
6021 fn f64_sub(
6022 &mut self,
6023 loc_a: Location,
6024 loc_b: Location,
6025 ret: Location,
6026 ) -> Result<(), CompileError> {
6027 self.emit_relaxed_binop3_fp(
6028 Assembler::emit_sub,
6029 Size::S64,
6030 loc_a,
6031 loc_b,
6032 ret,
6033 ImmType::None,
6034 false,
6035 )
6036 }
6037 fn f64_mul(
6038 &mut self,
6039 loc_a: Location,
6040 loc_b: Location,
6041 ret: Location,
6042 ) -> Result<(), CompileError> {
6043 self.emit_relaxed_binop3_fp(
6044 Assembler::emit_mul,
6045 Size::S64,
6046 loc_a,
6047 loc_b,
6048 ret,
6049 ImmType::None,
6050 false,
6051 )
6052 }
6053 fn f64_div(
6054 &mut self,
6055 loc_a: Location,
6056 loc_b: Location,
6057 ret: Location,
6058 ) -> Result<(), CompileError> {
6059 self.emit_relaxed_binop3_fp(
6060 Assembler::emit_fdiv,
6061 Size::S64,
6062 loc_a,
6063 loc_b,
6064 ret,
6065 ImmType::None,
6066 false,
6067 )
6068 }
6069 fn f32_neg(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6070 self.emit_relaxed_binop_fp(Assembler::emit_fneg, Size::S32, loc, ret, true)
6071 }
6072 fn f32_abs(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6073 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
6074 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6075 })?;
6076 let mask = self.acquire_temp_gpr().ok_or_else(|| {
6077 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6078 })?;
6079
6080 self.move_location(Size::S32, loc, Location::GPR(tmp))?;
6081 self.assembler
6082 .emit_mov_imm(Location::GPR(mask), 0x7fffffffi64)?;
6083 self.assembler.emit_and(
6084 Size::S32,
6085 Location::GPR(tmp),
6086 Location::GPR(mask),
6087 Location::GPR(tmp),
6088 )?;
6089 self.move_location(Size::S32, Location::GPR(tmp), ret)?;
6090
6091 self.release_gpr(tmp);
6092 self.release_gpr(mask);
6093 Ok(())
6094 }
6095 fn emit_i32_copysign(&mut self, tmp1: Self::GPR, tmp2: Self::GPR) -> Result<(), CompileError> {
6096 let mask = self.acquire_temp_gpr().ok_or_else(|| {
6097 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6098 })?;
6099
6100 self.assembler
6101 .emit_mov_imm(Location::GPR(mask), 0x7fffffffu32 as _)?;
6102 self.assembler.emit_and(
6103 Size::S32,
6104 Location::GPR(tmp1),
6105 Location::GPR(mask),
6106 Location::GPR(tmp1),
6107 )?;
6108
6109 self.assembler
6110 .emit_mov_imm(Location::GPR(mask), 0x80000000u32 as _)?;
6111 self.assembler.emit_and(
6112 Size::S32,
6113 Location::GPR(tmp2),
6114 Location::GPR(mask),
6115 Location::GPR(tmp2),
6116 )?;
6117
6118 self.release_gpr(mask);
6119 self.assembler.emit_or(
6120 Size::S32,
6121 Location::GPR(tmp1),
6122 Location::GPR(tmp2),
6123 Location::GPR(tmp1),
6124 )
6125 }
6126 fn f32_sqrt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6127 self.emit_relaxed_binop_fp(Assembler::emit_fsqrt, Size::S32, loc, ret, true)
6128 }
6129 fn f32_trunc(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6130 self.emit_relaxed_fcvt_with_rounding(RoundingMode::Rtz, Size::S32, loc, ret)
6131 }
6132 fn f32_ceil(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6133 self.emit_relaxed_fcvt_with_rounding(RoundingMode::Rup, Size::S32, loc, ret)
6134 }
6135 fn f32_floor(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6136 self.emit_relaxed_fcvt_with_rounding(RoundingMode::Rdn, Size::S32, loc, ret)
6137 }
6138 fn f32_nearest(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
6139 self.emit_relaxed_fcvt_with_rounding(RoundingMode::Rne, Size::S32, loc, ret)
6140 }
6141 fn f32_cmp_ge(
6142 &mut self,
6143 loc_a: Location,
6144 loc_b: Location,
6145 ret: Location,
6146 ) -> Result<(), CompileError> {
6147 self.emit_relaxed_fcmp(Condition::Ge, Size::S32, loc_a, loc_b, ret)
6148 }
6149 fn f32_cmp_gt(
6150 &mut self,
6151 loc_a: Location,
6152 loc_b: Location,
6153 ret: Location,
6154 ) -> Result<(), CompileError> {
6155 self.emit_relaxed_fcmp(Condition::Gt, Size::S32, loc_a, loc_b, ret)
6156 }
6157 fn f32_cmp_le(
6158 &mut self,
6159 loc_a: Location,
6160 loc_b: Location,
6161 ret: Location,
6162 ) -> Result<(), CompileError> {
6163 self.emit_relaxed_fcmp(Condition::Le, Size::S32, loc_a, loc_b, ret)
6164 }
6165 fn f32_cmp_lt(
6166 &mut self,
6167 loc_a: Location,
6168 loc_b: Location,
6169 ret: Location,
6170 ) -> Result<(), CompileError> {
6171 self.emit_relaxed_fcmp(Condition::Lt, Size::S32, loc_a, loc_b, ret)
6172 }
6173 fn f32_cmp_ne(
6174 &mut self,
6175 loc_a: Location,
6176 loc_b: Location,
6177 ret: Location,
6178 ) -> Result<(), CompileError> {
6179 self.emit_relaxed_fcmp(Condition::Ne, Size::S32, loc_a, loc_b, ret)
6180 }
6181 fn f32_cmp_eq(
6182 &mut self,
6183 loc_a: Location,
6184 loc_b: Location,
6185 ret: Location,
6186 ) -> Result<(), CompileError> {
6187 self.emit_relaxed_fcmp(Condition::Eq, Size::S32, loc_a, loc_b, ret)
6188 }
6189 fn f32_min(
6190 &mut self,
6191 loc_a: Location,
6192 loc_b: Location,
6193 ret: Location,
6194 ) -> Result<(), CompileError> {
6195 self.emit_relaxed_binop3_fp(
6196 Assembler::emit_fmin,
6197 Size::S32,
6198 loc_a,
6199 loc_b,
6200 ret,
6201 ImmType::None,
6202 true,
6203 )
6204 }
6205 fn f32_max(
6206 &mut self,
6207 loc_a: Location,
6208 loc_b: Location,
6209 ret: Location,
6210 ) -> Result<(), CompileError> {
6211 self.emit_relaxed_binop3_fp(
6212 Assembler::emit_fmax,
6213 Size::S32,
6214 loc_a,
6215 loc_b,
6216 ret,
6217 ImmType::None,
6218 true,
6219 )
6220 }
6221 fn f32_add(
6222 &mut self,
6223 loc_a: Location,
6224 loc_b: Location,
6225 ret: Location,
6226 ) -> Result<(), CompileError> {
6227 self.emit_relaxed_binop3_fp(
6228 Assembler::emit_add,
6229 Size::S32,
6230 loc_a,
6231 loc_b,
6232 ret,
6233 ImmType::None,
6234 false,
6235 )
6236 }
6237 fn f32_sub(
6238 &mut self,
6239 loc_a: Location,
6240 loc_b: Location,
6241 ret: Location,
6242 ) -> Result<(), CompileError> {
6243 self.emit_relaxed_binop3_fp(
6244 Assembler::emit_sub,
6245 Size::S32,
6246 loc_a,
6247 loc_b,
6248 ret,
6249 ImmType::None,
6250 false,
6251 )
6252 }
6253 fn f32_mul(
6254 &mut self,
6255 loc_a: Location,
6256 loc_b: Location,
6257 ret: Location,
6258 ) -> Result<(), CompileError> {
6259 self.emit_relaxed_binop3_fp(
6260 Assembler::emit_mul,
6261 Size::S32,
6262 loc_a,
6263 loc_b,
6264 ret,
6265 ImmType::None,
6266 false,
6267 )
6268 }
6269 fn f32_div(
6270 &mut self,
6271 loc_a: Location,
6272 loc_b: Location,
6273 ret: Location,
6274 ) -> Result<(), CompileError> {
6275 self.emit_relaxed_binop3_fp(
6276 Assembler::emit_fdiv,
6277 Size::S32,
6278 loc_a,
6279 loc_b,
6280 ret,
6281 ImmType::None,
6282 false,
6283 )
6284 }
6285 fn gen_std_trampoline(
6286 &self,
6287 sig: &FunctionType,
6288 calling_convention: CallingConvention,
6289 ) -> Result<FunctionBody, CompileError> {
6290 gen_std_trampoline_riscv(sig, calling_convention)
6291 }
6292 fn gen_std_dynamic_import_trampoline(
6293 &self,
6294 vmoffsets: &VMOffsets,
6295 sig: &FunctionType,
6296 _calling_convention: CallingConvention,
6297 ) -> Result<FunctionBody, CompileError> {
6298 gen_std_dynamic_import_trampoline_riscv(vmoffsets, sig)
6299 }
6300 fn gen_import_call_trampoline(
6302 &self,
6303 vmoffsets: &VMOffsets,
6304 index: FunctionIndex,
6305 sig: &FunctionType,
6306 calling_convention: CallingConvention,
6307 ) -> Result<CustomSection, CompileError> {
6308 gen_import_call_trampoline_riscv(vmoffsets, index, sig, calling_convention)
6309 }
6310 #[cfg(feature = "unwind")]
6311 fn gen_dwarf_unwind_info(&mut self, code_len: usize) -> Option<UnwindInstructions> {
6312 let mut instructions = vec![];
6313 for &(instruction_offset, ref inst) in &self.unwind_ops {
6314 let instruction_offset = instruction_offset as u32;
6315 match *inst {
6316 UnwindOps::PushFP { up_to_sp } => {
6317 instructions.push((
6318 instruction_offset,
6319 CallFrameInstruction::CfaOffset(up_to_sp as i32),
6320 ));
6321 instructions.push((
6322 instruction_offset,
6323 CallFrameInstruction::Offset(RiscV::X8, -(up_to_sp as i32)),
6325 ));
6326 }
6327 UnwindOps::DefineNewFrame => {
6328 instructions.push((
6329 instruction_offset,
6330 CallFrameInstruction::CfaRegister(RiscV::X8),
6331 ));
6332 }
6333 UnwindOps::SaveRegister { reg, bp_neg_offset } => instructions.push((
6334 instruction_offset,
6335 CallFrameInstruction::Offset(reg.dwarf_index(), -bp_neg_offset),
6336 )),
6337 UnwindOps::SubtractFP { up_to_sp } => {
6338 instructions.push((
6339 instruction_offset,
6340 CallFrameInstruction::CfaOffset(up_to_sp as i32),
6341 ));
6342 }
6343 UnwindOps::Push2Regs { .. } => unimplemented!(),
6344 }
6345 }
6346 Some(UnwindInstructions {
6347 instructions,
6348 len: code_len as u32,
6349 })
6350 }
6351 #[cfg(not(feature = "unwind"))]
6352 fn gen_dwarf_unwind_info(&mut self, _code_len: usize) -> Option<UnwindInstructions> {
6353 None
6354 }
6355 fn gen_windows_unwind_info(&mut self, _code_len: usize) -> Option<Vec<u8>> {
6356 None
6357 }
6358}