1#[cfg(feature = "unwind")]
2use crate::unwind_winx64::create_unwind_info_from_insts;
3use crate::{
4 codegen_error,
5 common_decl::*,
6 emitter_x64::*,
7 location::{Location as AbstractLocation, Reg},
8 machine::*,
9 output_reporter::ChunkedOutputReporter,
10 unwind::{UnwindInstructions, UnwindOps, UnwindRegister},
11 x64_decl::{ArgumentRegisterAllocator, GPR, X64Register, XMM},
12};
13use dynasmrt::{DynasmError, VecAssembler, x64::X64Relocation};
14use fixedbitset::FixedBitSet;
15#[cfg(feature = "unwind")]
16use gimli::{X86_64, write::CallFrameInstruction};
17use std::{
18 collections::HashMap,
19 ops::{Deref, DerefMut},
20};
21use wasmer_compiler::{
22 types::{
23 address_map::InstructionAddressMap,
24 function::FunctionBody,
25 relocation::{Relocation, RelocationKind, RelocationTarget},
26 section::{CustomSection, CustomSectionProtection, SectionBody},
27 },
28 wasmparser::MemArg,
29};
30use wasmer_types::{
31 CompilationProgressCallback, CompileError, FunctionIndex, FunctionType, SourceLoc, TrapCode,
32 TrapInformation, Type, VMOffsets,
33 target::{CallingConvention, CpuFeature, Target},
34 vmctx_offset,
35};
36
37type Assembler = VecAssembler<X64Relocation>;
38
39pub struct AssemblerX64 {
40 pub inner: Assembler,
42 pub simd_arch: CpuFeature,
45 pub target: Option<Target>,
47}
48
49impl AssemblerX64 {
50 fn new(baseaddr: usize, target: Option<Target>) -> Result<Self, CompileError> {
51 let simd_arch = target.as_ref().map_or_else(
52 || Ok(CpuFeature::SSE42),
53 |target| {
54 if target.cpu_features().contains(CpuFeature::AVX) {
55 Ok(CpuFeature::AVX)
56 } else if target.cpu_features().contains(CpuFeature::SSE42) {
57 Ok(CpuFeature::SSE42)
58 } else {
59 Err(CompileError::UnsupportedTarget(
60 "x86_64 without AVX or SSE 4.2, use -m avx to enable".to_string(),
61 ))
62 }
63 },
64 )?;
65
66 Ok(Self {
67 inner: Assembler::new(baseaddr),
68 simd_arch,
69 target,
70 })
71 }
72
73 fn finalize(self) -> Result<Vec<u8>, DynasmError> {
74 self.inner.finalize()
75 }
76}
77
78impl Deref for AssemblerX64 {
79 type Target = Assembler;
80
81 fn deref(&self) -> &Self::Target {
82 &self.inner
83 }
84}
85
86impl DerefMut for AssemblerX64 {
87 fn deref_mut(&mut self) -> &mut Self::Target {
88 &mut self.inner
89 }
90}
91
92type Location = AbstractLocation<GPR, XMM>;
93
94pub struct MachineX86_64 {
95 assembler: AssemblerX64,
96 used_gprs: FixedBitSet,
97 used_simd: FixedBitSet,
98 trap_table: TrapTable,
99 instructions_address_map: Vec<InstructionAddressMap>,
103 src_loc: u32,
105 unwind_ops: Vec<(usize, UnwindOps<GPR, XMM>)>,
107}
108
109pub(crate) const X86_64_RETURN_VALUE_REGISTERS: [GPR; 2] = [GPR::RAX, GPR::RDX];
112
113const DIV_REM_REGISTERS: [GPR; 2] = [GPR::RAX, GPR::RDX];
115
116impl MachineX86_64 {
117 pub fn new(target: Option<Target>) -> Result<Self, CompileError> {
118 let assembler = AssemblerX64::new(0, target)?;
119 Ok(MachineX86_64 {
120 assembler,
121 used_gprs: FixedBitSet::with_capacity(16),
122 used_simd: FixedBitSet::with_capacity(16),
123 trap_table: TrapTable::default(),
124 instructions_address_map: vec![],
125 src_loc: 0,
126 unwind_ops: vec![],
127 })
128 }
129 pub fn emit_relaxed_binop(
130 &mut self,
131 op: fn(&mut AssemblerX64, Size, Location, Location) -> Result<(), CompileError>,
132 sz: Size,
133 src: Location,
134 dst: Location,
135 ) -> Result<(), CompileError> {
136 enum RelaxMode {
137 Direct,
138 SrcToGPR,
139 DstToGPR,
140 BothToGPR,
141 }
142 let mode = match (src, dst) {
143 (Location::GPR(_), Location::GPR(_))
144 if std::ptr::eq(op as *const u8, AssemblerX64::emit_imul as *const u8) =>
145 {
146 RelaxMode::Direct
147 }
148 _ if std::ptr::eq(op as *const u8, AssemblerX64::emit_imul as *const u8) => {
149 RelaxMode::BothToGPR
150 }
151
152 (Location::Memory(_, _), Location::Memory(_, _)) => RelaxMode::SrcToGPR,
153 (Location::Imm64(_), Location::Imm64(_)) | (Location::Imm64(_), Location::Imm32(_)) => {
154 RelaxMode::BothToGPR
155 }
156 (_, Location::Imm32(_)) | (_, Location::Imm64(_)) => RelaxMode::DstToGPR,
157 (Location::Imm64(_), Location::Memory(_, _)) => RelaxMode::SrcToGPR,
158 (Location::Imm64(_), Location::GPR(_))
159 if (op as *const u8 != AssemblerX64::emit_mov as *const u8) =>
160 {
161 RelaxMode::SrcToGPR
162 }
163 (_, Location::SIMD(_)) => RelaxMode::SrcToGPR,
164 _ => RelaxMode::Direct,
165 };
166
167 match mode {
168 RelaxMode::SrcToGPR => {
169 let temp = self.acquire_temp_gpr().ok_or_else(|| {
170 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
171 })?;
172 self.move_location(sz, src, Location::GPR(temp))?;
173 op(&mut self.assembler, sz, Location::GPR(temp), dst)?;
174 self.release_gpr(temp);
175 }
176 RelaxMode::DstToGPR => {
177 let temp = self.acquire_temp_gpr().ok_or_else(|| {
178 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
179 })?;
180 self.move_location(sz, dst, Location::GPR(temp))?;
181 op(&mut self.assembler, sz, src, Location::GPR(temp))?;
182 self.release_gpr(temp);
183 }
184 RelaxMode::BothToGPR => {
185 let temp_src = self.acquire_temp_gpr().ok_or_else(|| {
186 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
187 })?;
188 let temp_dst = self.acquire_temp_gpr().ok_or_else(|| {
189 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
190 })?;
191 self.move_location(sz, src, Location::GPR(temp_src))?;
192 self.move_location(sz, dst, Location::GPR(temp_dst))?;
193 op(
194 &mut self.assembler,
195 sz,
196 Location::GPR(temp_src),
197 Location::GPR(temp_dst),
198 )?;
199 match dst {
200 Location::Memory(_, _) | Location::GPR(_) => {
201 self.move_location(sz, Location::GPR(temp_dst), dst)?;
202 }
203 _ => {}
204 }
205 self.release_gpr(temp_dst);
206 self.release_gpr(temp_src);
207 }
208 RelaxMode::Direct => {
209 op(&mut self.assembler, sz, src, dst)?;
210 }
211 }
212 Ok(())
213 }
214 pub fn emit_relaxed_zx_sx(
215 &mut self,
216 op: fn(&mut AssemblerX64, Size, Location, Size, Location) -> Result<(), CompileError>,
217 sz_src: Size,
218 src: Location,
219 sz_dst: Size,
220 dst: Location,
221 ) -> Result<(), CompileError> {
222 match src {
223 Location::Imm32(_) | Location::Imm64(_) => {
224 let tmp_src = self.acquire_temp_gpr().ok_or_else(|| {
225 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
226 })?;
227 self.assembler
228 .emit_mov(Size::S64, src, Location::GPR(tmp_src))?;
229 let src = Location::GPR(tmp_src);
230
231 match dst {
232 Location::Imm32(_) | Location::Imm64(_) => unreachable!(),
233 Location::Memory(_, _) => {
234 let tmp_dst = self.acquire_temp_gpr().ok_or_else(|| {
235 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
236 })?;
237 op(
238 &mut self.assembler,
239 sz_src,
240 src,
241 sz_dst,
242 Location::GPR(tmp_dst),
243 )?;
244 self.move_location(Size::S64, Location::GPR(tmp_dst), dst)?;
245
246 self.release_gpr(tmp_dst);
247 }
248 Location::GPR(_) => {
249 op(&mut self.assembler, sz_src, src, sz_dst, dst)?;
250 }
251 _ => {
252 codegen_error!("singlepass emit_relaxed_zx_sx unreachable");
253 }
254 };
255
256 self.release_gpr(tmp_src);
257 }
258 Location::GPR(_) | Location::Memory(_, _) => {
259 match dst {
260 Location::Imm32(_) | Location::Imm64(_) => unreachable!(),
261 Location::Memory(_, _) => {
262 let tmp_dst = self.acquire_temp_gpr().ok_or_else(|| {
263 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
264 })?;
265 op(
266 &mut self.assembler,
267 sz_src,
268 src,
269 sz_dst,
270 Location::GPR(tmp_dst),
271 )?;
272 self.move_location(Size::S64, Location::GPR(tmp_dst), dst)?;
273
274 self.release_gpr(tmp_dst);
275 }
276 Location::GPR(_) => {
277 op(&mut self.assembler, sz_src, src, sz_dst, dst)?;
278 }
279 _ => {
280 codegen_error!("singlepass emit_relaxed_zx_sx unreachable");
281 }
282 };
283 }
284 _ => {
285 codegen_error!("singlepass emit_relaxed_zx_sx unreachable");
286 }
287 }
288 Ok(())
289 }
290 fn emit_binop_i32(
292 &mut self,
293 f: fn(&mut AssemblerX64, Size, Location, Location) -> Result<(), CompileError>,
294 loc_a: Location,
295 loc_b: Location,
296 ret: Location,
297 ) -> Result<(), CompileError> {
298 if loc_a != ret {
299 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
300 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
301 })?;
302 self.emit_relaxed_mov(Size::S32, loc_a, Location::GPR(tmp))?;
303 self.emit_relaxed_binop(f, Size::S32, loc_b, Location::GPR(tmp))?;
304 self.emit_relaxed_mov(Size::S32, Location::GPR(tmp), ret)?;
305 self.release_gpr(tmp);
306 } else {
307 self.emit_relaxed_binop(f, Size::S32, loc_b, ret)?;
308 }
309 Ok(())
310 }
311 fn emit_binop_i64(
313 &mut self,
314 f: fn(&mut AssemblerX64, Size, Location, Location) -> Result<(), CompileError>,
315 loc_a: Location,
316 loc_b: Location,
317 ret: Location,
318 ) -> Result<(), CompileError> {
319 if loc_a != ret {
320 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
321 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
322 })?;
323 self.emit_relaxed_mov(Size::S64, loc_a, Location::GPR(tmp))?;
324 self.emit_relaxed_binop(f, Size::S64, loc_b, Location::GPR(tmp))?;
325 self.emit_relaxed_mov(Size::S64, Location::GPR(tmp), ret)?;
326 self.release_gpr(tmp);
327 } else {
328 self.emit_relaxed_binop(f, Size::S64, loc_b, ret)?;
329 }
330 Ok(())
331 }
332 fn emit_cmpop_i64_dynamic_b(
334 &mut self,
335 c: Condition,
336 loc_a: Location,
337 loc_b: Location,
338 ret: Location,
339 ) -> Result<(), CompileError> {
340 match ret {
341 Location::GPR(x) => {
342 self.emit_relaxed_cmp(Size::S64, loc_b, loc_a)?;
343 self.assembler.emit_set(c, x)?;
344 self.assembler
345 .emit_and(Size::S32, Location::Imm32(0xff), Location::GPR(x))?;
346 }
347 Location::Memory(_, _) => {
348 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
349 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
350 })?;
351 self.emit_relaxed_cmp(Size::S64, loc_b, loc_a)?;
352 self.assembler.emit_set(c, tmp)?;
353 self.assembler
354 .emit_and(Size::S32, Location::Imm32(0xff), Location::GPR(tmp))?;
355 self.move_location(Size::S32, Location::GPR(tmp), ret)?;
356 self.release_gpr(tmp);
357 }
358 _ => {
359 codegen_error!("singlepass emit_cmpop_i64_dynamic_b unreachable");
360 }
361 }
362 Ok(())
363 }
364 fn emit_shift_i64(
366 &mut self,
367 f: fn(&mut AssemblerX64, Size, Location, Location) -> Result<(), CompileError>,
368 loc_a: Location,
369 loc_b: Location,
370 ret: Location,
371 ) -> Result<(), CompileError> {
372 self.assembler
373 .emit_mov(Size::S64, loc_b, Location::GPR(GPR::RCX))?;
374
375 if loc_a != ret {
376 self.emit_relaxed_mov(Size::S64, loc_a, ret)?;
377 }
378
379 f(&mut self.assembler, Size::S64, Location::GPR(GPR::RCX), ret)
380 }
381 fn emit_relaxed_xdiv(
383 &mut self,
384 op: fn(&mut AssemblerX64, Size, Location) -> Result<(), CompileError>,
385 sz: Size,
386 loc: Location,
387 integer_division_by_zero: Label,
388 ) -> Result<usize, CompileError> {
389 self.emit_relaxed_cmp(sz, Location::Imm32(0), loc)?;
390 self.assembler
391 .emit_jmp(Condition::Equal, integer_division_by_zero)?;
392
393 match loc {
394 Location::Imm64(_) | Location::Imm32(_) => {
395 self.move_location(sz, loc, Location::GPR(GPR::RCX))?; let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
397 op(&mut self.assembler, sz, Location::GPR(GPR::RCX))?;
398 self.mark_instruction_address_end(offset);
399 Ok(offset)
400 }
401 _ => {
402 let offset = self.mark_instruction_with_trap_code(TrapCode::IntegerOverflow);
403 op(&mut self.assembler, sz, loc)?;
404 self.mark_instruction_address_end(offset);
405 Ok(offset)
406 }
407 }
408 }
409 fn emit_cmpop_i32_dynamic_b(
411 &mut self,
412 c: Condition,
413 loc_a: Location,
414 loc_b: Location,
415 ret: Location,
416 ) -> Result<(), CompileError> {
417 match ret {
418 Location::GPR(x) => {
419 self.emit_relaxed_cmp(Size::S32, loc_b, loc_a)?;
420 self.assembler.emit_set(c, x)?;
421 self.assembler
422 .emit_and(Size::S32, Location::Imm32(0xff), Location::GPR(x))?;
423 }
424 Location::Memory(_, _) => {
425 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
426 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
427 })?;
428 self.emit_relaxed_cmp(Size::S32, loc_b, loc_a)?;
429 self.assembler.emit_set(c, tmp)?;
430 self.assembler
431 .emit_and(Size::S32, Location::Imm32(0xff), Location::GPR(tmp))?;
432 self.move_location(Size::S32, Location::GPR(tmp), ret)?;
433 self.release_gpr(tmp);
434 }
435 _ => {
436 codegen_error!("singlepass emit_cmpop_i32_dynamic_b unreachable");
437 }
438 }
439 Ok(())
440 }
441 fn emit_shift_i32(
443 &mut self,
444 f: fn(&mut AssemblerX64, Size, Location, Location) -> Result<(), CompileError>,
445 loc_a: Location,
446 loc_b: Location,
447 ret: Location,
448 ) -> Result<(), CompileError> {
449 self.assembler
450 .emit_mov(Size::S32, loc_b, Location::GPR(GPR::RCX))?;
451
452 if loc_a != ret {
453 self.emit_relaxed_mov(Size::S32, loc_a, ret)?;
454 }
455
456 f(&mut self.assembler, Size::S32, Location::GPR(GPR::RCX), ret)
457 }
458
459 #[allow(clippy::too_many_arguments)]
460 fn memory_op<F: FnOnce(&mut Self, GPR) -> Result<(), CompileError>>(
461 &mut self,
462 addr: Location,
463 memarg: &MemArg,
464 check_alignment: bool,
465 value_size: usize,
466 need_check: bool,
467 imported_memories: bool,
468 offset: i32,
469 heap_access_oob: Label,
470 unaligned_atomic: Label,
471 cb: F,
472 ) -> Result<(), CompileError> {
473 let tmp_addr = self.acquire_temp_gpr().ok_or_else(|| {
479 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
480 })?;
481 let tmp2 = self.acquire_temp_gpr().ok_or_else(|| {
482 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
483 })?;
484
485 let base_loc = if imported_memories {
487 self.emit_relaxed_binop(
489 AssemblerX64::emit_mov,
490 Size::S64,
491 Location::Memory(self.get_vmctx_reg(), offset),
492 Location::GPR(tmp2),
493 )?;
494 Location::Memory(tmp2, 0)
495 } else {
496 Location::Memory(self.get_vmctx_reg(), offset)
497 };
498
499 self.assembler
501 .emit_mov(Size::S64, base_loc, Location::GPR(tmp2))?;
502
503 self.assembler
507 .emit_mov(Size::S32, addr, Location::GPR(tmp_addr))?;
508
509 if memarg.offset != 0 {
511 self.assembler.emit_add(
512 Size::S32,
513 Location::Imm32(memarg.offset as u32),
514 Location::GPR(tmp_addr),
515 )?;
516
517 self.assembler.emit_jmp(Condition::Carry, heap_access_oob)?;
519 }
520
521 if need_check {
522 let bound_loc = if imported_memories {
523 self.emit_relaxed_binop(
525 AssemblerX64::emit_mov,
526 Size::S64,
527 Location::Memory(self.get_vmctx_reg(), offset),
528 Location::GPR(tmp2),
529 )?;
530 Location::Memory(tmp2, 8)
531 } else {
532 Location::Memory(self.get_vmctx_reg(), offset + 8)
533 };
534 self.assembler
535 .emit_mov(Size::S64, bound_loc, Location::GPR(tmp2))?;
536
537 self.assembler.emit_lea(
539 Size::S64,
540 Location::Memory(tmp2, -(value_size as i32)),
541 Location::GPR(tmp2),
542 )?;
543 self.emit_relaxed_cmp(Size::S64, Location::GPR(tmp2), Location::GPR(tmp_addr))?;
545
546 self.assembler.emit_jmp(Condition::Above, heap_access_oob)?;
548 }
549 let base_loc = if imported_memories {
551 self.emit_relaxed_binop(
553 AssemblerX64::emit_mov,
554 Size::S64,
555 Location::Memory(self.get_vmctx_reg(), offset),
556 Location::GPR(tmp2),
557 )?;
558 Location::Memory(tmp2, 0)
559 } else {
560 Location::Memory(self.get_vmctx_reg(), offset)
561 };
562 self.assembler
564 .emit_add(Size::S64, base_loc, Location::GPR(tmp_addr))?;
565
566 self.release_gpr(tmp2);
567
568 let align = value_size as u32;
569 if check_alignment && align != 1 {
570 let tmp_aligncheck = self.acquire_temp_gpr().ok_or_else(|| {
571 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
572 })?;
573 self.assembler.emit_mov(
574 Size::S32,
575 Location::GPR(tmp_addr),
576 Location::GPR(tmp_aligncheck),
577 )?;
578 self.assembler.emit_and(
579 Size::S64,
580 Location::Imm32(align - 1),
581 Location::GPR(tmp_aligncheck),
582 )?;
583 self.assembler
584 .emit_jmp(Condition::NotEqual, unaligned_atomic)?;
585 self.release_gpr(tmp_aligncheck);
586 }
587 let begin = self.assembler.get_offset().0;
588 cb(self, tmp_addr)?;
589 let end = self.assembler.get_offset().0;
590 self.mark_address_range_with_trap_code(TrapCode::HeapAccessOutOfBounds, begin, end);
591
592 self.release_gpr(tmp_addr);
593 Ok(())
594 }
595
596 #[allow(clippy::too_many_arguments)]
597 fn emit_compare_and_swap<F: FnOnce(&mut Self, GPR, GPR) -> Result<(), CompileError>>(
598 &mut self,
599 loc: Location,
600 target: Location,
601 ret: Location,
602 memarg: &MemArg,
603 value_size: usize,
604 memory_sz: Size,
605 stack_sz: Size,
606 need_check: bool,
607 imported_memories: bool,
608 offset: i32,
609 heap_access_oob: Label,
610 unaligned_atomic: Label,
611 cb: F,
612 ) -> Result<(), CompileError> {
613 if memory_sz > stack_sz {
614 codegen_error!("singlepass emit_compare_and_swap unreachable");
615 }
616
617 let compare = self.reserve_unused_temp_gpr(GPR::RAX);
618 let value = if loc == Location::GPR(GPR::R14) {
619 GPR::R13
620 } else {
621 GPR::R14
622 };
623 self.assembler.emit_push(Size::S64, Location::GPR(value))?;
624
625 self.move_location(stack_sz, loc, Location::GPR(value))?;
626
627 let retry = self.assembler.get_label();
628 self.emit_label(retry)?;
629
630 self.memory_op(
631 target,
632 memarg,
633 true,
634 value_size,
635 need_check,
636 imported_memories,
637 offset,
638 heap_access_oob,
639 unaligned_atomic,
640 |this, addr| {
641 this.load_address(memory_sz, Location::GPR(compare), Location::Memory(addr, 0))?;
642 this.move_location(stack_sz, Location::GPR(compare), ret)?;
643 cb(this, compare, value)?;
644 this.assembler.emit_lock_cmpxchg(
645 memory_sz,
646 Location::GPR(value),
647 Location::Memory(addr, 0),
648 )
649 },
650 )?;
651
652 self.assembler.emit_jmp(Condition::NotEqual, retry)?;
653
654 self.assembler.emit_pop(Size::S64, Location::GPR(value))?;
655 self.release_gpr(compare);
656 Ok(())
657 }
658
659 #[allow(clippy::too_many_arguments)]
661 fn emit_f32_int_conv_check(
662 &mut self,
663 reg: XMM,
664 lower_bound: f32,
665 upper_bound: f32,
666 underflow_label: Label,
667 overflow_label: Label,
668 nan_label: Label,
669 succeed_label: Label,
670 ) -> Result<(), CompileError> {
671 let lower_bound = f32::to_bits(lower_bound);
672 let upper_bound = f32::to_bits(upper_bound);
673
674 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
675 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
676 })?;
677 let tmp_x = self.acquire_temp_simd().ok_or_else(|| {
678 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
679 })?;
680
681 self.move_location(Size::S32, Location::Imm32(lower_bound), Location::GPR(tmp))?;
683 self.move_location(Size::S32, Location::GPR(tmp), Location::SIMD(tmp_x))?;
684 self.assembler
685 .emit_vcmpless(reg, XMMOrMemory::XMM(tmp_x), tmp_x)?;
686 self.move_location(Size::S32, Location::SIMD(tmp_x), Location::GPR(tmp))?;
687 self.emit_relaxed_cmp(Size::S32, Location::Imm32(0), Location::GPR(tmp))?;
688 self.assembler
689 .emit_jmp(Condition::NotEqual, underflow_label)?;
690
691 self.move_location(Size::S32, Location::Imm32(upper_bound), Location::GPR(tmp))?;
693 self.move_location(Size::S32, Location::GPR(tmp), Location::SIMD(tmp_x))?;
694 self.assembler
695 .emit_vcmpgess(reg, XMMOrMemory::XMM(tmp_x), tmp_x)?;
696 self.move_location(Size::S32, Location::SIMD(tmp_x), Location::GPR(tmp))?;
697 self.emit_relaxed_cmp(Size::S32, Location::Imm32(0), Location::GPR(tmp))?;
698 self.assembler
699 .emit_jmp(Condition::NotEqual, overflow_label)?;
700
701 self.assembler
703 .emit_vcmpeqss(reg, XMMOrMemory::XMM(reg), tmp_x)?;
704 self.move_location(Size::S32, Location::SIMD(tmp_x), Location::GPR(tmp))?;
705 self.emit_relaxed_cmp(Size::S32, Location::Imm32(0), Location::GPR(tmp))?;
706 self.assembler.emit_jmp(Condition::Equal, nan_label)?;
707
708 self.assembler.emit_jmp(Condition::None, succeed_label)?;
709
710 self.release_simd(tmp_x);
711 self.release_gpr(tmp);
712 Ok(())
713 }
714
715 fn emit_f32_int_conv_check_trap(
717 &mut self,
718 reg: XMM,
719 lower_bound: f32,
720 upper_bound: f32,
721 ) -> Result<(), CompileError> {
722 let trap_overflow = self.assembler.get_label();
723 let trap_badconv = self.assembler.get_label();
724 let end = self.assembler.get_label();
725
726 self.emit_f32_int_conv_check(
727 reg,
728 lower_bound,
729 upper_bound,
730 trap_overflow,
731 trap_overflow,
732 trap_badconv,
733 end,
734 )?;
735
736 self.emit_label(trap_overflow)?;
737
738 self.emit_illegal_op_internal(TrapCode::IntegerOverflow)?;
739
740 self.emit_label(trap_badconv)?;
741
742 self.emit_illegal_op_internal(TrapCode::BadConversionToInteger)?;
743
744 self.emit_label(end)?;
745 Ok(())
746 }
747 #[allow(clippy::too_many_arguments)]
748 fn emit_f32_int_conv_check_sat<
749 F1: FnOnce(&mut Self) -> Result<(), CompileError>,
750 F2: FnOnce(&mut Self) -> Result<(), CompileError>,
751 F3: FnOnce(&mut Self) -> Result<(), CompileError>,
752 F4: FnOnce(&mut Self) -> Result<(), CompileError>,
753 >(
754 &mut self,
755 reg: XMM,
756 lower_bound: f32,
757 upper_bound: f32,
758 underflow_cb: F1,
759 overflow_cb: F2,
760 nan_cb: Option<F3>,
761 convert_cb: F4,
762 ) -> Result<(), CompileError> {
763 let underflow = self.assembler.get_label();
768 let overflow = self.assembler.get_label();
769 let nan = if nan_cb.is_some() {
770 self.assembler.get_label()
771 } else {
772 underflow
773 };
774 let convert = self.assembler.get_label();
775 let end = self.assembler.get_label();
776
777 self.emit_f32_int_conv_check(
778 reg,
779 lower_bound,
780 upper_bound,
781 underflow,
782 overflow,
783 nan,
784 convert,
785 )?;
786
787 self.emit_label(underflow)?;
788 underflow_cb(self)?;
789 self.assembler.emit_jmp(Condition::None, end)?;
790
791 self.emit_label(overflow)?;
792 overflow_cb(self)?;
793 self.assembler.emit_jmp(Condition::None, end)?;
794
795 if let Some(cb) = nan_cb {
796 self.emit_label(nan)?;
797 cb(self)?;
798 self.assembler.emit_jmp(Condition::None, end)?;
799 }
800
801 self.emit_label(convert)?;
802 convert_cb(self)?;
803 self.emit_label(end)
804 }
805 #[allow(clippy::too_many_arguments)]
807 fn emit_f64_int_conv_check(
808 &mut self,
809 reg: XMM,
810 lower_bound: f64,
811 upper_bound: f64,
812 underflow_label: Label,
813 overflow_label: Label,
814 nan_label: Label,
815 succeed_label: Label,
816 ) -> Result<(), CompileError> {
817 let lower_bound = f64::to_bits(lower_bound);
818 let upper_bound = f64::to_bits(upper_bound);
819
820 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
821 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
822 })?;
823 let tmp_x = self.acquire_temp_simd().ok_or_else(|| {
824 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
825 })?;
826
827 self.move_location(Size::S64, Location::Imm64(lower_bound), Location::GPR(tmp))?;
829 self.move_location(Size::S64, Location::GPR(tmp), Location::SIMD(tmp_x))?;
830 self.assembler
831 .emit_vcmplesd(reg, XMMOrMemory::XMM(tmp_x), tmp_x)?;
832 self.move_location(Size::S32, Location::SIMD(tmp_x), Location::GPR(tmp))?;
833 self.emit_relaxed_cmp(Size::S32, Location::Imm32(0), Location::GPR(tmp))?;
834 self.assembler
835 .emit_jmp(Condition::NotEqual, underflow_label)?;
836
837 self.move_location(Size::S64, Location::Imm64(upper_bound), Location::GPR(tmp))?;
839 self.move_location(Size::S64, Location::GPR(tmp), Location::SIMD(tmp_x))?;
840 self.assembler
841 .emit_vcmpgesd(reg, XMMOrMemory::XMM(tmp_x), tmp_x)?;
842 self.move_location(Size::S32, Location::SIMD(tmp_x), Location::GPR(tmp))?;
843 self.emit_relaxed_cmp(Size::S32, Location::Imm32(0), Location::GPR(tmp))?;
844 self.assembler
845 .emit_jmp(Condition::NotEqual, overflow_label)?;
846
847 self.assembler
849 .emit_vcmpeqsd(reg, XMMOrMemory::XMM(reg), tmp_x)?;
850 self.move_location(Size::S32, Location::SIMD(tmp_x), Location::GPR(tmp))?;
851 self.emit_relaxed_cmp(Size::S32, Location::Imm32(0), Location::GPR(tmp))?;
852 self.assembler.emit_jmp(Condition::Equal, nan_label)?;
853
854 self.assembler.emit_jmp(Condition::None, succeed_label)?;
855
856 self.release_simd(tmp_x);
857 self.release_gpr(tmp);
858 Ok(())
859 }
860 fn emit_f64_int_conv_check_trap(
862 &mut self,
863 reg: XMM,
864 lower_bound: f64,
865 upper_bound: f64,
866 ) -> Result<(), CompileError> {
867 let trap_overflow = self.assembler.get_label();
868 let trap_badconv = self.assembler.get_label();
869 let end = self.assembler.get_label();
870
871 self.emit_f64_int_conv_check(
872 reg,
873 lower_bound,
874 upper_bound,
875 trap_overflow,
876 trap_overflow,
877 trap_badconv,
878 end,
879 )?;
880
881 self.emit_label(trap_overflow)?;
882 self.emit_illegal_op_internal(TrapCode::IntegerOverflow)?;
883
884 self.emit_label(trap_badconv)?;
885 self.emit_illegal_op_internal(TrapCode::BadConversionToInteger)?;
886
887 self.emit_label(end)
888 }
889 #[allow(clippy::too_many_arguments)]
890 fn emit_f64_int_conv_check_sat<
891 F1: FnOnce(&mut Self) -> Result<(), CompileError>,
892 F2: FnOnce(&mut Self) -> Result<(), CompileError>,
893 F3: FnOnce(&mut Self) -> Result<(), CompileError>,
894 F4: FnOnce(&mut Self) -> Result<(), CompileError>,
895 >(
896 &mut self,
897 reg: XMM,
898 lower_bound: f64,
899 upper_bound: f64,
900 underflow_cb: F1,
901 overflow_cb: F2,
902 nan_cb: Option<F3>,
903 convert_cb: F4,
904 ) -> Result<(), CompileError> {
905 let underflow = self.assembler.get_label();
910 let overflow = self.assembler.get_label();
911 let nan = if nan_cb.is_some() {
912 self.assembler.get_label()
913 } else {
914 underflow
915 };
916 let convert = self.assembler.get_label();
917 let end = self.assembler.get_label();
918
919 self.emit_f64_int_conv_check(
920 reg,
921 lower_bound,
922 upper_bound,
923 underflow,
924 overflow,
925 nan,
926 convert,
927 )?;
928
929 self.emit_label(underflow)?;
930 underflow_cb(self)?;
931 self.assembler.emit_jmp(Condition::None, end)?;
932
933 self.emit_label(overflow)?;
934 overflow_cb(self)?;
935 self.assembler.emit_jmp(Condition::None, end)?;
936
937 if let Some(cb) = nan_cb {
938 self.emit_label(nan)?;
939 cb(self)?;
940 self.assembler.emit_jmp(Condition::None, end)?;
941 }
942
943 self.emit_label(convert)?;
944 convert_cb(self)?;
945 self.emit_label(end)
946 }
947 fn emit_relaxed_avx(
949 &mut self,
950 op: fn(&mut AssemblerX64, XMM, XMMOrMemory, XMM) -> Result<(), CompileError>,
951 src1: Location,
952 src2: Location,
953 dst: Location,
954 ) -> Result<(), CompileError> {
955 self.emit_relaxed_avx_base(
956 |this, src1, src2, dst| op(&mut this.assembler, src1, src2, dst),
957 src1,
958 src2,
959 dst,
960 )
961 }
962
963 fn emit_relaxed_avx_base<
965 F: FnOnce(&mut Self, XMM, XMMOrMemory, XMM) -> Result<(), CompileError>,
966 >(
967 &mut self,
968 op: F,
969 src1: Location,
970 src2: Location,
971 dst: Location,
972 ) -> Result<(), CompileError> {
973 let tmp1 = self.acquire_temp_simd().ok_or_else(|| {
974 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
975 })?;
976 let tmp2 = self.acquire_temp_simd().ok_or_else(|| {
977 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
978 })?;
979 let tmp3 = self.acquire_temp_simd().ok_or_else(|| {
980 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
981 })?;
982 let tmpg = self.acquire_temp_gpr().ok_or_else(|| {
983 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
984 })?;
985
986 let src1 = match src1 {
987 Location::SIMD(x) => x,
988 Location::GPR(_) | Location::Memory(_, _) => {
989 self.assembler
990 .emit_mov(Size::S64, src1, Location::SIMD(tmp1))?;
991 tmp1
992 }
993 Location::Imm32(_) => {
994 self.assembler
995 .emit_mov(Size::S32, src1, Location::GPR(tmpg))?;
996 self.move_location(Size::S32, Location::GPR(tmpg), Location::SIMD(tmp1))?;
997 tmp1
998 }
999 Location::Imm64(_) => {
1000 self.assembler
1001 .emit_mov(Size::S64, src1, Location::GPR(tmpg))?;
1002 self.move_location(Size::S64, Location::GPR(tmpg), Location::SIMD(tmp1))?;
1003 tmp1
1004 }
1005 _ => {
1006 codegen_error!("singlepass emit_relaxed_avx_base unreachable")
1007 }
1008 };
1009
1010 let src2 = match src2 {
1011 Location::SIMD(x) => XMMOrMemory::XMM(x),
1012 Location::Memory(base, disp) => XMMOrMemory::Memory(base, disp),
1013 Location::GPR(_) => {
1014 self.assembler
1015 .emit_mov(Size::S64, src2, Location::SIMD(tmp2))?;
1016 XMMOrMemory::XMM(tmp2)
1017 }
1018 Location::Imm32(_) => {
1019 self.assembler
1020 .emit_mov(Size::S32, src2, Location::GPR(tmpg))?;
1021 self.move_location(Size::S32, Location::GPR(tmpg), Location::SIMD(tmp2))?;
1022 XMMOrMemory::XMM(tmp2)
1023 }
1024 Location::Imm64(_) => {
1025 self.assembler
1026 .emit_mov(Size::S64, src2, Location::GPR(tmpg))?;
1027 self.move_location(Size::S64, Location::GPR(tmpg), Location::SIMD(tmp2))?;
1028 XMMOrMemory::XMM(tmp2)
1029 }
1030 _ => {
1031 codegen_error!("singlepass emit_relaxed_avx_base unreachable")
1032 }
1033 };
1034
1035 match dst {
1036 Location::SIMD(x) => {
1037 op(self, src1, src2, x)?;
1038 }
1039 Location::Memory(_, _) | Location::GPR(_) => {
1040 op(self, src1, src2, tmp3)?;
1041 self.assembler
1042 .emit_mov(Size::S64, Location::SIMD(tmp3), dst)?;
1043 }
1044 _ => {
1045 codegen_error!("singlepass emit_relaxed_avx_base unreachable")
1046 }
1047 }
1048
1049 self.release_gpr(tmpg);
1050 self.release_simd(tmp3);
1051 self.release_simd(tmp2);
1052 self.release_simd(tmp1);
1053 Ok(())
1054 }
1055
1056 fn convert_i64_f64_u_s(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
1057 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1058 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1059 })?;
1060 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1061 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1062 })?;
1063
1064 self.emit_relaxed_mov(Size::S64, loc, Location::SIMD(tmp_in))?;
1065 self.emit_f64_int_conv_check_sat(
1066 tmp_in,
1067 GEF64_LT_U64_MIN,
1068 LEF64_GT_U64_MAX,
1069 |this| {
1070 this.assembler
1071 .emit_mov(Size::S64, Location::Imm64(0), Location::GPR(tmp_out))
1072 },
1073 |this| {
1074 this.assembler.emit_mov(
1075 Size::S64,
1076 Location::Imm64(u64::MAX),
1077 Location::GPR(tmp_out),
1078 )
1079 },
1080 None::<fn(this: &mut Self) -> Result<(), CompileError>>,
1081 |this| {
1082 if this.assembler.arch_has_itruncf() {
1083 this.assembler.arch_emit_i64_trunc_uf64(tmp_in, tmp_out)
1084 } else {
1085 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
1086 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1087 })?;
1088 let tmp_x1 = this.acquire_temp_simd().ok_or_else(|| {
1089 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1090 })?;
1091 let tmp_x2 = this.acquire_temp_simd().ok_or_else(|| {
1092 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1093 })?;
1094
1095 this.assembler.emit_mov(
1096 Size::S64,
1097 Location::Imm64(4890909195324358656u64),
1098 Location::GPR(tmp),
1099 )?; this.assembler.emit_mov(
1101 Size::S64,
1102 Location::GPR(tmp),
1103 Location::SIMD(tmp_x1),
1104 )?;
1105 this.assembler.emit_mov(
1106 Size::S64,
1107 Location::SIMD(tmp_in),
1108 Location::SIMD(tmp_x2),
1109 )?;
1110 this.assembler
1111 .emit_vsubsd(tmp_in, XMMOrMemory::XMM(tmp_x1), tmp_in)?;
1112 this.assembler
1113 .emit_cvttsd2si_64(XMMOrMemory::XMM(tmp_in), tmp_out)?;
1114 this.assembler.emit_mov(
1115 Size::S64,
1116 Location::Imm64(0x8000000000000000u64),
1117 Location::GPR(tmp),
1118 )?;
1119 this.assembler.emit_xor(
1120 Size::S64,
1121 Location::GPR(tmp_out),
1122 Location::GPR(tmp),
1123 )?;
1124 this.assembler
1125 .emit_cvttsd2si_64(XMMOrMemory::XMM(tmp_x2), tmp_out)?;
1126 this.assembler
1127 .emit_ucomisd(XMMOrMemory::XMM(tmp_x1), tmp_x2)?;
1128 this.assembler.emit_cmovae_gpr_64(tmp, tmp_out)?;
1129
1130 this.release_simd(tmp_x2);
1131 this.release_simd(tmp_x1);
1132 this.release_gpr(tmp);
1133 Ok(())
1134 }
1135 },
1136 )?;
1137
1138 self.assembler
1139 .emit_mov(Size::S64, Location::GPR(tmp_out), ret)?;
1140 self.release_simd(tmp_in);
1141 self.release_gpr(tmp_out);
1142 Ok(())
1143 }
1144 fn convert_i64_f64_u_u(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
1145 if self.assembler.arch_has_itruncf() {
1146 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1147 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1148 })?;
1149 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1150 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1151 })?;
1152 self.emit_relaxed_mov(Size::S64, loc, Location::SIMD(tmp_in))?;
1153 self.assembler.arch_emit_i64_trunc_uf64(tmp_in, tmp_out)?;
1154 self.emit_relaxed_mov(Size::S64, Location::GPR(tmp_out), ret)?;
1155 self.release_simd(tmp_in);
1156 self.release_gpr(tmp_out);
1157 } else {
1158 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1159 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1160 })?;
1161 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1162 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1163 })?; self.emit_relaxed_mov(Size::S64, loc, Location::SIMD(tmp_in))?;
1166 self.emit_f64_int_conv_check_trap(tmp_in, GEF64_LT_U64_MIN, LEF64_GT_U64_MAX)?;
1167
1168 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
1169 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1170 })?; let tmp_x1 = self.acquire_temp_simd().ok_or_else(|| {
1172 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1173 })?; let tmp_x2 = self.acquire_temp_simd().ok_or_else(|| {
1175 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1176 })?; self.move_location(
1179 Size::S64,
1180 Location::Imm64(4890909195324358656u64),
1181 Location::GPR(tmp),
1182 )?; self.move_location(Size::S64, Location::GPR(tmp), Location::SIMD(tmp_x1))?;
1184 self.move_location(Size::S64, Location::SIMD(tmp_in), Location::SIMD(tmp_x2))?;
1185 self.assembler
1186 .emit_vsubsd(tmp_in, XMMOrMemory::XMM(tmp_x1), tmp_in)?;
1187 self.assembler
1188 .emit_cvttsd2si_64(XMMOrMemory::XMM(tmp_in), tmp_out)?;
1189 self.move_location(
1190 Size::S64,
1191 Location::Imm64(0x8000000000000000u64),
1192 Location::GPR(tmp),
1193 )?;
1194 self.assembler
1195 .emit_xor(Size::S64, Location::GPR(tmp_out), Location::GPR(tmp))?;
1196 self.assembler
1197 .emit_cvttsd2si_64(XMMOrMemory::XMM(tmp_x2), tmp_out)?;
1198 self.assembler
1199 .emit_ucomisd(XMMOrMemory::XMM(tmp_x1), tmp_x2)?;
1200 self.assembler.emit_cmovae_gpr_64(tmp, tmp_out)?;
1201 self.move_location(Size::S64, Location::GPR(tmp_out), ret)?;
1202
1203 self.release_simd(tmp_x2);
1204 self.release_simd(tmp_x1);
1205 self.release_gpr(tmp);
1206 self.release_simd(tmp_in);
1207 self.release_gpr(tmp_out);
1208 }
1209 Ok(())
1210 }
1211 fn convert_i64_f64_s_s(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
1212 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1213 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1214 })?;
1215 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1216 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1217 })?;
1218
1219 self.emit_relaxed_mov(Size::S64, loc, Location::SIMD(tmp_in))?;
1220 self.emit_f64_int_conv_check_sat(
1221 tmp_in,
1222 GEF64_LT_I64_MIN,
1223 LEF64_GT_I64_MAX,
1224 |this| {
1225 this.assembler.emit_mov(
1226 Size::S64,
1227 Location::Imm64(i64::MIN as u64),
1228 Location::GPR(tmp_out),
1229 )
1230 },
1231 |this| {
1232 this.assembler.emit_mov(
1233 Size::S64,
1234 Location::Imm64(i64::MAX as u64),
1235 Location::GPR(tmp_out),
1236 )
1237 },
1238 Some(|this: &mut Self| {
1239 this.assembler
1240 .emit_mov(Size::S64, Location::Imm64(0), Location::GPR(tmp_out))
1241 }),
1242 |this| {
1243 if this.assembler.arch_has_itruncf() {
1244 this.assembler.arch_emit_i64_trunc_sf64(tmp_in, tmp_out)
1245 } else {
1246 this.assembler
1247 .emit_cvttsd2si_64(XMMOrMemory::XMM(tmp_in), tmp_out)
1248 }
1249 },
1250 )?;
1251
1252 self.assembler
1253 .emit_mov(Size::S64, Location::GPR(tmp_out), ret)?;
1254 self.release_simd(tmp_in);
1255 self.release_gpr(tmp_out);
1256 Ok(())
1257 }
1258 fn convert_i64_f64_s_u(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
1259 if self.assembler.arch_has_itruncf() {
1260 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1261 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1262 })?;
1263 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1264 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1265 })?;
1266 self.emit_relaxed_mov(Size::S64, loc, Location::SIMD(tmp_in))?;
1267 self.assembler.arch_emit_i64_trunc_sf64(tmp_in, tmp_out)?;
1268 self.emit_relaxed_mov(Size::S64, Location::GPR(tmp_out), ret)?;
1269 self.release_simd(tmp_in);
1270 self.release_gpr(tmp_out);
1271 } else {
1272 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1273 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1274 })?;
1275 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1276 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1277 })?;
1278
1279 self.emit_relaxed_mov(Size::S64, loc, Location::SIMD(tmp_in))?;
1280 self.emit_f64_int_conv_check_trap(tmp_in, GEF64_LT_I64_MIN, LEF64_GT_I64_MAX)?;
1281
1282 self.assembler
1283 .emit_cvttsd2si_64(XMMOrMemory::XMM(tmp_in), tmp_out)?;
1284 self.move_location(Size::S64, Location::GPR(tmp_out), ret)?;
1285
1286 self.release_simd(tmp_in);
1287 self.release_gpr(tmp_out);
1288 }
1289 Ok(())
1290 }
1291 fn convert_i32_f64_s_s(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
1292 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1293 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1294 })?;
1295 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1296 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1297 })?;
1298
1299 let real_in = match loc {
1300 Location::Imm32(_) | Location::Imm64(_) => {
1301 self.move_location(Size::S64, loc, Location::GPR(tmp_out))?;
1302 self.move_location(Size::S64, Location::GPR(tmp_out), Location::SIMD(tmp_in))?;
1303 tmp_in
1304 }
1305 Location::SIMD(x) => x,
1306 _ => {
1307 self.move_location(Size::S64, loc, Location::SIMD(tmp_in))?;
1308 tmp_in
1309 }
1310 };
1311
1312 self.emit_f64_int_conv_check_sat(
1313 real_in,
1314 GEF64_LT_I32_MIN,
1315 LEF64_GT_I32_MAX,
1316 |this| {
1317 this.assembler.emit_mov(
1318 Size::S32,
1319 Location::Imm32(i32::MIN as u32),
1320 Location::GPR(tmp_out),
1321 )
1322 },
1323 |this| {
1324 this.assembler.emit_mov(
1325 Size::S32,
1326 Location::Imm32(i32::MAX as u32),
1327 Location::GPR(tmp_out),
1328 )
1329 },
1330 Some(|this: &mut Self| {
1331 this.assembler
1332 .emit_mov(Size::S32, Location::Imm32(0), Location::GPR(tmp_out))
1333 }),
1334 |this| {
1335 if this.assembler.arch_has_itruncf() {
1336 this.assembler.arch_emit_i32_trunc_sf64(tmp_in, tmp_out)
1337 } else {
1338 this.assembler
1339 .emit_cvttsd2si_32(XMMOrMemory::XMM(real_in), tmp_out)
1340 }
1341 },
1342 )?;
1343
1344 self.assembler
1345 .emit_mov(Size::S32, Location::GPR(tmp_out), ret)?;
1346 self.release_simd(tmp_in);
1347 self.release_gpr(tmp_out);
1348 Ok(())
1349 }
1350 fn convert_i32_f64_s_u(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
1351 if self.assembler.arch_has_itruncf() {
1352 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1353 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1354 })?;
1355 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1356 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1357 })?;
1358 self.emit_relaxed_mov(Size::S64, loc, Location::SIMD(tmp_in))?;
1359 self.assembler.arch_emit_i32_trunc_sf64(tmp_in, tmp_out)?;
1360 self.emit_relaxed_mov(Size::S32, Location::GPR(tmp_out), ret)?;
1361 self.release_simd(tmp_in);
1362 self.release_gpr(tmp_out);
1363 } else {
1364 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1365 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1366 })?;
1367 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1368 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1369 })?;
1370
1371 let real_in = match loc {
1372 Location::Imm32(_) | Location::Imm64(_) => {
1373 self.move_location(Size::S64, loc, Location::GPR(tmp_out))?;
1374 self.move_location(Size::S64, Location::GPR(tmp_out), Location::SIMD(tmp_in))?;
1375 tmp_in
1376 }
1377 Location::SIMD(x) => x,
1378 _ => {
1379 self.move_location(Size::S64, loc, Location::SIMD(tmp_in))?;
1380 tmp_in
1381 }
1382 };
1383
1384 self.emit_f64_int_conv_check_trap(real_in, GEF64_LT_I32_MIN, LEF64_GT_I32_MAX)?;
1385
1386 self.assembler
1387 .emit_cvttsd2si_32(XMMOrMemory::XMM(real_in), tmp_out)?;
1388 self.move_location(Size::S32, Location::GPR(tmp_out), ret)?;
1389
1390 self.release_simd(tmp_in);
1391 self.release_gpr(tmp_out);
1392 }
1393 Ok(())
1394 }
1395 fn convert_i32_f64_u_s(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
1396 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1397 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1398 })?;
1399 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1400 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1401 })?;
1402
1403 self.emit_relaxed_mov(Size::S64, loc, Location::SIMD(tmp_in))?;
1404 self.emit_f64_int_conv_check_sat(
1405 tmp_in,
1406 GEF64_LT_U32_MIN,
1407 LEF64_GT_U32_MAX,
1408 |this| {
1409 this.assembler
1410 .emit_mov(Size::S32, Location::Imm32(0), Location::GPR(tmp_out))
1411 },
1412 |this| {
1413 this.assembler.emit_mov(
1414 Size::S32,
1415 Location::Imm32(u32::MAX),
1416 Location::GPR(tmp_out),
1417 )
1418 },
1419 None::<fn(this: &mut Self) -> Result<(), CompileError>>,
1420 |this| {
1421 if this.assembler.arch_has_itruncf() {
1422 this.assembler.arch_emit_i32_trunc_uf64(tmp_in, tmp_out)
1423 } else {
1424 this.assembler
1425 .emit_cvttsd2si_64(XMMOrMemory::XMM(tmp_in), tmp_out)
1426 }
1427 },
1428 )?;
1429
1430 self.assembler
1431 .emit_mov(Size::S32, Location::GPR(tmp_out), ret)?;
1432 self.release_simd(tmp_in);
1433 self.release_gpr(tmp_out);
1434 Ok(())
1435 }
1436 fn convert_i32_f64_u_u(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
1437 if self.assembler.arch_has_itruncf() {
1438 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1439 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1440 })?;
1441 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1442 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1443 })?;
1444 self.emit_relaxed_mov(Size::S64, loc, Location::SIMD(tmp_in))?;
1445 self.assembler.arch_emit_i32_trunc_uf64(tmp_in, tmp_out)?;
1446 self.emit_relaxed_mov(Size::S32, Location::GPR(tmp_out), ret)?;
1447 self.release_simd(tmp_in);
1448 self.release_gpr(tmp_out);
1449 } else {
1450 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1451 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1452 })?;
1453 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1454 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1455 })?;
1456
1457 self.emit_relaxed_mov(Size::S64, loc, Location::SIMD(tmp_in))?;
1458 self.emit_f64_int_conv_check_trap(tmp_in, GEF64_LT_U32_MIN, LEF64_GT_U32_MAX)?;
1459
1460 self.assembler
1461 .emit_cvttsd2si_64(XMMOrMemory::XMM(tmp_in), tmp_out)?;
1462 self.move_location(Size::S32, Location::GPR(tmp_out), ret)?;
1463
1464 self.release_simd(tmp_in);
1465 self.release_gpr(tmp_out);
1466 }
1467 Ok(())
1468 }
1469 fn convert_i64_f32_u_s(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
1470 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1471 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1472 })?;
1473 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1474 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1475 })?;
1476
1477 self.emit_relaxed_mov(Size::S32, loc, Location::SIMD(tmp_in))?;
1478 self.emit_f32_int_conv_check_sat(
1479 tmp_in,
1480 GEF32_LT_U64_MIN,
1481 LEF32_GT_U64_MAX,
1482 |this| {
1483 this.assembler
1484 .emit_mov(Size::S64, Location::Imm64(0), Location::GPR(tmp_out))
1485 },
1486 |this| {
1487 this.assembler.emit_mov(
1488 Size::S64,
1489 Location::Imm64(u64::MAX),
1490 Location::GPR(tmp_out),
1491 )
1492 },
1493 None::<fn(this: &mut Self) -> Result<(), CompileError>>,
1494 |this| {
1495 if this.assembler.arch_has_itruncf() {
1496 this.assembler.arch_emit_i64_trunc_uf32(tmp_in, tmp_out)
1497 } else {
1498 let tmp = this.acquire_temp_gpr().ok_or_else(|| {
1499 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1500 })?;
1501 let tmp_x1 = this.acquire_temp_simd().ok_or_else(|| {
1502 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1503 })?;
1504 let tmp_x2 = this.acquire_temp_simd().ok_or_else(|| {
1505 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1506 })?;
1507
1508 this.assembler.emit_mov(
1509 Size::S32,
1510 Location::Imm32(1593835520u32),
1511 Location::GPR(tmp),
1512 )?; this.assembler.emit_mov(
1514 Size::S32,
1515 Location::GPR(tmp),
1516 Location::SIMD(tmp_x1),
1517 )?;
1518 this.assembler.emit_mov(
1519 Size::S32,
1520 Location::SIMD(tmp_in),
1521 Location::SIMD(tmp_x2),
1522 )?;
1523 this.assembler
1524 .emit_vsubss(tmp_in, XMMOrMemory::XMM(tmp_x1), tmp_in)?;
1525 this.assembler
1526 .emit_cvttss2si_64(XMMOrMemory::XMM(tmp_in), tmp_out)?;
1527 this.assembler.emit_mov(
1528 Size::S64,
1529 Location::Imm64(0x8000000000000000u64),
1530 Location::GPR(tmp),
1531 )?;
1532 this.assembler.emit_xor(
1533 Size::S64,
1534 Location::GPR(tmp_out),
1535 Location::GPR(tmp),
1536 )?;
1537 this.assembler
1538 .emit_cvttss2si_64(XMMOrMemory::XMM(tmp_x2), tmp_out)?;
1539 this.assembler
1540 .emit_ucomiss(XMMOrMemory::XMM(tmp_x1), tmp_x2)?;
1541 this.assembler.emit_cmovae_gpr_64(tmp, tmp_out)?;
1542
1543 this.release_simd(tmp_x2);
1544 this.release_simd(tmp_x1);
1545 this.release_gpr(tmp);
1546 Ok(())
1547 }
1548 },
1549 )?;
1550
1551 self.assembler
1552 .emit_mov(Size::S64, Location::GPR(tmp_out), ret)?;
1553 self.release_simd(tmp_in);
1554 self.release_gpr(tmp_out);
1555 Ok(())
1556 }
1557 fn convert_i64_f32_u_u(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
1558 if self.assembler.arch_has_itruncf() {
1559 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1560 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1561 })?;
1562 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1563 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1564 })?;
1565 self.emit_relaxed_mov(Size::S32, loc, Location::SIMD(tmp_in))?;
1566 self.assembler.arch_emit_i64_trunc_uf32(tmp_in, tmp_out)?;
1567 self.emit_relaxed_mov(Size::S64, Location::GPR(tmp_out), ret)?;
1568 self.release_simd(tmp_in);
1569 self.release_gpr(tmp_out);
1570 } else {
1571 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1572 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1573 })?;
1574 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1575 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1576 })?; self.emit_relaxed_mov(Size::S32, loc, Location::SIMD(tmp_in))?;
1579 self.emit_f32_int_conv_check_trap(tmp_in, GEF32_LT_U64_MIN, LEF32_GT_U64_MAX)?;
1580
1581 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
1582 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1583 })?; let tmp_x1 = self.acquire_temp_simd().ok_or_else(|| {
1585 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1586 })?; let tmp_x2 = self.acquire_temp_simd().ok_or_else(|| {
1588 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1589 })?; self.move_location(
1592 Size::S32,
1593 Location::Imm32(1593835520u32),
1594 Location::GPR(tmp),
1595 )?; self.move_location(Size::S32, Location::GPR(tmp), Location::SIMD(tmp_x1))?;
1597 self.move_location(Size::S32, Location::SIMD(tmp_in), Location::SIMD(tmp_x2))?;
1598 self.assembler
1599 .emit_vsubss(tmp_in, XMMOrMemory::XMM(tmp_x1), tmp_in)?;
1600 self.assembler
1601 .emit_cvttss2si_64(XMMOrMemory::XMM(tmp_in), tmp_out)?;
1602 self.move_location(
1603 Size::S64,
1604 Location::Imm64(0x8000000000000000u64),
1605 Location::GPR(tmp),
1606 )?;
1607 self.assembler
1608 .emit_xor(Size::S64, Location::GPR(tmp_out), Location::GPR(tmp))?;
1609 self.assembler
1610 .emit_cvttss2si_64(XMMOrMemory::XMM(tmp_x2), tmp_out)?;
1611 self.assembler
1612 .emit_ucomiss(XMMOrMemory::XMM(tmp_x1), tmp_x2)?;
1613 self.assembler.emit_cmovae_gpr_64(tmp, tmp_out)?;
1614 self.move_location(Size::S64, Location::GPR(tmp_out), ret)?;
1615
1616 self.release_simd(tmp_x2);
1617 self.release_simd(tmp_x1);
1618 self.release_gpr(tmp);
1619 self.release_simd(tmp_in);
1620 self.release_gpr(tmp_out);
1621 }
1622 Ok(())
1623 }
1624 fn convert_i64_f32_s_s(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
1625 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1626 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1627 })?;
1628 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1629 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1630 })?;
1631
1632 self.emit_relaxed_mov(Size::S32, loc, Location::SIMD(tmp_in))?;
1633 self.emit_f32_int_conv_check_sat(
1634 tmp_in,
1635 GEF32_LT_I64_MIN,
1636 LEF32_GT_I64_MAX,
1637 |this| {
1638 this.assembler.emit_mov(
1639 Size::S64,
1640 Location::Imm64(i64::MIN as u64),
1641 Location::GPR(tmp_out),
1642 )
1643 },
1644 |this| {
1645 this.assembler.emit_mov(
1646 Size::S64,
1647 Location::Imm64(i64::MAX as u64),
1648 Location::GPR(tmp_out),
1649 )
1650 },
1651 Some(|this: &mut Self| {
1652 this.assembler
1653 .emit_mov(Size::S64, Location::Imm64(0), Location::GPR(tmp_out))
1654 }),
1655 |this| {
1656 if this.assembler.arch_has_itruncf() {
1657 this.assembler.arch_emit_i64_trunc_sf32(tmp_in, tmp_out)
1658 } else {
1659 this.assembler
1660 .emit_cvttss2si_64(XMMOrMemory::XMM(tmp_in), tmp_out)
1661 }
1662 },
1663 )?;
1664
1665 self.assembler
1666 .emit_mov(Size::S64, Location::GPR(tmp_out), ret)?;
1667 self.release_simd(tmp_in);
1668 self.release_gpr(tmp_out);
1669 Ok(())
1670 }
1671 fn convert_i64_f32_s_u(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
1672 if self.assembler.arch_has_itruncf() {
1673 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1674 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1675 })?;
1676 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1677 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1678 })?;
1679 self.emit_relaxed_mov(Size::S32, loc, Location::SIMD(tmp_in))?;
1680 self.assembler.arch_emit_i64_trunc_sf32(tmp_in, tmp_out)?;
1681 self.emit_relaxed_mov(Size::S64, Location::GPR(tmp_out), ret)?;
1682 self.release_simd(tmp_in);
1683 self.release_gpr(tmp_out);
1684 } else {
1685 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1686 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1687 })?;
1688 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1689 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1690 })?;
1691
1692 self.emit_relaxed_mov(Size::S32, loc, Location::SIMD(tmp_in))?;
1693 self.emit_f32_int_conv_check_trap(tmp_in, GEF32_LT_I64_MIN, LEF32_GT_I64_MAX)?;
1694 self.assembler
1695 .emit_cvttss2si_64(XMMOrMemory::XMM(tmp_in), tmp_out)?;
1696 self.move_location(Size::S64, Location::GPR(tmp_out), ret)?;
1697
1698 self.release_simd(tmp_in);
1699 self.release_gpr(tmp_out);
1700 }
1701 Ok(())
1702 }
1703 fn convert_i32_f32_s_s(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
1704 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1705 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1706 })?;
1707 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1708 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1709 })?;
1710
1711 self.emit_relaxed_mov(Size::S32, loc, Location::SIMD(tmp_in))?;
1712 self.emit_f32_int_conv_check_sat(
1713 tmp_in,
1714 GEF32_LT_I32_MIN,
1715 LEF32_GT_I32_MAX,
1716 |this| {
1717 this.assembler.emit_mov(
1718 Size::S32,
1719 Location::Imm32(i32::MIN as u32),
1720 Location::GPR(tmp_out),
1721 )
1722 },
1723 |this| {
1724 this.assembler.emit_mov(
1725 Size::S32,
1726 Location::Imm32(i32::MAX as u32),
1727 Location::GPR(tmp_out),
1728 )
1729 },
1730 Some(|this: &mut Self| {
1731 this.assembler
1732 .emit_mov(Size::S32, Location::Imm32(0), Location::GPR(tmp_out))
1733 }),
1734 |this| {
1735 if this.assembler.arch_has_itruncf() {
1736 this.assembler.arch_emit_i32_trunc_sf32(tmp_in, tmp_out)
1737 } else {
1738 this.assembler
1739 .emit_cvttss2si_32(XMMOrMemory::XMM(tmp_in), tmp_out)
1740 }
1741 },
1742 )?;
1743
1744 self.assembler
1745 .emit_mov(Size::S32, Location::GPR(tmp_out), ret)?;
1746 self.release_simd(tmp_in);
1747 self.release_gpr(tmp_out);
1748 Ok(())
1749 }
1750 fn convert_i32_f32_s_u(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
1751 if self.assembler.arch_has_itruncf() {
1752 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1753 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1754 })?;
1755 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1756 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1757 })?;
1758 self.emit_relaxed_mov(Size::S32, loc, Location::SIMD(tmp_in))?;
1759 self.assembler.arch_emit_i32_trunc_sf32(tmp_in, tmp_out)?;
1760 self.emit_relaxed_mov(Size::S32, Location::GPR(tmp_out), ret)?;
1761 self.release_simd(tmp_in);
1762 self.release_gpr(tmp_out);
1763 } else {
1764 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1765 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1766 })?;
1767 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1768 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1769 })?;
1770
1771 self.emit_relaxed_mov(Size::S32, loc, Location::SIMD(tmp_in))?;
1772 self.emit_f32_int_conv_check_trap(tmp_in, GEF32_LT_I32_MIN, LEF32_GT_I32_MAX)?;
1773
1774 self.assembler
1775 .emit_cvttss2si_32(XMMOrMemory::XMM(tmp_in), tmp_out)?;
1776 self.move_location(Size::S32, Location::GPR(tmp_out), ret)?;
1777
1778 self.release_simd(tmp_in);
1779 self.release_gpr(tmp_out);
1780 }
1781 Ok(())
1782 }
1783 fn convert_i32_f32_u_s(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
1784 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1785 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1786 })?;
1787 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1788 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1789 })?;
1790 self.emit_relaxed_mov(Size::S32, loc, Location::SIMD(tmp_in))?;
1791 self.emit_f32_int_conv_check_sat(
1792 tmp_in,
1793 GEF32_LT_U32_MIN,
1794 LEF32_GT_U32_MAX,
1795 |this| {
1796 this.assembler
1797 .emit_mov(Size::S32, Location::Imm32(0), Location::GPR(tmp_out))
1798 },
1799 |this| {
1800 this.assembler.emit_mov(
1801 Size::S32,
1802 Location::Imm32(u32::MAX),
1803 Location::GPR(tmp_out),
1804 )
1805 },
1806 None::<fn(this: &mut Self) -> Result<(), CompileError>>,
1807 |this| {
1808 if this.assembler.arch_has_itruncf() {
1809 this.assembler.arch_emit_i32_trunc_uf32(tmp_in, tmp_out)
1810 } else {
1811 this.assembler
1812 .emit_cvttss2si_64(XMMOrMemory::XMM(tmp_in), tmp_out)
1813 }
1814 },
1815 )?;
1816
1817 self.assembler
1818 .emit_mov(Size::S32, Location::GPR(tmp_out), ret)?;
1819 self.release_simd(tmp_in);
1820 self.release_gpr(tmp_out);
1821 Ok(())
1822 }
1823 fn convert_i32_f32_u_u(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
1824 if self.assembler.arch_has_itruncf() {
1825 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1826 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1827 })?;
1828 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1829 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1830 })?;
1831 self.emit_relaxed_mov(Size::S32, loc, Location::SIMD(tmp_in))?;
1832 self.assembler.arch_emit_i32_trunc_uf32(tmp_in, tmp_out)?;
1833 self.emit_relaxed_mov(Size::S32, Location::GPR(tmp_out), ret)?;
1834 self.release_simd(tmp_in);
1835 self.release_gpr(tmp_out);
1836 } else {
1837 let tmp_out = self.acquire_temp_gpr().ok_or_else(|| {
1838 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
1839 })?;
1840 let tmp_in = self.acquire_temp_simd().ok_or_else(|| {
1841 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
1842 })?;
1843 self.emit_relaxed_mov(Size::S32, loc, Location::SIMD(tmp_in))?;
1844 self.emit_f32_int_conv_check_trap(tmp_in, GEF32_LT_U32_MIN, LEF32_GT_U32_MAX)?;
1845
1846 self.assembler
1847 .emit_cvttss2si_64(XMMOrMemory::XMM(tmp_in), tmp_out)?;
1848 self.move_location(Size::S32, Location::GPR(tmp_out), ret)?;
1849
1850 self.release_simd(tmp_in);
1851 self.release_gpr(tmp_out);
1852 }
1853 Ok(())
1854 }
1855
1856 fn emit_relaxed_atomic_xchg(
1857 &mut self,
1858 sz: Size,
1859 src: Location,
1860 dst: Location,
1861 ) -> Result<(), CompileError> {
1862 self.emit_relaxed_binop(AssemblerX64::emit_xchg, sz, src, dst)
1863 }
1864
1865 fn used_gprs_contains(&self, r: &GPR) -> bool {
1866 self.used_gprs.contains(r.into_index())
1867 }
1868 fn used_simd_contains(&self, r: &XMM) -> bool {
1869 self.used_simd.contains(r.into_index())
1870 }
1871 fn used_gprs_insert(&mut self, r: GPR) {
1872 self.used_gprs.insert(r.into_index());
1873 }
1874 fn used_simd_insert(&mut self, r: XMM) {
1875 self.used_simd.insert(r.into_index());
1876 }
1877 fn used_gprs_remove(&mut self, r: &GPR) -> bool {
1878 let ret = self.used_gprs_contains(r);
1879 self.used_gprs.set(r.into_index(), false);
1880 ret
1881 }
1882 fn used_simd_remove(&mut self, r: &XMM) -> bool {
1883 let ret = self.used_simd_contains(r);
1884 self.used_simd.set(r.into_index(), false);
1885 ret
1886 }
1887 fn emit_unwind_op(&mut self, op: UnwindOps<GPR, XMM>) -> Result<(), CompileError> {
1888 self.unwind_ops.push((self.get_offset().0, op));
1889 Ok(())
1890 }
1891 fn emit_illegal_op_internal(&mut self, trap: TrapCode) -> Result<(), CompileError> {
1892 let v = trap as u8;
1893 self.assembler.emit_ud1_payload(v)
1894 }
1895
1896 fn location_xor(
1898 &mut self,
1899 size: Size,
1900 source: Location,
1901 dest: Location,
1902 _flags: bool,
1903 ) -> Result<(), CompileError> {
1904 self.assembler.emit_xor(size, source, dest)
1905 }
1906 fn location_or(
1907 &mut self,
1908 size: Size,
1909 source: Location,
1910 dest: Location,
1911 _flags: bool,
1912 ) -> Result<(), CompileError> {
1913 self.assembler.emit_or(size, source, dest)
1914 }
1915 fn load_address(
1916 &mut self,
1917 size: Size,
1918 reg: Location,
1919 mem: Location,
1920 ) -> Result<(), CompileError> {
1921 match reg {
1922 Location::GPR(_) => {
1923 match mem {
1924 Location::Memory(_, _) | Location::Memory2(_, _, _, _) => {
1925 if size < Size::S32 {
1927 self.assembler.emit_xor(Size::S32, reg, reg)?;
1928 }
1929 self.assembler.emit_mov(size, mem, reg)?;
1930 }
1931 _ => codegen_error!("singlepass load_address unreachable"),
1932 }
1933 }
1934 _ => codegen_error!("singlepass load_address unreachable"),
1935 }
1936 Ok(())
1937 }
1938
1939 fn location_neg(
1940 &mut self,
1941 size_val: Size, signed: bool,
1943 source: Location,
1944 size_op: Size,
1945 dest: Location,
1946 ) -> Result<(), CompileError> {
1947 self.move_location_extend(size_val, signed, source, size_op, dest)?;
1948 self.assembler.emit_neg(size_val, dest)
1949 }
1950
1951 fn emit_relaxed_zero_extension(
1952 &mut self,
1953 sz_src: Size,
1954 src: Location,
1955 sz_dst: Size,
1956 dst: Location,
1957 ) -> Result<(), CompileError> {
1958 if (sz_src == Size::S32 || sz_src == Size::S64) && sz_dst == Size::S64 {
1959 self.emit_relaxed_binop(AssemblerX64::emit_mov, sz_src, src, dst)
1960 } else {
1961 self.emit_relaxed_zx_sx(AssemblerX64::emit_movzx, sz_src, src, sz_dst, dst)
1962 }
1963 }
1964}
1965
1966impl Machine for MachineX86_64 {
1967 type GPR = GPR;
1968 type SIMD = XMM;
1969
1970 const STACK_ALIGNMENT: usize = 8;
1972
1973 fn assembler_get_offset(&self) -> Offset {
1974 self.assembler.get_offset()
1975 }
1976
1977 fn get_vmctx_reg(&self) -> GPR {
1978 GPR::R15
1979 }
1980
1981 fn get_used_gprs(&self) -> Vec<GPR> {
1982 GPR::iterator()
1983 .filter(|x| self.used_gprs.contains(x.into_index()))
1984 .cloned()
1985 .collect()
1986 }
1987
1988 fn get_used_simd(&self) -> Vec<XMM> {
1989 XMM::iterator()
1990 .filter(|x| self.used_simd.contains(x.into_index()))
1991 .cloned()
1992 .collect()
1993 }
1994
1995 fn pick_gpr(&self) -> Option<GPR> {
1996 use GPR::*;
1997 static REGS: &[GPR] = &[R8, R9, R10, R11];
1998 for r in REGS {
1999 if !self.used_gprs_contains(r) {
2000 return Some(*r);
2001 }
2002 }
2003 None
2004 }
2005
2006 fn pick_temp_gpr(&self) -> Option<GPR> {
2007 use GPR::*;
2008 static REGS: &[GPR] = &[RAX, RCX, RDX, RDI, RSI];
2009 for r in REGS {
2010 if !self.used_gprs_contains(r) {
2011 return Some(*r);
2012 }
2013 }
2014 None
2015 }
2016
2017 fn acquire_temp_gpr(&mut self) -> Option<GPR> {
2018 let gpr = self.pick_temp_gpr();
2019 if let Some(x) = gpr {
2020 self.used_gprs_insert(x);
2021 }
2022 gpr
2023 }
2024
2025 fn release_gpr(&mut self, gpr: GPR) {
2026 assert!(self.used_gprs_remove(&gpr));
2027 }
2028
2029 fn reserve_unused_temp_gpr(&mut self, gpr: GPR) -> GPR {
2030 assert!(!self.used_gprs_contains(&gpr));
2031 self.used_gprs_insert(gpr);
2032 gpr
2033 }
2034
2035 fn reserve_gpr(&mut self, gpr: GPR) {
2036 self.used_gprs_insert(gpr);
2037 }
2038
2039 fn push_used_gpr(&mut self, used_gprs: &[GPR]) -> Result<usize, CompileError> {
2040 for r in used_gprs.iter() {
2041 self.assembler.emit_push(Size::S64, Location::GPR(*r))?;
2042 }
2043 Ok(used_gprs.len() * 8)
2044 }
2045
2046 fn pop_used_gpr(&mut self, used_gprs: &[GPR]) -> Result<(), CompileError> {
2047 for r in used_gprs.iter().rev() {
2048 self.assembler.emit_pop(Size::S64, Location::GPR(*r))?;
2049 }
2050 Ok(())
2051 }
2052
2053 fn pick_simd(&self) -> Option<XMM> {
2054 use XMM::*;
2055 static REGS: &[XMM] = &[XMM3, XMM4, XMM5, XMM6, XMM7];
2056 for r in REGS {
2057 if !self.used_simd_contains(r) {
2058 return Some(*r);
2059 }
2060 }
2061 None
2062 }
2063
2064 fn pick_temp_simd(&self) -> Option<XMM> {
2065 use XMM::*;
2066 static REGS: &[XMM] = &[XMM0, XMM1, XMM2];
2067 for r in REGS {
2068 if !self.used_simd_contains(r) {
2069 return Some(*r);
2070 }
2071 }
2072 None
2073 }
2074
2075 fn acquire_temp_simd(&mut self) -> Option<XMM> {
2076 let simd = self.pick_temp_simd();
2077 if let Some(x) = simd {
2078 self.used_simd_insert(x);
2079 }
2080 simd
2081 }
2082
2083 fn reserve_simd(&mut self, simd: XMM) {
2084 self.used_simd_insert(simd);
2085 }
2086
2087 fn release_simd(&mut self, simd: XMM) {
2088 assert!(self.used_simd_remove(&simd));
2089 }
2090
2091 fn push_used_simd(&mut self, used_xmms: &[XMM]) -> Result<usize, CompileError> {
2092 self.extend_stack((used_xmms.len() * 8) as u32)?;
2093
2094 for (i, r) in used_xmms.iter().enumerate() {
2095 self.move_location(
2096 Size::S64,
2097 Location::SIMD(*r),
2098 Location::Memory(GPR::RSP, (i * 8) as i32),
2099 )?;
2100 }
2101
2102 Ok(used_xmms.len() * 8)
2103 }
2104
2105 fn pop_used_simd(&mut self, used_xmms: &[XMM]) -> Result<(), CompileError> {
2106 for (i, r) in used_xmms.iter().enumerate() {
2107 self.move_location(
2108 Size::S64,
2109 Location::Memory(GPR::RSP, (i * 8) as i32),
2110 Location::SIMD(*r),
2111 )?;
2112 }
2113 self.assembler.emit_add(
2114 Size::S64,
2115 Location::Imm32((used_xmms.len() * 8) as u32),
2116 Location::GPR(GPR::RSP),
2117 )
2118 }
2119
2120 fn set_srcloc(&mut self, offset: u32) {
2121 self.src_loc = offset;
2122 }
2123
2124 fn mark_address_range_with_trap_code(&mut self, code: TrapCode, begin: usize, end: usize) {
2125 for i in begin..end {
2126 self.trap_table.offset_to_code.insert(i, code);
2127 }
2128 self.mark_instruction_address_end(begin);
2129 }
2130
2131 fn mark_address_with_trap_code(&mut self, code: TrapCode) {
2132 let offset = self.assembler.get_offset().0;
2133 self.trap_table.offset_to_code.insert(offset, code);
2134 self.mark_instruction_address_end(offset);
2135 }
2136
2137 fn mark_instruction_with_trap_code(&mut self, code: TrapCode) -> usize {
2138 let offset = self.assembler.get_offset().0;
2139 self.trap_table.offset_to_code.insert(offset, code);
2140 offset
2141 }
2142
2143 fn mark_instruction_address_end(&mut self, begin: usize) {
2144 self.instructions_address_map.push(InstructionAddressMap {
2145 srcloc: SourceLoc::new(self.src_loc),
2146 code_offset: begin,
2147 code_len: self.assembler.get_offset().0 - begin,
2148 });
2149 }
2150
2151 fn insert_stackoverflow(&mut self) {
2152 let offset = 0;
2153 self.trap_table
2154 .offset_to_code
2155 .insert(offset, TrapCode::StackOverflow);
2156 self.mark_instruction_address_end(offset);
2157 }
2158
2159 fn collect_trap_information(&self) -> Vec<TrapInformation> {
2160 self.trap_table
2161 .offset_to_code
2162 .clone()
2163 .into_iter()
2164 .map(|(offset, code)| TrapInformation {
2165 code_offset: offset as u32,
2166 trap_code: code,
2167 })
2168 .collect()
2169 }
2170
2171 fn instructions_address_map(&self) -> Vec<InstructionAddressMap> {
2172 self.instructions_address_map.clone()
2173 }
2174
2175 fn local_on_stack(&mut self, stack_offset: i32) -> Location {
2176 Location::Memory(GPR::RBP, -stack_offset)
2177 }
2178
2179 fn extend_stack(&mut self, delta_stack_offset: u32) -> Result<(), CompileError> {
2180 self.assembler.emit_sub(
2181 Size::S64,
2182 Location::Imm32(delta_stack_offset),
2183 Location::GPR(GPR::RSP),
2184 )
2185 }
2186
2187 fn truncate_stack(&mut self, delta_stack_offset: u32) -> Result<(), CompileError> {
2188 self.assembler.emit_add(
2189 Size::S64,
2190 Location::Imm32(delta_stack_offset),
2191 Location::GPR(GPR::RSP),
2192 )
2193 }
2194
2195 fn move_location_for_native(
2196 &mut self,
2197 _size: Size,
2198 loc: Location,
2199 dest: Location,
2200 ) -> Result<(), CompileError> {
2201 match loc {
2202 Location::Imm64(_) | Location::Memory(_, _) | Location::Memory2(_, _, _, _) => {
2203 let tmp = self.pick_temp_gpr();
2204 if let Some(x) = tmp {
2205 self.assembler.emit_mov(Size::S64, loc, Location::GPR(x))?;
2206 self.assembler.emit_mov(Size::S64, Location::GPR(x), dest)
2207 } else {
2208 self.assembler
2209 .emit_mov(Size::S64, Location::GPR(GPR::RAX), dest)?;
2210 self.assembler
2211 .emit_mov(Size::S64, loc, Location::GPR(GPR::RAX))?;
2212 self.assembler
2213 .emit_xchg(Size::S64, Location::GPR(GPR::RAX), dest)
2214 }
2215 }
2216 _ => self.assembler.emit_mov(Size::S64, loc, dest),
2217 }
2218 }
2219
2220 fn zero_location(&mut self, size: Size, location: Location) -> Result<(), CompileError> {
2221 self.assembler.emit_mov(size, Location::Imm32(0), location)
2222 }
2223
2224 fn local_pointer(&self) -> GPR {
2225 GPR::RBP
2226 }
2227
2228 fn is_local_on_stack(&self, idx: usize) -> bool {
2229 idx > 3
2230 }
2231
2232 fn get_local_location(&self, idx: usize, callee_saved_regs_size: usize) -> Location {
2233 match idx {
2235 0 => Location::GPR(GPR::R12),
2236 1 => Location::GPR(GPR::R13),
2237 2 => Location::GPR(GPR::R14),
2238 3 => Location::GPR(GPR::RBX),
2239 _ => Location::Memory(GPR::RBP, -(((idx - 3) * 8 + callee_saved_regs_size) as i32)),
2240 }
2241 }
2242
2243 fn move_local(&mut self, stack_offset: i32, location: Location) -> Result<(), CompileError> {
2244 self.assembler.emit_mov(
2245 Size::S64,
2246 location,
2247 Location::Memory(GPR::RBP, -stack_offset),
2248 )?;
2249 match location {
2250 Location::GPR(x) => self.emit_unwind_op(UnwindOps::SaveRegister {
2251 reg: UnwindRegister::GPR(x),
2252 bp_neg_offset: stack_offset,
2253 }),
2254 Location::SIMD(x) => self.emit_unwind_op(UnwindOps::SaveRegister {
2255 reg: UnwindRegister::FPR(x),
2256 bp_neg_offset: stack_offset,
2257 }),
2258 _ => Ok(()),
2259 }
2260 }
2261
2262 fn get_param_registers(&self) -> &'static [Self::GPR] {
2263 &[GPR::RDI, GPR::RSI, GPR::RDX, GPR::RCX, GPR::R8, GPR::R9]
2264 }
2265
2266 fn get_param_location(
2267 &self,
2268 idx: usize,
2269 _sz: Size,
2270 stack_location: &mut usize,
2271 _calling_convention: CallingConvention,
2272 ) -> Location {
2273 self.get_param_registers().get(idx).map_or_else(
2274 || {
2275 let loc = Location::Memory(GPR::RSP, *stack_location as i32);
2276 *stack_location += 8;
2277 loc
2278 },
2279 |reg| Location::GPR(*reg),
2280 )
2281 }
2282
2283 fn get_call_param_location(
2284 &self,
2285 return_slots: usize,
2286 idx: usize,
2287 _sz: Size,
2288 _stack_location: &mut usize,
2289 _calling_convention: CallingConvention,
2290 ) -> Location {
2291 let register_params = self.get_param_registers();
2292 let return_values_memory_size =
2293 8 * return_slots.saturating_sub(X86_64_RETURN_VALUE_REGISTERS.len());
2294 register_params.get(idx).map_or_else(
2295 || {
2296 Location::Memory(
2297 GPR::RBP,
2298 (16 + return_values_memory_size + (idx - register_params.len()) * 8) as i32,
2299 )
2300 },
2301 |reg| Location::GPR(*reg),
2302 )
2303 }
2304
2305 fn get_simple_param_location(&self, idx: usize) -> Self::GPR {
2306 self.get_param_registers()[idx]
2307 }
2308
2309 fn adjust_gpr_param_location(
2310 &mut self,
2311 _register: Self::GPR,
2312 _size: Size,
2313 ) -> Result<(), CompileError> {
2314 Ok(())
2315 }
2316
2317 fn get_return_value_location(&self, idx: usize, stack_location: &mut usize) -> Location {
2318 X86_64_RETURN_VALUE_REGISTERS.get(idx).map_or_else(
2319 || {
2320 let loc = Location::Memory(GPR::RSP, *stack_location as i32);
2321 *stack_location += 8;
2322 loc
2323 },
2324 |reg| Location::GPR(*reg),
2325 )
2326 }
2327
2328 fn get_call_return_value_location(&self, idx: usize) -> Location {
2329 X86_64_RETURN_VALUE_REGISTERS.get(idx).map_or_else(
2330 || {
2331 Location::Memory(
2332 GPR::RBP,
2333 (16 + (idx - X86_64_RETURN_VALUE_REGISTERS.len()) * 8) as i32,
2334 )
2335 },
2336 |reg| Location::GPR(*reg),
2337 )
2338 }
2339
2340 fn move_location(
2341 &mut self,
2342 size: Size,
2343 source: Location,
2344 dest: Location,
2345 ) -> Result<(), CompileError> {
2346 match source {
2347 Location::GPR(_) => self.assembler.emit_mov(size, source, dest),
2348 Location::Memory(_, _) => match dest {
2349 Location::GPR(_) | Location::SIMD(_) => self.assembler.emit_mov(size, source, dest),
2350 Location::Memory(_, _) | Location::Memory2(_, _, _, _) => {
2351 let tmp = self.pick_temp_gpr().ok_or_else(|| {
2352 CompileError::Codegen("singlepass can't pick a temp gpr".to_owned())
2353 })?;
2354 self.assembler.emit_mov(size, source, Location::GPR(tmp))?;
2355 self.assembler.emit_mov(size, Location::GPR(tmp), dest)
2356 }
2357 _ => codegen_error!("singlepass move_location unreachable"),
2358 },
2359 Location::Memory2(_, _, _, _) => match dest {
2360 Location::GPR(_) | Location::SIMD(_) => self.assembler.emit_mov(size, source, dest),
2361 Location::Memory(_, _) | Location::Memory2(_, _, _, _) => {
2362 let tmp = self.pick_temp_gpr().ok_or_else(|| {
2363 CompileError::Codegen("singlepass can't pick a temp gpr".to_owned())
2364 })?;
2365 self.assembler.emit_mov(size, source, Location::GPR(tmp))?;
2366 self.assembler.emit_mov(size, Location::GPR(tmp), dest)
2367 }
2368 _ => codegen_error!("singlepass move_location unreachable"),
2369 },
2370 Location::Imm8(_) | Location::Imm32(_) | Location::Imm64(_) => match dest {
2371 Location::GPR(_) | Location::SIMD(_) => self.assembler.emit_mov(size, source, dest),
2372 Location::Memory(_, _) | Location::Memory2(_, _, _, _) => {
2373 let tmp = self.pick_temp_gpr().ok_or_else(|| {
2374 CompileError::Codegen("singlepass can't pick a temp gpr".to_owned())
2375 })?;
2376 self.assembler.emit_mov(size, source, Location::GPR(tmp))?;
2377 self.assembler.emit_mov(size, Location::GPR(tmp), dest)
2378 }
2379 _ => codegen_error!("singlepass move_location unreachable"),
2380 },
2381 Location::SIMD(_) => self.assembler.emit_mov(size, source, dest),
2382 _ => codegen_error!("singlepass move_location unreachable"),
2383 }
2384 }
2385
2386 fn move_location_extend(
2387 &mut self,
2388 size_val: Size,
2389 signed: bool,
2390 source: Location,
2391 size_op: Size,
2392 dest: Location,
2393 ) -> Result<(), CompileError> {
2394 let dst = match dest {
2395 Location::Memory(_, _) | Location::Memory2(_, _, _, _) => {
2396 Location::GPR(self.acquire_temp_gpr().ok_or_else(|| {
2397 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2398 })?)
2399 }
2400 Location::GPR(_) | Location::SIMD(_) => dest,
2401 _ => codegen_error!("singlepass move_location_extend unreachable"),
2402 };
2403 match source {
2404 Location::GPR(_)
2405 | Location::Memory(_, _)
2406 | Location::Memory2(_, _, _, _)
2407 | Location::Imm32(_)
2408 | Location::Imm64(_) => match size_val {
2409 Size::S32 | Size::S64 => self.assembler.emit_mov(size_val, source, dst),
2410 Size::S16 | Size::S8 => {
2411 if signed {
2412 self.assembler.emit_movsx(size_val, source, size_op, dst)
2413 } else {
2414 self.assembler.emit_movzx(size_val, source, size_op, dst)
2415 }
2416 }
2417 },
2418 _ => panic!(
2419 "unimplemented move_location_extend({size_val:?}, {signed}, {source:?}, {size_op:?}, {dest:?}"
2420 ),
2421 }?;
2422 if dst != dest {
2423 self.assembler.emit_mov(size_op, dst, dest)?;
2424 match dst {
2425 Location::GPR(x) => self.release_gpr(x),
2426 _ => codegen_error!("singlepass move_location_extend unreachable"),
2427 };
2428 }
2429 Ok(())
2430 }
2431
2432 fn init_stack_loc(
2433 &mut self,
2434 init_stack_loc_cnt: u64,
2435 last_stack_loc: Location,
2436 ) -> Result<(), CompileError> {
2437 self.assembler.emit_mov(
2439 Size::S64,
2440 Location::Imm64(init_stack_loc_cnt),
2441 Location::GPR(GPR::RCX),
2442 )?;
2443 self.assembler
2444 .emit_xor(Size::S64, Location::GPR(GPR::RAX), Location::GPR(GPR::RAX))?;
2445 self.assembler
2446 .emit_lea(Size::S64, last_stack_loc, Location::GPR(GPR::RDI))?;
2447 self.assembler.emit_rep_stosq()
2448 }
2449
2450 fn restore_saved_area(&mut self, saved_area_offset: i32) -> Result<(), CompileError> {
2451 self.assembler.emit_lea(
2452 Size::S64,
2453 Location::Memory(GPR::RBP, -saved_area_offset),
2454 Location::GPR(GPR::RSP),
2455 )
2456 }
2457
2458 fn pop_location(&mut self, location: Location) -> Result<(), CompileError> {
2459 self.assembler.emit_pop(Size::S64, location)
2460 }
2461
2462 fn assembler_finalize(
2463 self,
2464 assembly_comments: HashMap<usize, AssemblyComment>,
2465 ) -> Result<FinalizedAssembly, CompileError> {
2466 Ok(FinalizedAssembly {
2467 body: self.assembler.finalize().map_err(|e| {
2468 CompileError::Codegen(format!("Assembler failed finalization with: {e:?}"))
2469 })?,
2470 assembly_comments,
2471 })
2472 }
2473
2474 fn get_offset(&self) -> Offset {
2475 self.assembler.get_offset()
2476 }
2477
2478 fn finalize_function(&mut self) -> Result<(), CompileError> {
2479 self.assembler.finalize_function()?;
2480 Ok(())
2481 }
2482
2483 fn emit_function_prolog(&mut self) -> Result<(), CompileError> {
2484 self.emit_push(Size::S64, Location::GPR(GPR::RBP))?;
2485 self.emit_unwind_op(UnwindOps::PushFP { up_to_sp: 16 })?;
2486 self.move_location(Size::S64, Location::GPR(GPR::RSP), Location::GPR(GPR::RBP))?;
2487 self.emit_unwind_op(UnwindOps::DefineNewFrame)
2488 }
2489
2490 fn emit_function_epilog(&mut self) -> Result<(), CompileError> {
2491 self.move_location(Size::S64, Location::GPR(GPR::RBP), Location::GPR(GPR::RSP))?;
2492 self.emit_pop(Size::S64, Location::GPR(GPR::RBP))
2493 }
2494
2495 fn emit_function_return_float(&mut self) -> Result<(), CompileError> {
2496 self.move_location(
2497 Size::S64,
2498 Location::GPR(GPR::RAX),
2499 Location::SIMD(XMM::XMM0),
2500 )
2501 }
2502
2503 fn canonicalize_nan(
2504 &mut self,
2505 sz: Size,
2506 input: Location,
2507 output: Location,
2508 ) -> Result<(), CompileError> {
2509 let tmp1 = self.acquire_temp_simd().ok_or_else(|| {
2510 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
2511 })?;
2512 let tmp2 = self.acquire_temp_simd().ok_or_else(|| {
2513 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
2514 })?;
2515 let tmp3 = self.acquire_temp_simd().ok_or_else(|| {
2516 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
2517 })?;
2518
2519 self.emit_relaxed_mov(sz, input, Location::SIMD(tmp1))?;
2520 let tmpg1 = self.acquire_temp_gpr().ok_or_else(|| {
2521 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
2522 })?;
2523
2524 match sz {
2525 Size::S32 => {
2526 self.assembler
2527 .emit_vcmpunordss(tmp1, XMMOrMemory::XMM(tmp1), tmp2)?;
2528 self.move_location(
2529 Size::S32,
2530 Location::Imm32(0x7FC0_0000), Location::GPR(tmpg1),
2532 )?;
2533 self.move_location(Size::S64, Location::GPR(tmpg1), Location::SIMD(tmp3))?;
2534 self.assembler
2535 .emit_vblendvps(tmp2, XMMOrMemory::XMM(tmp3), tmp1, tmp1)?;
2536 }
2537 Size::S64 => {
2538 self.assembler
2539 .emit_vcmpunordsd(tmp1, XMMOrMemory::XMM(tmp1), tmp2)?;
2540 self.move_location(
2541 Size::S64,
2542 Location::Imm64(0x7FF8_0000_0000_0000), Location::GPR(tmpg1),
2544 )?;
2545 self.move_location(Size::S64, Location::GPR(tmpg1), Location::SIMD(tmp3))?;
2546 self.assembler
2547 .emit_vblendvpd(tmp2, XMMOrMemory::XMM(tmp3), tmp1, tmp1)?;
2548 }
2549 _ => codegen_error!("singlepass canonicalize_nan unreachable"),
2550 }
2551
2552 self.emit_relaxed_mov(sz, Location::SIMD(tmp1), output)?;
2553
2554 self.release_gpr(tmpg1);
2555 self.release_simd(tmp3);
2556 self.release_simd(tmp2);
2557 self.release_simd(tmp1);
2558 Ok(())
2559 }
2560
2561 fn emit_illegal_op(&mut self, trap: TrapCode) -> Result<(), CompileError> {
2562 let v = trap as u8;
2572 let offset = self.assembler.get_offset().0;
2575 self.assembler.emit_ud1_payload(v)?;
2576 self.mark_instruction_address_end(offset);
2577 Ok(())
2578 }
2579
2580 fn get_label(&mut self) -> Label {
2581 self.assembler.new_dynamic_label()
2582 }
2583
2584 fn emit_label(&mut self, label: Label) -> Result<(), CompileError> {
2585 self.assembler.emit_label(label)
2586 }
2587
2588 fn get_gpr_for_call(&self) -> GPR {
2589 GPR::RAX
2590 }
2591
2592 fn emit_call_register(&mut self, reg: GPR) -> Result<(), CompileError> {
2593 self.assembler.emit_call_register(reg)
2594 }
2595
2596 fn emit_call_label(&mut self, label: Label) -> Result<(), CompileError> {
2597 self.assembler.emit_call_label(label)
2598 }
2599
2600 fn arch_emit_indirect_call_with_trampoline(
2601 &mut self,
2602 location: Location,
2603 ) -> Result<(), CompileError> {
2604 self.assembler
2605 .arch_emit_indirect_call_with_trampoline(location)
2606 }
2607
2608 fn emit_debug_breakpoint(&mut self) -> Result<(), CompileError> {
2609 self.assembler.emit_bkpt()
2610 }
2611
2612 fn emit_call_location(&mut self, location: Location) -> Result<(), CompileError> {
2613 self.assembler.emit_call_location(location)
2614 }
2615
2616 fn location_add(
2617 &mut self,
2618 size: Size,
2619 source: Location,
2620 dest: Location,
2621 _flags: bool,
2622 ) -> Result<(), CompileError> {
2623 self.assembler.emit_add(size, source, dest)
2624 }
2625
2626 fn location_cmp(
2627 &mut self,
2628 size: Size,
2629 source: Location,
2630 dest: Location,
2631 ) -> Result<(), CompileError> {
2632 self.emit_relaxed_cmp(size, source, dest)
2633 }
2634
2635 fn jmp_unconditional(&mut self, label: Label) -> Result<(), CompileError> {
2636 self.assembler.emit_jmp(Condition::None, label)
2637 }
2638
2639 fn jmp_on_condition(
2640 &mut self,
2641 cond: UnsignedCondition,
2642 size: Size,
2643 loc_a: AbstractLocation<Self::GPR, Self::SIMD>,
2644 loc_b: AbstractLocation<Self::GPR, Self::SIMD>,
2645 label: Label,
2646 ) -> Result<(), CompileError> {
2647 self.emit_relaxed_cmp(size, loc_b, loc_a)?;
2648 let cond = match cond {
2649 UnsignedCondition::Equal => Condition::Equal,
2650 UnsignedCondition::NotEqual => Condition::NotEqual,
2651 UnsignedCondition::Above => Condition::Above,
2652 UnsignedCondition::AboveEqual => Condition::AboveEqual,
2653 UnsignedCondition::Below => Condition::Below,
2654 UnsignedCondition::BelowEqual => Condition::BelowEqual,
2655 };
2656 self.assembler.emit_jmp(cond, label)
2657 }
2658
2659 fn emit_jmp_to_jumptable(&mut self, label: Label, cond: Location) -> Result<(), CompileError> {
2660 let tmp1 = self
2661 .pick_temp_gpr()
2662 .ok_or_else(|| CompileError::Codegen("singlepass can't pick a temp gpr".to_owned()))?;
2663 self.reserve_gpr(tmp1);
2664 let tmp2 = self
2665 .pick_temp_gpr()
2666 .ok_or_else(|| CompileError::Codegen("singlepass can't pick a temp gpr".to_owned()))?;
2667 self.reserve_gpr(tmp2);
2668
2669 self.assembler.emit_lea_label(label, Location::GPR(tmp1))?;
2670 self.move_location(Size::S32, cond, Location::GPR(tmp2))?;
2671
2672 let instr_size = self.assembler.get_jmp_instr_size();
2673 self.assembler
2674 .emit_imul_imm32_gpr64(instr_size as _, tmp2)?;
2675 self.assembler
2676 .emit_add(Size::S64, Location::GPR(tmp1), Location::GPR(tmp2))?;
2677 self.assembler.emit_jmp_location(Location::GPR(tmp2))?;
2678 self.release_gpr(tmp2);
2679 self.release_gpr(tmp1);
2680 Ok(())
2681 }
2682
2683 fn align_for_loop(&mut self) -> Result<(), CompileError> {
2684 match self.assembler.get_offset().0 % 16 {
2688 0 => {}
2689 x => {
2690 self.assembler.emit_nop_n(16 - x)?;
2691 }
2692 }
2693 assert_eq!(self.assembler.get_offset().0 % 16, 0);
2694 Ok(())
2695 }
2696
2697 fn emit_ret(&mut self) -> Result<(), CompileError> {
2698 self.assembler.emit_ret()
2699 }
2700
2701 fn emit_push(&mut self, size: Size, loc: Location) -> Result<(), CompileError> {
2702 self.assembler.emit_push(size, loc)
2703 }
2704
2705 fn emit_pop(&mut self, size: Size, loc: Location) -> Result<(), CompileError> {
2706 self.assembler.emit_pop(size, loc)
2707 }
2708
2709 fn emit_memory_fence(&mut self) -> Result<(), CompileError> {
2710 Ok(())
2712 }
2713
2714 fn emit_imul_imm32(&mut self, size: Size, imm32: u32, gpr: GPR) -> Result<(), CompileError> {
2715 match size {
2716 Size::S64 => self.assembler.emit_imul_imm32_gpr64(imm32, gpr),
2717 _ => {
2718 codegen_error!("singlepass emit_imul_imm32 unreachable");
2719 }
2720 }
2721 }
2722
2723 fn emit_relaxed_mov(
2724 &mut self,
2725 sz: Size,
2726 src: Location,
2727 dst: Location,
2728 ) -> Result<(), CompileError> {
2729 self.emit_relaxed_binop(AssemblerX64::emit_mov, sz, src, dst)
2730 }
2731
2732 fn emit_relaxed_cmp(
2733 &mut self,
2734 sz: Size,
2735 src: Location,
2736 dst: Location,
2737 ) -> Result<(), CompileError> {
2738 self.emit_relaxed_binop(AssemblerX64::emit_cmp, sz, src, dst)
2739 }
2740
2741 fn emit_relaxed_sign_extension(
2742 &mut self,
2743 sz_src: Size,
2744 src: Location,
2745 sz_dst: Size,
2746 dst: Location,
2747 ) -> Result<(), CompileError> {
2748 self.emit_relaxed_zx_sx(AssemblerX64::emit_movsx, sz_src, src, sz_dst, dst)
2749 }
2750
2751 fn emit_binop_add32(
2752 &mut self,
2753 loc_a: Location,
2754 loc_b: Location,
2755 ret: Location,
2756 ) -> Result<(), CompileError> {
2757 self.emit_binop_i32(AssemblerX64::emit_add, loc_a, loc_b, ret)
2758 }
2759
2760 fn emit_binop_sub32(
2761 &mut self,
2762 loc_a: Location,
2763 loc_b: Location,
2764 ret: Location,
2765 ) -> Result<(), CompileError> {
2766 self.emit_binop_i32(AssemblerX64::emit_sub, loc_a, loc_b, ret)
2767 }
2768
2769 fn emit_binop_mul32(
2770 &mut self,
2771 loc_a: Location,
2772 loc_b: Location,
2773 ret: Location,
2774 ) -> Result<(), CompileError> {
2775 self.emit_binop_i32(AssemblerX64::emit_imul, loc_a, loc_b, ret)
2776 }
2777
2778 fn emit_binop_udiv32(
2779 &mut self,
2780 loc_a: Location,
2781 loc_b: Location,
2782 ret: Location,
2783 integer_division_by_zero: Label,
2784 ) -> Result<usize, CompileError> {
2785 for reg in DIV_REM_REGISTERS {
2786 self.reserve_unused_temp_gpr(reg);
2787 }
2788 self.assembler
2789 .emit_mov(Size::S32, loc_a, Location::GPR(GPR::RAX))?;
2790 self.assembler
2791 .emit_xor(Size::S32, Location::GPR(GPR::RDX), Location::GPR(GPR::RDX))?;
2792 let offset = self.emit_relaxed_xdiv(
2793 AssemblerX64::emit_div,
2794 Size::S32,
2795 loc_b,
2796 integer_division_by_zero,
2797 )?;
2798 self.assembler
2799 .emit_mov(Size::S32, Location::GPR(GPR::RAX), ret)?;
2800 for reg in DIV_REM_REGISTERS {
2801 self.release_gpr(reg);
2802 }
2803 Ok(offset)
2804 }
2805
2806 fn emit_binop_sdiv32(
2807 &mut self,
2808 loc_a: Location,
2809 loc_b: Location,
2810 ret: Location,
2811 integer_division_by_zero: Label,
2812 _integer_overflow: Label,
2813 ) -> Result<usize, CompileError> {
2814 for reg in DIV_REM_REGISTERS {
2815 self.reserve_unused_temp_gpr(reg);
2816 }
2817 self.assembler
2818 .emit_mov(Size::S32, loc_a, Location::GPR(GPR::RAX))?;
2819 self.assembler.emit_cdq()?;
2820 let offset = self.emit_relaxed_xdiv(
2821 AssemblerX64::emit_idiv,
2822 Size::S32,
2823 loc_b,
2824 integer_division_by_zero,
2825 )?;
2826 self.assembler
2827 .emit_mov(Size::S32, Location::GPR(GPR::RAX), ret)?;
2828 for reg in DIV_REM_REGISTERS {
2829 self.release_gpr(reg);
2830 }
2831 Ok(offset)
2832 }
2833
2834 fn emit_binop_urem32(
2835 &mut self,
2836 loc_a: Location,
2837 loc_b: Location,
2838 ret: Location,
2839 integer_division_by_zero: Label,
2840 ) -> Result<usize, CompileError> {
2841 for reg in DIV_REM_REGISTERS {
2842 self.reserve_unused_temp_gpr(reg);
2843 }
2844 self.assembler
2845 .emit_mov(Size::S32, loc_a, Location::GPR(GPR::RAX))?;
2846 self.assembler
2847 .emit_xor(Size::S32, Location::GPR(GPR::RDX), Location::GPR(GPR::RDX))?;
2848 let offset = self.emit_relaxed_xdiv(
2849 AssemblerX64::emit_div,
2850 Size::S32,
2851 loc_b,
2852 integer_division_by_zero,
2853 )?;
2854 self.assembler
2855 .emit_mov(Size::S32, Location::GPR(GPR::RDX), ret)?;
2856 for reg in DIV_REM_REGISTERS {
2857 self.release_gpr(reg);
2858 }
2859 Ok(offset)
2860 }
2861
2862 fn emit_binop_srem32(
2863 &mut self,
2864 loc_a: Location,
2865 loc_b: Location,
2866 ret: Location,
2867 integer_division_by_zero: Label,
2868 ) -> Result<usize, CompileError> {
2869 for reg in DIV_REM_REGISTERS {
2870 self.reserve_unused_temp_gpr(reg);
2871 }
2872 let normal_path = self.assembler.get_label();
2873 let end = self.assembler.get_label();
2874
2875 self.emit_relaxed_cmp(Size::S32, Location::Imm32(0x80000000), loc_a)?;
2876 self.assembler.emit_jmp(Condition::NotEqual, normal_path)?;
2877 self.emit_relaxed_cmp(Size::S32, Location::Imm32(0xffffffff), loc_b)?;
2878 self.assembler.emit_jmp(Condition::NotEqual, normal_path)?;
2879 self.move_location(Size::S32, Location::Imm32(0), ret)?;
2880 self.assembler.emit_jmp(Condition::None, end)?;
2881
2882 self.emit_label(normal_path)?;
2883 self.assembler
2884 .emit_mov(Size::S32, loc_a, Location::GPR(GPR::RAX))?;
2885 self.assembler.emit_cdq()?;
2886 let offset = self.emit_relaxed_xdiv(
2887 AssemblerX64::emit_idiv,
2888 Size::S32,
2889 loc_b,
2890 integer_division_by_zero,
2891 )?;
2892 self.assembler
2893 .emit_mov(Size::S32, Location::GPR(GPR::RDX), ret)?;
2894
2895 self.emit_label(end)?;
2896 for reg in DIV_REM_REGISTERS {
2897 self.release_gpr(reg);
2898 }
2899 Ok(offset)
2900 }
2901
2902 fn emit_binop_and32(
2903 &mut self,
2904 loc_a: Location,
2905 loc_b: Location,
2906 ret: Location,
2907 ) -> Result<(), CompileError> {
2908 self.emit_binop_i32(AssemblerX64::emit_and, loc_a, loc_b, ret)
2909 }
2910
2911 fn emit_binop_or32(
2912 &mut self,
2913 loc_a: Location,
2914 loc_b: Location,
2915 ret: Location,
2916 ) -> Result<(), CompileError> {
2917 self.emit_binop_i32(AssemblerX64::emit_or, loc_a, loc_b, ret)
2918 }
2919
2920 fn emit_binop_xor32(
2921 &mut self,
2922 loc_a: Location,
2923 loc_b: Location,
2924 ret: Location,
2925 ) -> Result<(), CompileError> {
2926 self.emit_binop_i32(AssemblerX64::emit_xor, loc_a, loc_b, ret)
2927 }
2928
2929 fn i32_cmp_ge_s(
2930 &mut self,
2931 loc_a: Location,
2932 loc_b: Location,
2933 ret: Location,
2934 ) -> Result<(), CompileError> {
2935 self.emit_cmpop_i32_dynamic_b(Condition::GreaterEqual, loc_a, loc_b, ret)
2936 }
2937
2938 fn i32_cmp_gt_s(
2939 &mut self,
2940 loc_a: Location,
2941 loc_b: Location,
2942 ret: Location,
2943 ) -> Result<(), CompileError> {
2944 self.emit_cmpop_i32_dynamic_b(Condition::Greater, loc_a, loc_b, ret)
2945 }
2946
2947 fn i32_cmp_le_s(
2948 &mut self,
2949 loc_a: Location,
2950 loc_b: Location,
2951 ret: Location,
2952 ) -> Result<(), CompileError> {
2953 self.emit_cmpop_i32_dynamic_b(Condition::LessEqual, loc_a, loc_b, ret)
2954 }
2955
2956 fn i32_cmp_lt_s(
2957 &mut self,
2958 loc_a: Location,
2959 loc_b: Location,
2960 ret: Location,
2961 ) -> Result<(), CompileError> {
2962 self.emit_cmpop_i32_dynamic_b(Condition::Less, loc_a, loc_b, ret)
2963 }
2964
2965 fn i32_cmp_ge_u(
2966 &mut self,
2967 loc_a: Location,
2968 loc_b: Location,
2969 ret: Location,
2970 ) -> Result<(), CompileError> {
2971 self.emit_cmpop_i32_dynamic_b(Condition::AboveEqual, loc_a, loc_b, ret)
2972 }
2973
2974 fn i32_cmp_gt_u(
2975 &mut self,
2976 loc_a: Location,
2977 loc_b: Location,
2978 ret: Location,
2979 ) -> Result<(), CompileError> {
2980 self.emit_cmpop_i32_dynamic_b(Condition::Above, loc_a, loc_b, ret)
2981 }
2982
2983 fn i32_cmp_le_u(
2984 &mut self,
2985 loc_a: Location,
2986 loc_b: Location,
2987 ret: Location,
2988 ) -> Result<(), CompileError> {
2989 self.emit_cmpop_i32_dynamic_b(Condition::BelowEqual, loc_a, loc_b, ret)
2990 }
2991
2992 fn i32_cmp_lt_u(
2993 &mut self,
2994 loc_a: Location,
2995 loc_b: Location,
2996 ret: Location,
2997 ) -> Result<(), CompileError> {
2998 self.emit_cmpop_i32_dynamic_b(Condition::Below, loc_a, loc_b, ret)
2999 }
3000
3001 fn i32_cmp_ne(
3002 &mut self,
3003 loc_a: Location,
3004 loc_b: Location,
3005 ret: Location,
3006 ) -> Result<(), CompileError> {
3007 self.emit_cmpop_i32_dynamic_b(Condition::NotEqual, loc_a, loc_b, ret)
3008 }
3009
3010 fn i32_cmp_eq(
3011 &mut self,
3012 loc_a: Location,
3013 loc_b: Location,
3014 ret: Location,
3015 ) -> Result<(), CompileError> {
3016 self.emit_cmpop_i32_dynamic_b(Condition::Equal, loc_a, loc_b, ret)
3017 }
3018
3019 fn i32_clz(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
3020 let src = match loc {
3021 Location::Imm32(_) | Location::Memory(_, _) => {
3022 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
3023 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3024 })?;
3025 self.move_location(Size::S32, loc, Location::GPR(tmp))?;
3026 tmp
3027 }
3028 Location::GPR(reg) => reg,
3029 _ => {
3030 codegen_error!("singlepass i32_clz unreachable");
3031 }
3032 };
3033 let dst = match ret {
3034 Location::Memory(_, _) => self.acquire_temp_gpr().ok_or_else(|| {
3035 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3036 })?,
3037 Location::GPR(reg) => reg,
3038 _ => {
3039 codegen_error!("singlepass i32_clz unreachable");
3040 }
3041 };
3042
3043 if self.assembler.arch_has_xzcnt() {
3044 self.assembler
3045 .arch_emit_lzcnt(Size::S32, Location::GPR(src), Location::GPR(dst))?;
3046 } else {
3047 let zero_path = self.assembler.get_label();
3048 let end = self.assembler.get_label();
3049
3050 self.assembler.emit_test_gpr_64(src)?;
3051 self.assembler.emit_jmp(Condition::Equal, zero_path)?;
3052 self.assembler
3053 .emit_bsr(Size::S32, Location::GPR(src), Location::GPR(dst))?;
3054 self.assembler
3055 .emit_xor(Size::S32, Location::Imm32(31), Location::GPR(dst))?;
3056 self.assembler.emit_jmp(Condition::None, end)?;
3057 self.emit_label(zero_path)?;
3058 self.move_location(Size::S32, Location::Imm32(32), Location::GPR(dst))?;
3059 self.emit_label(end)?;
3060 }
3061 match loc {
3062 Location::Imm32(_) | Location::Memory(_, _) => {
3063 self.release_gpr(src);
3064 }
3065 _ => {}
3066 };
3067 if let Location::Memory(_, _) = ret {
3068 self.move_location(Size::S32, Location::GPR(dst), ret)?;
3069 self.release_gpr(dst);
3070 };
3071 Ok(())
3072 }
3073
3074 fn i32_ctz(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
3075 let src = match loc {
3076 Location::Imm32(_) | Location::Memory(_, _) => {
3077 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
3078 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3079 })?;
3080 self.move_location(Size::S32, loc, Location::GPR(tmp))?;
3081 tmp
3082 }
3083 Location::GPR(reg) => reg,
3084 _ => {
3085 codegen_error!("singlepass i32_ctz unreachable");
3086 }
3087 };
3088 let dst = match ret {
3089 Location::Memory(_, _) => self.acquire_temp_gpr().ok_or_else(|| {
3090 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3091 })?,
3092 Location::GPR(reg) => reg,
3093 _ => {
3094 codegen_error!("singlepass i32_ctz unreachable");
3095 }
3096 };
3097
3098 if self.assembler.arch_has_xzcnt() {
3099 self.assembler
3100 .arch_emit_tzcnt(Size::S32, Location::GPR(src), Location::GPR(dst))?;
3101 } else {
3102 let zero_path = self.assembler.get_label();
3103 let end = self.assembler.get_label();
3104
3105 self.assembler.emit_test_gpr_64(src)?;
3106 self.assembler.emit_jmp(Condition::Equal, zero_path)?;
3107 self.assembler
3108 .emit_bsf(Size::S32, Location::GPR(src), Location::GPR(dst))?;
3109 self.assembler.emit_jmp(Condition::None, end)?;
3110 self.emit_label(zero_path)?;
3111 self.move_location(Size::S32, Location::Imm32(32), Location::GPR(dst))?;
3112 self.emit_label(end)?;
3113 }
3114
3115 match loc {
3116 Location::Imm32(_) | Location::Memory(_, _) => {
3117 self.release_gpr(src);
3118 }
3119 _ => {}
3120 };
3121 if let Location::Memory(_, _) = ret {
3122 self.move_location(Size::S32, Location::GPR(dst), ret)?;
3123 self.release_gpr(dst);
3124 };
3125 Ok(())
3126 }
3127
3128 fn i32_popcnt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
3129 match loc {
3130 Location::Imm32(_) => {
3131 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
3132 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3133 })?;
3134 self.move_location(Size::S32, loc, Location::GPR(tmp))?;
3135 if let Location::Memory(_, _) = ret {
3136 let out_tmp = self.acquire_temp_gpr().ok_or_else(|| {
3137 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3138 })?;
3139 self.assembler.emit_popcnt(
3140 Size::S32,
3141 Location::GPR(tmp),
3142 Location::GPR(out_tmp),
3143 )?;
3144 self.move_location(Size::S32, Location::GPR(out_tmp), ret)?;
3145 self.release_gpr(out_tmp);
3146 } else {
3147 self.assembler
3148 .emit_popcnt(Size::S32, Location::GPR(tmp), ret)?;
3149 }
3150 self.release_gpr(tmp);
3151 }
3152 Location::Memory(_, _) | Location::GPR(_) => {
3153 if let Location::Memory(_, _) = ret {
3154 let out_tmp = self.acquire_temp_gpr().ok_or_else(|| {
3155 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3156 })?;
3157 self.assembler
3158 .emit_popcnt(Size::S32, loc, Location::GPR(out_tmp))?;
3159 self.move_location(Size::S32, Location::GPR(out_tmp), ret)?;
3160 self.release_gpr(out_tmp);
3161 } else {
3162 self.assembler.emit_popcnt(Size::S32, loc, ret)?;
3163 }
3164 }
3165 _ => {
3166 codegen_error!("singlepass i32_popcnt unreachable");
3167 }
3168 }
3169 Ok(())
3170 }
3171
3172 fn i32_shl(
3173 &mut self,
3174 loc_a: Location,
3175 loc_b: Location,
3176 ret: Location,
3177 ) -> Result<(), CompileError> {
3178 self.emit_shift_i32(AssemblerX64::emit_shl, loc_a, loc_b, ret)
3179 }
3180
3181 fn i32_shr(
3182 &mut self,
3183 loc_a: Location,
3184 loc_b: Location,
3185 ret: Location,
3186 ) -> Result<(), CompileError> {
3187 self.emit_shift_i32(AssemblerX64::emit_shr, loc_a, loc_b, ret)
3188 }
3189
3190 fn i32_sar(
3191 &mut self,
3192 loc_a: Location,
3193 loc_b: Location,
3194 ret: Location,
3195 ) -> Result<(), CompileError> {
3196 self.emit_shift_i32(AssemblerX64::emit_sar, loc_a, loc_b, ret)
3197 }
3198
3199 fn i32_rol(
3200 &mut self,
3201 loc_a: Location,
3202 loc_b: Location,
3203 ret: Location,
3204 ) -> Result<(), CompileError> {
3205 self.emit_shift_i32(AssemblerX64::emit_rol, loc_a, loc_b, ret)
3206 }
3207
3208 fn i32_ror(
3209 &mut self,
3210 loc_a: Location,
3211 loc_b: Location,
3212 ret: Location,
3213 ) -> Result<(), CompileError> {
3214 self.emit_shift_i32(AssemblerX64::emit_ror, loc_a, loc_b, ret)
3215 }
3216
3217 fn i32_load(
3218 &mut self,
3219 addr: Location,
3220 memarg: &MemArg,
3221 ret: Location,
3222 need_check: bool,
3223 imported_memories: bool,
3224 offset: i32,
3225 heap_access_oob: Label,
3226 unaligned_atomic: Label,
3227 ) -> Result<(), CompileError> {
3228 self.memory_op(
3229 addr,
3230 memarg,
3231 false,
3232 4,
3233 need_check,
3234 imported_memories,
3235 offset,
3236 heap_access_oob,
3237 unaligned_atomic,
3238 |this, addr| {
3239 this.emit_relaxed_binop(
3240 AssemblerX64::emit_mov,
3241 Size::S32,
3242 Location::Memory(addr, 0),
3243 ret,
3244 )
3245 },
3246 )
3247 }
3248
3249 fn i32_load_8u(
3250 &mut self,
3251 addr: Location,
3252 memarg: &MemArg,
3253 ret: Location,
3254 need_check: bool,
3255 imported_memories: bool,
3256 offset: i32,
3257 heap_access_oob: Label,
3258 unaligned_atomic: Label,
3259 ) -> Result<(), CompileError> {
3260 self.memory_op(
3261 addr,
3262 memarg,
3263 false,
3264 1,
3265 need_check,
3266 imported_memories,
3267 offset,
3268 heap_access_oob,
3269 unaligned_atomic,
3270 |this, addr| {
3271 this.emit_relaxed_zx_sx(
3272 AssemblerX64::emit_movzx,
3273 Size::S8,
3274 Location::Memory(addr, 0),
3275 Size::S32,
3276 ret,
3277 )
3278 },
3279 )
3280 }
3281
3282 fn i32_load_8s(
3283 &mut self,
3284 addr: Location,
3285 memarg: &MemArg,
3286 ret: Location,
3287 need_check: bool,
3288 imported_memories: bool,
3289 offset: i32,
3290 heap_access_oob: Label,
3291 unaligned_atomic: Label,
3292 ) -> Result<(), CompileError> {
3293 self.memory_op(
3294 addr,
3295 memarg,
3296 false,
3297 1,
3298 need_check,
3299 imported_memories,
3300 offset,
3301 heap_access_oob,
3302 unaligned_atomic,
3303 |this, addr| {
3304 this.emit_relaxed_zx_sx(
3305 AssemblerX64::emit_movsx,
3306 Size::S8,
3307 Location::Memory(addr, 0),
3308 Size::S32,
3309 ret,
3310 )
3311 },
3312 )
3313 }
3314
3315 fn i32_load_16u(
3316 &mut self,
3317 addr: Location,
3318 memarg: &MemArg,
3319 ret: Location,
3320 need_check: bool,
3321 imported_memories: bool,
3322 offset: i32,
3323 heap_access_oob: Label,
3324 unaligned_atomic: Label,
3325 ) -> Result<(), CompileError> {
3326 self.memory_op(
3327 addr,
3328 memarg,
3329 false,
3330 2,
3331 need_check,
3332 imported_memories,
3333 offset,
3334 heap_access_oob,
3335 unaligned_atomic,
3336 |this, addr| {
3337 this.emit_relaxed_zx_sx(
3338 AssemblerX64::emit_movzx,
3339 Size::S16,
3340 Location::Memory(addr, 0),
3341 Size::S32,
3342 ret,
3343 )
3344 },
3345 )
3346 }
3347
3348 fn i32_load_16s(
3349 &mut self,
3350 addr: Location,
3351 memarg: &MemArg,
3352 ret: Location,
3353 need_check: bool,
3354 imported_memories: bool,
3355 offset: i32,
3356 heap_access_oob: Label,
3357 unaligned_atomic: Label,
3358 ) -> Result<(), CompileError> {
3359 self.memory_op(
3360 addr,
3361 memarg,
3362 false,
3363 2,
3364 need_check,
3365 imported_memories,
3366 offset,
3367 heap_access_oob,
3368 unaligned_atomic,
3369 |this, addr| {
3370 this.emit_relaxed_zx_sx(
3371 AssemblerX64::emit_movsx,
3372 Size::S16,
3373 Location::Memory(addr, 0),
3374 Size::S32,
3375 ret,
3376 )
3377 },
3378 )
3379 }
3380
3381 fn i32_atomic_load(
3382 &mut self,
3383 addr: Location,
3384 memarg: &MemArg,
3385 ret: Location,
3386 need_check: bool,
3387 imported_memories: bool,
3388 offset: i32,
3389 heap_access_oob: Label,
3390 unaligned_atomic: Label,
3391 ) -> Result<(), CompileError> {
3392 self.memory_op(
3393 addr,
3394 memarg,
3395 true,
3396 4,
3397 need_check,
3398 imported_memories,
3399 offset,
3400 heap_access_oob,
3401 unaligned_atomic,
3402 |this, addr| this.emit_relaxed_mov(Size::S32, Location::Memory(addr, 0), ret),
3403 )
3404 }
3405
3406 fn i32_atomic_load_8u(
3407 &mut self,
3408 addr: Location,
3409 memarg: &MemArg,
3410 ret: Location,
3411 need_check: bool,
3412 imported_memories: bool,
3413 offset: i32,
3414 heap_access_oob: Label,
3415 unaligned_atomic: Label,
3416 ) -> Result<(), CompileError> {
3417 self.memory_op(
3418 addr,
3419 memarg,
3420 true,
3421 1,
3422 need_check,
3423 imported_memories,
3424 offset,
3425 heap_access_oob,
3426 unaligned_atomic,
3427 |this, addr| {
3428 this.emit_relaxed_zero_extension(
3429 Size::S8,
3430 Location::Memory(addr, 0),
3431 Size::S32,
3432 ret,
3433 )
3434 },
3435 )
3436 }
3437
3438 fn i32_atomic_load_16u(
3439 &mut self,
3440 addr: Location,
3441 memarg: &MemArg,
3442 ret: Location,
3443 need_check: bool,
3444 imported_memories: bool,
3445 offset: i32,
3446 heap_access_oob: Label,
3447 unaligned_atomic: Label,
3448 ) -> Result<(), CompileError> {
3449 self.memory_op(
3450 addr,
3451 memarg,
3452 true,
3453 2,
3454 need_check,
3455 imported_memories,
3456 offset,
3457 heap_access_oob,
3458 unaligned_atomic,
3459 |this, addr| {
3460 this.emit_relaxed_zero_extension(
3461 Size::S16,
3462 Location::Memory(addr, 0),
3463 Size::S32,
3464 ret,
3465 )
3466 },
3467 )
3468 }
3469
3470 fn i32_save(
3471 &mut self,
3472 target_value: Location,
3473 memarg: &MemArg,
3474 target_addr: Location,
3475 need_check: bool,
3476 imported_memories: bool,
3477 offset: i32,
3478 heap_access_oob: Label,
3479 unaligned_atomic: Label,
3480 ) -> Result<(), CompileError> {
3481 self.memory_op(
3482 target_addr,
3483 memarg,
3484 false,
3485 4,
3486 need_check,
3487 imported_memories,
3488 offset,
3489 heap_access_oob,
3490 unaligned_atomic,
3491 |this, addr| {
3492 this.emit_relaxed_binop(
3493 AssemblerX64::emit_mov,
3494 Size::S32,
3495 target_value,
3496 Location::Memory(addr, 0),
3497 )
3498 },
3499 )
3500 }
3501
3502 fn i32_save_8(
3503 &mut self,
3504 target_value: Location,
3505 memarg: &MemArg,
3506 target_addr: Location,
3507 need_check: bool,
3508 imported_memories: bool,
3509 offset: i32,
3510 heap_access_oob: Label,
3511 unaligned_atomic: Label,
3512 ) -> Result<(), CompileError> {
3513 self.memory_op(
3514 target_addr,
3515 memarg,
3516 false,
3517 1,
3518 need_check,
3519 imported_memories,
3520 offset,
3521 heap_access_oob,
3522 unaligned_atomic,
3523 |this, addr| {
3524 this.emit_relaxed_binop(
3525 AssemblerX64::emit_mov,
3526 Size::S8,
3527 target_value,
3528 Location::Memory(addr, 0),
3529 )
3530 },
3531 )
3532 }
3533
3534 fn i32_save_16(
3535 &mut self,
3536 target_value: Location,
3537 memarg: &MemArg,
3538 target_addr: Location,
3539 need_check: bool,
3540 imported_memories: bool,
3541 offset: i32,
3542 heap_access_oob: Label,
3543 unaligned_atomic: Label,
3544 ) -> Result<(), CompileError> {
3545 self.memory_op(
3546 target_addr,
3547 memarg,
3548 false,
3549 2,
3550 need_check,
3551 imported_memories,
3552 offset,
3553 heap_access_oob,
3554 unaligned_atomic,
3555 |this, addr| {
3556 this.emit_relaxed_binop(
3557 AssemblerX64::emit_mov,
3558 Size::S16,
3559 target_value,
3560 Location::Memory(addr, 0),
3561 )
3562 },
3563 )
3564 }
3565
3566 fn i32_atomic_save(
3567 &mut self,
3568 value: Location,
3569 memarg: &MemArg,
3570 target_addr: Location,
3571 need_check: bool,
3572 imported_memories: bool,
3573 offset: i32,
3574 heap_access_oob: Label,
3575 unaligned_atomic: Label,
3576 ) -> Result<(), CompileError> {
3577 self.memory_op(
3578 target_addr,
3579 memarg,
3580 true,
3581 4,
3582 need_check,
3583 imported_memories,
3584 offset,
3585 heap_access_oob,
3586 unaligned_atomic,
3587 |this, addr| {
3588 this.emit_relaxed_binop(
3589 AssemblerX64::emit_mov,
3590 Size::S32,
3591 value,
3592 Location::Memory(addr, 0),
3593 )
3594 },
3595 )
3596 }
3597
3598 fn i32_atomic_save_8(
3599 &mut self,
3600 value: Location,
3601 memarg: &MemArg,
3602 target_addr: Location,
3603 need_check: bool,
3604 imported_memories: bool,
3605 offset: i32,
3606 heap_access_oob: Label,
3607 unaligned_atomic: Label,
3608 ) -> Result<(), CompileError> {
3609 self.memory_op(
3610 target_addr,
3611 memarg,
3612 true,
3613 1,
3614 need_check,
3615 imported_memories,
3616 offset,
3617 heap_access_oob,
3618 unaligned_atomic,
3619 |this, addr| {
3620 this.emit_relaxed_binop(
3621 AssemblerX64::emit_mov,
3622 Size::S8,
3623 value,
3624 Location::Memory(addr, 0),
3625 )
3626 },
3627 )
3628 }
3629
3630 fn i32_atomic_save_16(
3631 &mut self,
3632 value: Location,
3633 memarg: &MemArg,
3634 target_addr: Location,
3635 need_check: bool,
3636 imported_memories: bool,
3637 offset: i32,
3638 heap_access_oob: Label,
3639 unaligned_atomic: Label,
3640 ) -> Result<(), CompileError> {
3641 self.memory_op(
3642 target_addr,
3643 memarg,
3644 true,
3645 2,
3646 need_check,
3647 imported_memories,
3648 offset,
3649 heap_access_oob,
3650 unaligned_atomic,
3651 |this, addr| {
3652 this.emit_relaxed_binop(
3653 AssemblerX64::emit_mov,
3654 Size::S16,
3655 value,
3656 Location::Memory(addr, 0),
3657 )
3658 },
3659 )
3660 }
3661
3662 fn i32_atomic_add(
3663 &mut self,
3664 loc: Location,
3665 target: Location,
3666 memarg: &MemArg,
3667 ret: Location,
3668 need_check: bool,
3669 imported_memories: bool,
3670 offset: i32,
3671 heap_access_oob: Label,
3672 unaligned_atomic: Label,
3673 ) -> Result<(), CompileError> {
3674 let value = self.acquire_temp_gpr().ok_or_else(|| {
3675 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3676 })?;
3677 self.move_location(Size::S32, loc, Location::GPR(value))?;
3678 self.memory_op(
3679 target,
3680 memarg,
3681 true,
3682 4,
3683 need_check,
3684 imported_memories,
3685 offset,
3686 heap_access_oob,
3687 unaligned_atomic,
3688 |this, addr| {
3689 this.assembler.emit_lock_xadd(
3690 Size::S32,
3691 Location::GPR(value),
3692 Location::Memory(addr, 0),
3693 )
3694 },
3695 )?;
3696 self.move_location(Size::S32, Location::GPR(value), ret)?;
3697 self.release_gpr(value);
3698 Ok(())
3699 }
3700
3701 fn i32_atomic_add_8u(
3702 &mut self,
3703 loc: Location,
3704 target: Location,
3705 memarg: &MemArg,
3706 ret: Location,
3707 need_check: bool,
3708 imported_memories: bool,
3709 offset: i32,
3710 heap_access_oob: Label,
3711 unaligned_atomic: Label,
3712 ) -> Result<(), CompileError> {
3713 let value = self.acquire_temp_gpr().ok_or_else(|| {
3714 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3715 })?;
3716 self.move_location_extend(Size::S8, false, loc, Size::S32, Location::GPR(value))?;
3717 self.memory_op(
3718 target,
3719 memarg,
3720 true,
3721 1,
3722 need_check,
3723 imported_memories,
3724 offset,
3725 heap_access_oob,
3726 unaligned_atomic,
3727 |this, addr| {
3728 this.assembler.emit_lock_xadd(
3729 Size::S8,
3730 Location::GPR(value),
3731 Location::Memory(addr, 0),
3732 )
3733 },
3734 )?;
3735 self.move_location_extend(
3737 Size::S8,
3738 false,
3739 Location::GPR(value),
3740 Size::S32,
3741 Location::GPR(value),
3742 )?;
3743 self.move_location(Size::S32, Location::GPR(value), ret)?;
3744 self.release_gpr(value);
3745 Ok(())
3746 }
3747
3748 fn i32_atomic_add_16u(
3749 &mut self,
3750 loc: Location,
3751 target: Location,
3752 memarg: &MemArg,
3753 ret: Location,
3754 need_check: bool,
3755 imported_memories: bool,
3756 offset: i32,
3757 heap_access_oob: Label,
3758 unaligned_atomic: Label,
3759 ) -> Result<(), CompileError> {
3760 let value = self.acquire_temp_gpr().ok_or_else(|| {
3761 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3762 })?;
3763 self.move_location_extend(Size::S16, false, loc, Size::S32, Location::GPR(value))?;
3764 self.memory_op(
3765 target,
3766 memarg,
3767 true,
3768 2,
3769 need_check,
3770 imported_memories,
3771 offset,
3772 heap_access_oob,
3773 unaligned_atomic,
3774 |this, addr| {
3775 this.assembler.emit_lock_xadd(
3776 Size::S16,
3777 Location::GPR(value),
3778 Location::Memory(addr, 0),
3779 )
3780 },
3781 )?;
3782 self.move_location_extend(
3784 Size::S16,
3785 false,
3786 Location::GPR(value),
3787 Size::S32,
3788 Location::GPR(value),
3789 )?;
3790 self.move_location(Size::S32, Location::GPR(value), ret)?;
3791 self.release_gpr(value);
3792 Ok(())
3793 }
3794
3795 fn i32_atomic_sub(
3796 &mut self,
3797 loc: Location,
3798 target: Location,
3799 memarg: &MemArg,
3800 ret: Location,
3801 need_check: bool,
3802 imported_memories: bool,
3803 offset: i32,
3804 heap_access_oob: Label,
3805 unaligned_atomic: Label,
3806 ) -> Result<(), CompileError> {
3807 let value = self.acquire_temp_gpr().ok_or_else(|| {
3808 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3809 })?;
3810 self.location_neg(Size::S32, false, loc, Size::S32, Location::GPR(value))?;
3811 self.memory_op(
3812 target,
3813 memarg,
3814 true,
3815 4,
3816 need_check,
3817 imported_memories,
3818 offset,
3819 heap_access_oob,
3820 unaligned_atomic,
3821 |this, addr| {
3822 this.assembler.emit_lock_xadd(
3823 Size::S32,
3824 Location::GPR(value),
3825 Location::Memory(addr, 0),
3826 )
3827 },
3828 )?;
3829 self.move_location(Size::S32, Location::GPR(value), ret)?;
3830 self.release_gpr(value);
3831 Ok(())
3832 }
3833
3834 fn i32_atomic_sub_8u(
3835 &mut self,
3836 loc: Location,
3837 target: Location,
3838 memarg: &MemArg,
3839 ret: Location,
3840 need_check: bool,
3841 imported_memories: bool,
3842 offset: i32,
3843 heap_access_oob: Label,
3844 unaligned_atomic: Label,
3845 ) -> Result<(), CompileError> {
3846 let value = self.acquire_temp_gpr().ok_or_else(|| {
3847 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3848 })?;
3849 self.location_neg(Size::S8, false, loc, Size::S32, Location::GPR(value))?;
3850 self.memory_op(
3851 target,
3852 memarg,
3853 true,
3854 1,
3855 need_check,
3856 imported_memories,
3857 offset,
3858 heap_access_oob,
3859 unaligned_atomic,
3860 |this, addr| {
3861 this.assembler.emit_lock_xadd(
3862 Size::S8,
3863 Location::GPR(value),
3864 Location::Memory(addr, 0),
3865 )
3866 },
3867 )?;
3868 self.move_location_extend(
3870 Size::S8,
3871 false,
3872 Location::GPR(value),
3873 Size::S32,
3874 Location::GPR(value),
3875 )?;
3876 self.move_location(Size::S32, Location::GPR(value), ret)?;
3877 self.release_gpr(value);
3878 Ok(())
3879 }
3880
3881 fn i32_atomic_sub_16u(
3882 &mut self,
3883 loc: Location,
3884 target: Location,
3885 memarg: &MemArg,
3886 ret: Location,
3887 need_check: bool,
3888 imported_memories: bool,
3889 offset: i32,
3890 heap_access_oob: Label,
3891 unaligned_atomic: Label,
3892 ) -> Result<(), CompileError> {
3893 let value = self.acquire_temp_gpr().ok_or_else(|| {
3894 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
3895 })?;
3896 self.location_neg(Size::S16, false, loc, Size::S32, Location::GPR(value))?;
3897 self.memory_op(
3898 target,
3899 memarg,
3900 true,
3901 2,
3902 need_check,
3903 imported_memories,
3904 offset,
3905 heap_access_oob,
3906 unaligned_atomic,
3907 |this, addr| {
3908 this.assembler.emit_lock_xadd(
3909 Size::S16,
3910 Location::GPR(value),
3911 Location::Memory(addr, 0),
3912 )
3913 },
3914 )?;
3915 self.move_location_extend(
3917 Size::S16,
3918 false,
3919 Location::GPR(value),
3920 Size::S32,
3921 Location::GPR(value),
3922 )?;
3923 self.move_location(Size::S32, Location::GPR(value), ret)?;
3924 self.release_gpr(value);
3925 Ok(())
3926 }
3927
3928 fn i32_atomic_and(
3929 &mut self,
3930 loc: Location,
3931 target: Location,
3932 memarg: &MemArg,
3933 ret: Location,
3934 need_check: bool,
3935 imported_memories: bool,
3936 offset: i32,
3937 heap_access_oob: Label,
3938 unaligned_atomic: Label,
3939 ) -> Result<(), CompileError> {
3940 self.emit_compare_and_swap(
3941 loc,
3942 target,
3943 ret,
3944 memarg,
3945 4,
3946 Size::S32,
3947 Size::S32,
3948 need_check,
3949 imported_memories,
3950 offset,
3951 heap_access_oob,
3952 unaligned_atomic,
3953 |this, src, dst| {
3954 this.assembler
3955 .emit_and(Size::S32, Location::GPR(src), Location::GPR(dst))
3956 },
3957 )
3958 }
3959
3960 fn i32_atomic_and_8u(
3961 &mut self,
3962 loc: Location,
3963 target: Location,
3964 memarg: &MemArg,
3965 ret: Location,
3966 need_check: bool,
3967 imported_memories: bool,
3968 offset: i32,
3969 heap_access_oob: Label,
3970 unaligned_atomic: Label,
3971 ) -> Result<(), CompileError> {
3972 self.emit_compare_and_swap(
3973 loc,
3974 target,
3975 ret,
3976 memarg,
3977 1,
3978 Size::S8,
3979 Size::S32,
3980 need_check,
3981 imported_memories,
3982 offset,
3983 heap_access_oob,
3984 unaligned_atomic,
3985 |this, src, dst| {
3986 this.assembler
3987 .emit_and(Size::S32, Location::GPR(src), Location::GPR(dst))
3988 },
3989 )
3990 }
3991
3992 fn i32_atomic_and_16u(
3993 &mut self,
3994 loc: Location,
3995 target: Location,
3996 memarg: &MemArg,
3997 ret: Location,
3998 need_check: bool,
3999 imported_memories: bool,
4000 offset: i32,
4001 heap_access_oob: Label,
4002 unaligned_atomic: Label,
4003 ) -> Result<(), CompileError> {
4004 self.emit_compare_and_swap(
4005 loc,
4006 target,
4007 ret,
4008 memarg,
4009 2,
4010 Size::S16,
4011 Size::S32,
4012 need_check,
4013 imported_memories,
4014 offset,
4015 heap_access_oob,
4016 unaligned_atomic,
4017 |this, src, dst| {
4018 this.assembler
4019 .emit_and(Size::S32, Location::GPR(src), Location::GPR(dst))
4020 },
4021 )
4022 }
4023
4024 fn i32_atomic_or(
4025 &mut self,
4026 loc: Location,
4027 target: Location,
4028 memarg: &MemArg,
4029 ret: Location,
4030 need_check: bool,
4031 imported_memories: bool,
4032 offset: i32,
4033 heap_access_oob: Label,
4034 unaligned_atomic: Label,
4035 ) -> Result<(), CompileError> {
4036 self.emit_compare_and_swap(
4037 loc,
4038 target,
4039 ret,
4040 memarg,
4041 4,
4042 Size::S32,
4043 Size::S32,
4044 need_check,
4045 imported_memories,
4046 offset,
4047 heap_access_oob,
4048 unaligned_atomic,
4049 |this, src, dst| {
4050 this.assembler
4051 .emit_or(Size::S32, Location::GPR(src), Location::GPR(dst))
4052 },
4053 )
4054 }
4055
4056 fn i32_atomic_or_8u(
4057 &mut self,
4058 loc: Location,
4059 target: Location,
4060 memarg: &MemArg,
4061 ret: Location,
4062 need_check: bool,
4063 imported_memories: bool,
4064 offset: i32,
4065 heap_access_oob: Label,
4066 unaligned_atomic: Label,
4067 ) -> Result<(), CompileError> {
4068 self.emit_compare_and_swap(
4069 loc,
4070 target,
4071 ret,
4072 memarg,
4073 1,
4074 Size::S8,
4075 Size::S32,
4076 need_check,
4077 imported_memories,
4078 offset,
4079 heap_access_oob,
4080 unaligned_atomic,
4081 |this, src, dst| {
4082 this.assembler
4083 .emit_or(Size::S32, Location::GPR(src), Location::GPR(dst))
4084 },
4085 )
4086 }
4087
4088 fn i32_atomic_or_16u(
4089 &mut self,
4090 loc: Location,
4091 target: Location,
4092 memarg: &MemArg,
4093 ret: Location,
4094 need_check: bool,
4095 imported_memories: bool,
4096 offset: i32,
4097 heap_access_oob: Label,
4098 unaligned_atomic: Label,
4099 ) -> Result<(), CompileError> {
4100 self.emit_compare_and_swap(
4101 loc,
4102 target,
4103 ret,
4104 memarg,
4105 2,
4106 Size::S16,
4107 Size::S32,
4108 need_check,
4109 imported_memories,
4110 offset,
4111 heap_access_oob,
4112 unaligned_atomic,
4113 |this, src, dst| {
4114 this.assembler
4115 .emit_or(Size::S32, Location::GPR(src), Location::GPR(dst))
4116 },
4117 )
4118 }
4119
4120 fn i32_atomic_xor(
4121 &mut self,
4122 loc: Location,
4123 target: Location,
4124 memarg: &MemArg,
4125 ret: Location,
4126 need_check: bool,
4127 imported_memories: bool,
4128 offset: i32,
4129 heap_access_oob: Label,
4130 unaligned_atomic: Label,
4131 ) -> Result<(), CompileError> {
4132 self.emit_compare_and_swap(
4133 loc,
4134 target,
4135 ret,
4136 memarg,
4137 4,
4138 Size::S32,
4139 Size::S32,
4140 need_check,
4141 imported_memories,
4142 offset,
4143 heap_access_oob,
4144 unaligned_atomic,
4145 |this, src, dst| {
4146 this.assembler
4147 .emit_xor(Size::S32, Location::GPR(src), Location::GPR(dst))
4148 },
4149 )
4150 }
4151
4152 fn i32_atomic_xor_8u(
4153 &mut self,
4154 loc: Location,
4155 target: Location,
4156 memarg: &MemArg,
4157 ret: Location,
4158 need_check: bool,
4159 imported_memories: bool,
4160 offset: i32,
4161 heap_access_oob: Label,
4162 unaligned_atomic: Label,
4163 ) -> Result<(), CompileError> {
4164 self.emit_compare_and_swap(
4165 loc,
4166 target,
4167 ret,
4168 memarg,
4169 1,
4170 Size::S8,
4171 Size::S32,
4172 need_check,
4173 imported_memories,
4174 offset,
4175 heap_access_oob,
4176 unaligned_atomic,
4177 |this, src, dst| {
4178 this.assembler
4179 .emit_xor(Size::S32, Location::GPR(src), Location::GPR(dst))
4180 },
4181 )
4182 }
4183
4184 fn i32_atomic_xor_16u(
4185 &mut self,
4186 loc: Location,
4187 target: Location,
4188 memarg: &MemArg,
4189 ret: Location,
4190 need_check: bool,
4191 imported_memories: bool,
4192 offset: i32,
4193 heap_access_oob: Label,
4194 unaligned_atomic: Label,
4195 ) -> Result<(), CompileError> {
4196 self.emit_compare_and_swap(
4197 loc,
4198 target,
4199 ret,
4200 memarg,
4201 2,
4202 Size::S16,
4203 Size::S32,
4204 need_check,
4205 imported_memories,
4206 offset,
4207 heap_access_oob,
4208 unaligned_atomic,
4209 |this, src, dst| {
4210 this.assembler
4211 .emit_xor(Size::S32, Location::GPR(src), Location::GPR(dst))
4212 },
4213 )
4214 }
4215
4216 fn i32_atomic_xchg(
4217 &mut self,
4218 loc: Location,
4219 target: Location,
4220 memarg: &MemArg,
4221 ret: Location,
4222 need_check: bool,
4223 imported_memories: bool,
4224 offset: i32,
4225 heap_access_oob: Label,
4226 unaligned_atomic: Label,
4227 ) -> Result<(), CompileError> {
4228 let value = self.acquire_temp_gpr().ok_or_else(|| {
4229 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4230 })?;
4231 self.move_location(Size::S32, loc, Location::GPR(value))?;
4232 self.memory_op(
4233 target,
4234 memarg,
4235 true,
4236 4,
4237 need_check,
4238 imported_memories,
4239 offset,
4240 heap_access_oob,
4241 unaligned_atomic,
4242 |this, addr| {
4243 this.assembler
4244 .emit_xchg(Size::S32, Location::GPR(value), Location::Memory(addr, 0))
4245 },
4246 )?;
4247 self.move_location(Size::S32, Location::GPR(value), ret)?;
4248 self.release_gpr(value);
4249 Ok(())
4250 }
4251
4252 fn i32_atomic_xchg_8u(
4253 &mut self,
4254 loc: Location,
4255 target: Location,
4256 memarg: &MemArg,
4257 ret: Location,
4258 need_check: bool,
4259 imported_memories: bool,
4260 offset: i32,
4261 heap_access_oob: Label,
4262 unaligned_atomic: Label,
4263 ) -> Result<(), CompileError> {
4264 let value = self.acquire_temp_gpr().ok_or_else(|| {
4265 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4266 })?;
4267 self.assembler
4268 .emit_movzx(Size::S8, loc, Size::S32, Location::GPR(value))?;
4269 self.memory_op(
4270 target,
4271 memarg,
4272 true,
4273 1,
4274 need_check,
4275 imported_memories,
4276 offset,
4277 heap_access_oob,
4278 unaligned_atomic,
4279 |this, addr| {
4280 this.assembler
4281 .emit_xchg(Size::S8, Location::GPR(value), Location::Memory(addr, 0))
4282 },
4283 )?;
4284 self.move_location_extend(
4286 Size::S8,
4287 false,
4288 Location::GPR(value),
4289 Size::S32,
4290 Location::GPR(value),
4291 )?;
4292 self.move_location(Size::S32, Location::GPR(value), ret)?;
4293 self.release_gpr(value);
4294 Ok(())
4295 }
4296
4297 fn i32_atomic_xchg_16u(
4298 &mut self,
4299 loc: Location,
4300 target: Location,
4301 memarg: &MemArg,
4302 ret: Location,
4303 need_check: bool,
4304 imported_memories: bool,
4305 offset: i32,
4306 heap_access_oob: Label,
4307 unaligned_atomic: Label,
4308 ) -> Result<(), CompileError> {
4309 let value = self.acquire_temp_gpr().ok_or_else(|| {
4310 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4311 })?;
4312 self.assembler
4313 .emit_movzx(Size::S16, loc, Size::S32, Location::GPR(value))?;
4314 self.memory_op(
4315 target,
4316 memarg,
4317 true,
4318 2,
4319 need_check,
4320 imported_memories,
4321 offset,
4322 heap_access_oob,
4323 unaligned_atomic,
4324 |this, addr| {
4325 this.assembler
4326 .emit_xchg(Size::S16, Location::GPR(value), Location::Memory(addr, 0))
4327 },
4328 )?;
4329 self.move_location_extend(
4331 Size::S16,
4332 false,
4333 Location::GPR(value),
4334 Size::S32,
4335 Location::GPR(value),
4336 )?;
4337 self.move_location(Size::S32, Location::GPR(value), ret)?;
4338 self.release_gpr(value);
4339 Ok(())
4340 }
4341
4342 fn i32_atomic_cmpxchg(
4343 &mut self,
4344 new: Location,
4345 cmp: Location,
4346 target: Location,
4347 memarg: &MemArg,
4348 ret: Location,
4349 need_check: bool,
4350 imported_memories: bool,
4351 offset: i32,
4352 heap_access_oob: Label,
4353 unaligned_atomic: Label,
4354 ) -> Result<(), CompileError> {
4355 let compare = self.reserve_unused_temp_gpr(GPR::RAX);
4356 let value = if cmp == Location::GPR(GPR::R14) {
4357 if new == Location::GPR(GPR::R13) {
4358 GPR::R12
4359 } else {
4360 GPR::R13
4361 }
4362 } else {
4363 GPR::R14
4364 };
4365 self.assembler.emit_push(Size::S64, Location::GPR(value))?;
4366 self.assembler
4367 .emit_mov(Size::S32, cmp, Location::GPR(compare))?;
4368 self.assembler
4369 .emit_mov(Size::S32, new, Location::GPR(value))?;
4370
4371 self.memory_op(
4372 target,
4373 memarg,
4374 true,
4375 4,
4376 need_check,
4377 imported_memories,
4378 offset,
4379 heap_access_oob,
4380 unaligned_atomic,
4381 |this, addr| {
4382 this.assembler.emit_lock_cmpxchg(
4383 Size::S32,
4384 Location::GPR(value),
4385 Location::Memory(addr, 0),
4386 )?;
4387 this.assembler
4388 .emit_mov(Size::S32, Location::GPR(compare), ret)
4389 },
4390 )?;
4391 self.assembler.emit_pop(Size::S64, Location::GPR(value))?;
4392 self.release_gpr(compare);
4393 Ok(())
4394 }
4395
4396 fn i32_atomic_cmpxchg_8u(
4397 &mut self,
4398 new: Location,
4399 cmp: Location,
4400 target: Location,
4401 memarg: &MemArg,
4402 ret: Location,
4403 need_check: bool,
4404 imported_memories: bool,
4405 offset: i32,
4406 heap_access_oob: Label,
4407 unaligned_atomic: Label,
4408 ) -> Result<(), CompileError> {
4409 let compare = self.reserve_unused_temp_gpr(GPR::RAX);
4410 let value = if cmp == Location::GPR(GPR::R14) {
4411 if new == Location::GPR(GPR::R13) {
4412 GPR::R12
4413 } else {
4414 GPR::R13
4415 }
4416 } else {
4417 GPR::R14
4418 };
4419 self.assembler.emit_push(Size::S64, Location::GPR(value))?;
4420 self.assembler
4421 .emit_mov(Size::S32, cmp, Location::GPR(compare))?;
4422 self.assembler
4423 .emit_mov(Size::S32, new, Location::GPR(value))?;
4424
4425 self.memory_op(
4426 target,
4427 memarg,
4428 true,
4429 1,
4430 need_check,
4431 imported_memories,
4432 offset,
4433 heap_access_oob,
4434 unaligned_atomic,
4435 |this, addr| {
4436 this.assembler.emit_lock_cmpxchg(
4437 Size::S8,
4438 Location::GPR(value),
4439 Location::Memory(addr, 0),
4440 )?;
4441 this.move_location_extend(
4442 Size::S8,
4443 false,
4444 Location::GPR(compare),
4445 Size::S32,
4446 Location::GPR(compare),
4447 )?;
4448 this.move_location(Size::S32, Location::GPR(compare), ret)
4449 },
4450 )?;
4451 self.assembler.emit_pop(Size::S64, Location::GPR(value))?;
4452 self.release_gpr(compare);
4453 Ok(())
4454 }
4455
4456 fn i32_atomic_cmpxchg_16u(
4457 &mut self,
4458 new: Location,
4459 cmp: Location,
4460 target: Location,
4461 memarg: &MemArg,
4462 ret: Location,
4463 need_check: bool,
4464 imported_memories: bool,
4465 offset: i32,
4466 heap_access_oob: Label,
4467 unaligned_atomic: Label,
4468 ) -> Result<(), CompileError> {
4469 let compare = self.reserve_unused_temp_gpr(GPR::RAX);
4470 let value = if cmp == Location::GPR(GPR::R14) {
4471 if new == Location::GPR(GPR::R13) {
4472 GPR::R12
4473 } else {
4474 GPR::R13
4475 }
4476 } else {
4477 GPR::R14
4478 };
4479 self.assembler.emit_push(Size::S64, Location::GPR(value))?;
4480 self.assembler
4481 .emit_mov(Size::S32, cmp, Location::GPR(compare))?;
4482 self.assembler
4483 .emit_mov(Size::S32, new, Location::GPR(value))?;
4484
4485 self.memory_op(
4486 target,
4487 memarg,
4488 true,
4489 2,
4490 need_check,
4491 imported_memories,
4492 offset,
4493 heap_access_oob,
4494 unaligned_atomic,
4495 |this, addr| {
4496 this.assembler.emit_lock_cmpxchg(
4497 Size::S16,
4498 Location::GPR(value),
4499 Location::Memory(addr, 0),
4500 )?;
4501 this.move_location_extend(
4502 Size::S16,
4503 false,
4504 Location::GPR(compare),
4505 Size::S32,
4506 Location::GPR(compare),
4507 )?;
4508 this.move_location(Size::S32, Location::GPR(compare), ret)
4509 },
4510 )?;
4511 self.assembler.emit_pop(Size::S64, Location::GPR(value))?;
4512 self.release_gpr(compare);
4513 Ok(())
4514 }
4515
4516 fn emit_call_with_reloc(
4517 &mut self,
4518 reloc_target: RelocationTarget,
4519 ) -> Result<Vec<Relocation>, CompileError> {
4520 let mut relocations = vec![];
4521 let next = self.get_label();
4522 let reloc_at = self.assembler.get_offset().0 + 1; self.assembler.emit_call_label(next)?;
4524 self.emit_label(next)?;
4525 relocations.push(Relocation {
4526 kind: RelocationKind::X86CallPCRel4,
4527 reloc_target,
4528 offset: reloc_at as u32,
4529 addend: -4,
4530 });
4531 Ok(relocations)
4532 }
4533
4534 fn emit_binop_add64(
4535 &mut self,
4536 loc_a: Location,
4537 loc_b: Location,
4538 ret: Location,
4539 ) -> Result<(), CompileError> {
4540 self.emit_binop_i64(AssemblerX64::emit_add, loc_a, loc_b, ret)
4541 }
4542
4543 fn emit_binop_sub64(
4544 &mut self,
4545 loc_a: Location,
4546 loc_b: Location,
4547 ret: Location,
4548 ) -> Result<(), CompileError> {
4549 self.emit_binop_i64(AssemblerX64::emit_sub, loc_a, loc_b, ret)
4550 }
4551
4552 fn emit_binop_mul64(
4553 &mut self,
4554 loc_a: Location,
4555 loc_b: Location,
4556 ret: Location,
4557 ) -> Result<(), CompileError> {
4558 self.emit_binop_i64(AssemblerX64::emit_imul, loc_a, loc_b, ret)
4559 }
4560
4561 fn emit_binop_udiv64(
4562 &mut self,
4563 loc_a: Location,
4564 loc_b: Location,
4565 ret: Location,
4566 integer_division_by_zero: Label,
4567 ) -> Result<usize, CompileError> {
4568 for reg in DIV_REM_REGISTERS {
4569 self.reserve_unused_temp_gpr(reg);
4570 }
4571 self.assembler
4572 .emit_mov(Size::S64, loc_a, Location::GPR(GPR::RAX))?;
4573 self.assembler
4574 .emit_xor(Size::S64, Location::GPR(GPR::RDX), Location::GPR(GPR::RDX))?;
4575 let offset = self.emit_relaxed_xdiv(
4576 AssemblerX64::emit_div,
4577 Size::S64,
4578 loc_b,
4579 integer_division_by_zero,
4580 )?;
4581 self.assembler
4582 .emit_mov(Size::S64, Location::GPR(GPR::RAX), ret)?;
4583 for reg in DIV_REM_REGISTERS {
4584 self.release_gpr(reg);
4585 }
4586 Ok(offset)
4587 }
4588
4589 fn emit_binop_sdiv64(
4590 &mut self,
4591 loc_a: Location,
4592 loc_b: Location,
4593 ret: Location,
4594 integer_division_by_zero: Label,
4595 _integer_overflow: Label,
4596 ) -> Result<usize, CompileError> {
4597 for reg in DIV_REM_REGISTERS {
4598 self.reserve_unused_temp_gpr(reg);
4599 }
4600 self.assembler
4601 .emit_mov(Size::S64, loc_a, Location::GPR(GPR::RAX))?;
4602 self.assembler.emit_cqo()?;
4603 let offset = self.emit_relaxed_xdiv(
4604 AssemblerX64::emit_idiv,
4605 Size::S64,
4606 loc_b,
4607 integer_division_by_zero,
4608 )?;
4609 self.assembler
4610 .emit_mov(Size::S64, Location::GPR(GPR::RAX), ret)?;
4611 for reg in DIV_REM_REGISTERS {
4612 self.release_gpr(reg);
4613 }
4614 Ok(offset)
4615 }
4616
4617 fn emit_binop_urem64(
4618 &mut self,
4619 loc_a: Location,
4620 loc_b: Location,
4621 ret: Location,
4622 integer_division_by_zero: Label,
4623 ) -> Result<usize, CompileError> {
4624 for reg in DIV_REM_REGISTERS {
4625 self.reserve_unused_temp_gpr(reg);
4626 }
4627 self.assembler
4628 .emit_mov(Size::S64, loc_a, Location::GPR(GPR::RAX))?;
4629 self.assembler
4630 .emit_xor(Size::S64, Location::GPR(GPR::RDX), Location::GPR(GPR::RDX))?;
4631 let offset = self.emit_relaxed_xdiv(
4632 AssemblerX64::emit_div,
4633 Size::S64,
4634 loc_b,
4635 integer_division_by_zero,
4636 )?;
4637 self.assembler
4638 .emit_mov(Size::S64, Location::GPR(GPR::RDX), ret)?;
4639 for reg in DIV_REM_REGISTERS {
4640 self.release_gpr(reg);
4641 }
4642 Ok(offset)
4643 }
4644
4645 fn emit_binop_srem64(
4646 &mut self,
4647 loc_a: Location,
4648 loc_b: Location,
4649 ret: Location,
4650 integer_division_by_zero: Label,
4651 ) -> Result<usize, CompileError> {
4652 for reg in DIV_REM_REGISTERS {
4653 self.reserve_unused_temp_gpr(reg);
4654 }
4655 let normal_path = self.assembler.get_label();
4656 let end = self.assembler.get_label();
4657
4658 self.emit_relaxed_cmp(Size::S64, Location::Imm64(0x8000000000000000u64), loc_a)?;
4659 self.assembler.emit_jmp(Condition::NotEqual, normal_path)?;
4660 self.emit_relaxed_cmp(Size::S64, Location::Imm64(0xffffffffffffffffu64), loc_b)?;
4661 self.assembler.emit_jmp(Condition::NotEqual, normal_path)?;
4662 self.move_location(Size::S64, Location::Imm64(0), ret)?;
4663 self.assembler.emit_jmp(Condition::None, end)?;
4664
4665 self.emit_label(normal_path)?;
4666 self.assembler
4667 .emit_mov(Size::S64, loc_a, Location::GPR(GPR::RAX))?;
4668 self.assembler.emit_cqo()?;
4669 let offset = self.emit_relaxed_xdiv(
4670 AssemblerX64::emit_idiv,
4671 Size::S64,
4672 loc_b,
4673 integer_division_by_zero,
4674 )?;
4675 self.assembler
4676 .emit_mov(Size::S64, Location::GPR(GPR::RDX), ret)?;
4677
4678 self.emit_label(end)?;
4679 for reg in DIV_REM_REGISTERS {
4680 self.release_gpr(reg);
4681 }
4682 Ok(offset)
4683 }
4684
4685 fn emit_binop_and64(
4686 &mut self,
4687 loc_a: Location,
4688 loc_b: Location,
4689 ret: Location,
4690 ) -> Result<(), CompileError> {
4691 self.emit_binop_i64(AssemblerX64::emit_and, loc_a, loc_b, ret)
4692 }
4693
4694 fn emit_binop_or64(
4695 &mut self,
4696 loc_a: Location,
4697 loc_b: Location,
4698 ret: Location,
4699 ) -> Result<(), CompileError> {
4700 self.emit_binop_i64(AssemblerX64::emit_or, loc_a, loc_b, ret)
4701 }
4702
4703 fn emit_binop_xor64(
4704 &mut self,
4705 loc_a: Location,
4706 loc_b: Location,
4707 ret: Location,
4708 ) -> Result<(), CompileError> {
4709 self.emit_binop_i64(AssemblerX64::emit_xor, loc_a, loc_b, ret)
4710 }
4711
4712 fn i64_cmp_ge_s(
4713 &mut self,
4714 loc_a: Location,
4715 loc_b: Location,
4716 ret: Location,
4717 ) -> Result<(), CompileError> {
4718 self.emit_cmpop_i64_dynamic_b(Condition::GreaterEqual, loc_a, loc_b, ret)
4719 }
4720
4721 fn i64_cmp_gt_s(
4722 &mut self,
4723 loc_a: Location,
4724 loc_b: Location,
4725 ret: Location,
4726 ) -> Result<(), CompileError> {
4727 self.emit_cmpop_i64_dynamic_b(Condition::Greater, loc_a, loc_b, ret)
4728 }
4729
4730 fn i64_cmp_le_s(
4731 &mut self,
4732 loc_a: Location,
4733 loc_b: Location,
4734 ret: Location,
4735 ) -> Result<(), CompileError> {
4736 self.emit_cmpop_i64_dynamic_b(Condition::LessEqual, loc_a, loc_b, ret)
4737 }
4738
4739 fn i64_cmp_lt_s(
4740 &mut self,
4741 loc_a: Location,
4742 loc_b: Location,
4743 ret: Location,
4744 ) -> Result<(), CompileError> {
4745 self.emit_cmpop_i64_dynamic_b(Condition::Less, loc_a, loc_b, ret)
4746 }
4747
4748 fn i64_cmp_ge_u(
4749 &mut self,
4750 loc_a: Location,
4751 loc_b: Location,
4752 ret: Location,
4753 ) -> Result<(), CompileError> {
4754 self.emit_cmpop_i64_dynamic_b(Condition::AboveEqual, loc_a, loc_b, ret)
4755 }
4756
4757 fn i64_cmp_gt_u(
4758 &mut self,
4759 loc_a: Location,
4760 loc_b: Location,
4761 ret: Location,
4762 ) -> Result<(), CompileError> {
4763 self.emit_cmpop_i64_dynamic_b(Condition::Above, loc_a, loc_b, ret)
4764 }
4765
4766 fn i64_cmp_le_u(
4767 &mut self,
4768 loc_a: Location,
4769 loc_b: Location,
4770 ret: Location,
4771 ) -> Result<(), CompileError> {
4772 self.emit_cmpop_i64_dynamic_b(Condition::BelowEqual, loc_a, loc_b, ret)
4773 }
4774
4775 fn i64_cmp_lt_u(
4776 &mut self,
4777 loc_a: Location,
4778 loc_b: Location,
4779 ret: Location,
4780 ) -> Result<(), CompileError> {
4781 self.emit_cmpop_i64_dynamic_b(Condition::Below, loc_a, loc_b, ret)
4782 }
4783
4784 fn i64_cmp_ne(
4785 &mut self,
4786 loc_a: Location,
4787 loc_b: Location,
4788 ret: Location,
4789 ) -> Result<(), CompileError> {
4790 self.emit_cmpop_i64_dynamic_b(Condition::NotEqual, loc_a, loc_b, ret)
4791 }
4792
4793 fn i64_cmp_eq(
4794 &mut self,
4795 loc_a: Location,
4796 loc_b: Location,
4797 ret: Location,
4798 ) -> Result<(), CompileError> {
4799 self.emit_cmpop_i64_dynamic_b(Condition::Equal, loc_a, loc_b, ret)
4800 }
4801
4802 fn i64_clz(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
4803 let src = match loc {
4804 Location::Imm64(_) | Location::Imm32(_) | Location::Memory(_, _) => {
4805 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
4806 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4807 })?;
4808 self.move_location(Size::S64, loc, Location::GPR(tmp))?;
4809 tmp
4810 }
4811 Location::GPR(reg) => reg,
4812 _ => {
4813 codegen_error!("singlepass i64_clz unreachable");
4814 }
4815 };
4816 let dst = match ret {
4817 Location::Memory(_, _) => self.acquire_temp_gpr().ok_or_else(|| {
4818 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4819 })?,
4820 Location::GPR(reg) => reg,
4821 _ => {
4822 codegen_error!("singlepass i64_clz unreachable");
4823 }
4824 };
4825
4826 if self.assembler.arch_has_xzcnt() {
4827 self.assembler
4828 .arch_emit_lzcnt(Size::S64, Location::GPR(src), Location::GPR(dst))?;
4829 } else {
4830 let zero_path = self.assembler.get_label();
4831 let end = self.assembler.get_label();
4832
4833 self.assembler.emit_test_gpr_64(src)?;
4834 self.assembler.emit_jmp(Condition::Equal, zero_path)?;
4835 self.assembler
4836 .emit_bsr(Size::S64, Location::GPR(src), Location::GPR(dst))?;
4837 self.assembler
4838 .emit_xor(Size::S64, Location::Imm32(63), Location::GPR(dst))?;
4839 self.assembler.emit_jmp(Condition::None, end)?;
4840 self.emit_label(zero_path)?;
4841 self.move_location(Size::S64, Location::Imm32(64), Location::GPR(dst))?;
4842 self.emit_label(end)?;
4843 }
4844 match loc {
4845 Location::Imm64(_) | Location::Memory(_, _) => {
4846 self.release_gpr(src);
4847 }
4848 _ => {}
4849 };
4850 if let Location::Memory(_, _) = ret {
4851 self.move_location(Size::S64, Location::GPR(dst), ret)?;
4852 self.release_gpr(dst);
4853 };
4854 Ok(())
4855 }
4856
4857 fn i64_ctz(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
4858 let src = match loc {
4859 Location::Imm64(_) | Location::Imm32(_) | Location::Memory(_, _) => {
4860 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
4861 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4862 })?;
4863 self.move_location(Size::S64, loc, Location::GPR(tmp))?;
4864 tmp
4865 }
4866 Location::GPR(reg) => reg,
4867 _ => {
4868 codegen_error!("singlepass i64_ctz unreachable");
4869 }
4870 };
4871 let dst = match ret {
4872 Location::Memory(_, _) => self.acquire_temp_gpr().ok_or_else(|| {
4873 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4874 })?,
4875 Location::GPR(reg) => reg,
4876 _ => {
4877 codegen_error!("singlepass i64_ctz unreachable");
4878 }
4879 };
4880
4881 if self.assembler.arch_has_xzcnt() {
4882 self.assembler
4883 .arch_emit_tzcnt(Size::S64, Location::GPR(src), Location::GPR(dst))?;
4884 } else {
4885 let zero_path = self.assembler.get_label();
4886 let end = self.assembler.get_label();
4887
4888 self.assembler.emit_test_gpr_64(src)?;
4889 self.assembler.emit_jmp(Condition::Equal, zero_path)?;
4890 self.assembler
4891 .emit_bsf(Size::S64, Location::GPR(src), Location::GPR(dst))?;
4892 self.assembler.emit_jmp(Condition::None, end)?;
4893 self.emit_label(zero_path)?;
4894 self.move_location(Size::S64, Location::Imm64(64), Location::GPR(dst))?;
4895 self.emit_label(end)?;
4896 }
4897
4898 match loc {
4899 Location::Imm64(_) | Location::Imm32(_) | Location::Memory(_, _) => {
4900 self.release_gpr(src);
4901 }
4902 _ => {}
4903 };
4904 if let Location::Memory(_, _) = ret {
4905 self.move_location(Size::S64, Location::GPR(dst), ret)?;
4906 self.release_gpr(dst);
4907 };
4908 Ok(())
4909 }
4910
4911 fn i64_popcnt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
4912 match loc {
4913 Location::Imm64(_) | Location::Imm32(_) => {
4914 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
4915 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4916 })?;
4917 self.move_location(Size::S64, loc, Location::GPR(tmp))?;
4918 if let Location::Memory(_, _) = ret {
4919 let out_tmp = self.acquire_temp_gpr().ok_or_else(|| {
4920 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4921 })?;
4922 self.assembler.emit_popcnt(
4923 Size::S64,
4924 Location::GPR(tmp),
4925 Location::GPR(out_tmp),
4926 )?;
4927 self.move_location(Size::S64, Location::GPR(out_tmp), ret)?;
4928 self.release_gpr(out_tmp);
4929 } else {
4930 self.assembler
4931 .emit_popcnt(Size::S64, Location::GPR(tmp), ret)?;
4932 }
4933 self.release_gpr(tmp);
4934 }
4935 Location::Memory(_, _) | Location::GPR(_) => {
4936 if let Location::Memory(_, _) = ret {
4937 let out_tmp = self.acquire_temp_gpr().ok_or_else(|| {
4938 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
4939 })?;
4940 self.assembler
4941 .emit_popcnt(Size::S64, loc, Location::GPR(out_tmp))?;
4942 self.move_location(Size::S64, Location::GPR(out_tmp), ret)?;
4943 self.release_gpr(out_tmp);
4944 } else {
4945 self.assembler.emit_popcnt(Size::S64, loc, ret)?;
4946 }
4947 }
4948 _ => {
4949 codegen_error!("singlepass i64_popcnt unreachable");
4950 }
4951 }
4952 Ok(())
4953 }
4954
4955 fn i64_shl(
4956 &mut self,
4957 loc_a: Location,
4958 loc_b: Location,
4959 ret: Location,
4960 ) -> Result<(), CompileError> {
4961 self.emit_shift_i64(AssemblerX64::emit_shl, loc_a, loc_b, ret)
4962 }
4963
4964 fn i64_shr(
4965 &mut self,
4966 loc_a: Location,
4967 loc_b: Location,
4968 ret: Location,
4969 ) -> Result<(), CompileError> {
4970 self.emit_shift_i64(AssemblerX64::emit_shr, loc_a, loc_b, ret)
4971 }
4972
4973 fn i64_sar(
4974 &mut self,
4975 loc_a: Location,
4976 loc_b: Location,
4977 ret: Location,
4978 ) -> Result<(), CompileError> {
4979 self.emit_shift_i64(AssemblerX64::emit_sar, loc_a, loc_b, ret)
4980 }
4981
4982 fn i64_rol(
4983 &mut self,
4984 loc_a: Location,
4985 loc_b: Location,
4986 ret: Location,
4987 ) -> Result<(), CompileError> {
4988 self.emit_shift_i64(AssemblerX64::emit_rol, loc_a, loc_b, ret)
4989 }
4990
4991 fn i64_ror(
4992 &mut self,
4993 loc_a: Location,
4994 loc_b: Location,
4995 ret: Location,
4996 ) -> Result<(), CompileError> {
4997 self.emit_shift_i64(AssemblerX64::emit_ror, loc_a, loc_b, ret)
4998 }
4999
5000 fn i64_load(
5001 &mut self,
5002 addr: Location,
5003 memarg: &MemArg,
5004 ret: Location,
5005 need_check: bool,
5006 imported_memories: bool,
5007 offset: i32,
5008 heap_access_oob: Label,
5009 unaligned_atomic: Label,
5010 ) -> Result<(), CompileError> {
5011 self.memory_op(
5012 addr,
5013 memarg,
5014 false,
5015 8,
5016 need_check,
5017 imported_memories,
5018 offset,
5019 heap_access_oob,
5020 unaligned_atomic,
5021 |this, addr| {
5022 this.emit_relaxed_binop(
5023 AssemblerX64::emit_mov,
5024 Size::S64,
5025 Location::Memory(addr, 0),
5026 ret,
5027 )
5028 },
5029 )
5030 }
5031
5032 fn i64_load_8u(
5033 &mut self,
5034 addr: Location,
5035 memarg: &MemArg,
5036 ret: Location,
5037 need_check: bool,
5038 imported_memories: bool,
5039 offset: i32,
5040 heap_access_oob: Label,
5041 unaligned_atomic: Label,
5042 ) -> Result<(), CompileError> {
5043 self.memory_op(
5044 addr,
5045 memarg,
5046 false,
5047 1,
5048 need_check,
5049 imported_memories,
5050 offset,
5051 heap_access_oob,
5052 unaligned_atomic,
5053 |this, addr| {
5054 this.emit_relaxed_zx_sx(
5055 AssemblerX64::emit_movzx,
5056 Size::S8,
5057 Location::Memory(addr, 0),
5058 Size::S64,
5059 ret,
5060 )
5061 },
5062 )
5063 }
5064
5065 fn i64_load_8s(
5066 &mut self,
5067 addr: Location,
5068 memarg: &MemArg,
5069 ret: Location,
5070 need_check: bool,
5071 imported_memories: bool,
5072 offset: i32,
5073 heap_access_oob: Label,
5074 unaligned_atomic: Label,
5075 ) -> Result<(), CompileError> {
5076 self.memory_op(
5077 addr,
5078 memarg,
5079 false,
5080 1,
5081 need_check,
5082 imported_memories,
5083 offset,
5084 heap_access_oob,
5085 unaligned_atomic,
5086 |this, addr| {
5087 this.emit_relaxed_zx_sx(
5088 AssemblerX64::emit_movsx,
5089 Size::S8,
5090 Location::Memory(addr, 0),
5091 Size::S64,
5092 ret,
5093 )
5094 },
5095 )
5096 }
5097
5098 fn i64_load_16u(
5099 &mut self,
5100 addr: Location,
5101 memarg: &MemArg,
5102 ret: Location,
5103 need_check: bool,
5104 imported_memories: bool,
5105 offset: i32,
5106 heap_access_oob: Label,
5107 unaligned_atomic: Label,
5108 ) -> Result<(), CompileError> {
5109 self.memory_op(
5110 addr,
5111 memarg,
5112 false,
5113 2,
5114 need_check,
5115 imported_memories,
5116 offset,
5117 heap_access_oob,
5118 unaligned_atomic,
5119 |this, addr| {
5120 this.emit_relaxed_zx_sx(
5121 AssemblerX64::emit_movzx,
5122 Size::S16,
5123 Location::Memory(addr, 0),
5124 Size::S64,
5125 ret,
5126 )
5127 },
5128 )
5129 }
5130
5131 fn i64_load_16s(
5132 &mut self,
5133 addr: Location,
5134 memarg: &MemArg,
5135 ret: Location,
5136 need_check: bool,
5137 imported_memories: bool,
5138 offset: i32,
5139 heap_access_oob: Label,
5140 unaligned_atomic: Label,
5141 ) -> Result<(), CompileError> {
5142 self.memory_op(
5143 addr,
5144 memarg,
5145 false,
5146 2,
5147 need_check,
5148 imported_memories,
5149 offset,
5150 heap_access_oob,
5151 unaligned_atomic,
5152 |this, addr| {
5153 this.emit_relaxed_zx_sx(
5154 AssemblerX64::emit_movsx,
5155 Size::S16,
5156 Location::Memory(addr, 0),
5157 Size::S64,
5158 ret,
5159 )
5160 },
5161 )
5162 }
5163
5164 fn i64_load_32u(
5165 &mut self,
5166 addr: Location,
5167 memarg: &MemArg,
5168 ret: Location,
5169 need_check: bool,
5170 imported_memories: bool,
5171 offset: i32,
5172 heap_access_oob: Label,
5173 unaligned_atomic: Label,
5174 ) -> Result<(), CompileError> {
5175 self.memory_op(
5176 addr,
5177 memarg,
5178 false,
5179 4,
5180 need_check,
5181 imported_memories,
5182 offset,
5183 heap_access_oob,
5184 unaligned_atomic,
5185 |this, addr| {
5186 match ret {
5187 Location::GPR(_) => {}
5188 Location::Memory(base, offset) => {
5189 this.assembler.emit_mov(
5190 Size::S32,
5191 Location::Imm32(0),
5192 Location::Memory(base, offset + 4),
5193 )?; }
5195 _ => {
5196 codegen_error!("singlepass i64_load_32u unreachable");
5197 }
5198 }
5199 this.emit_relaxed_binop(
5200 AssemblerX64::emit_mov,
5201 Size::S32,
5202 Location::Memory(addr, 0),
5203 ret,
5204 )
5205 },
5206 )
5207 }
5208
5209 fn i64_load_32s(
5210 &mut self,
5211 addr: Location,
5212 memarg: &MemArg,
5213 ret: Location,
5214 need_check: bool,
5215 imported_memories: bool,
5216 offset: i32,
5217 heap_access_oob: Label,
5218 unaligned_atomic: Label,
5219 ) -> Result<(), CompileError> {
5220 self.memory_op(
5221 addr,
5222 memarg,
5223 false,
5224 4,
5225 need_check,
5226 imported_memories,
5227 offset,
5228 heap_access_oob,
5229 unaligned_atomic,
5230 |this, addr| {
5231 this.emit_relaxed_zx_sx(
5232 AssemblerX64::emit_movsx,
5233 Size::S32,
5234 Location::Memory(addr, 0),
5235 Size::S64,
5236 ret,
5237 )
5238 },
5239 )
5240 }
5241
5242 fn i64_atomic_load(
5243 &mut self,
5244 addr: Location,
5245 memarg: &MemArg,
5246 ret: Location,
5247 need_check: bool,
5248 imported_memories: bool,
5249 offset: i32,
5250 heap_access_oob: Label,
5251 unaligned_atomic: Label,
5252 ) -> Result<(), CompileError> {
5253 self.memory_op(
5254 addr,
5255 memarg,
5256 true,
5257 8,
5258 need_check,
5259 imported_memories,
5260 offset,
5261 heap_access_oob,
5262 unaligned_atomic,
5263 |this, addr| this.emit_relaxed_mov(Size::S64, Location::Memory(addr, 0), ret),
5264 )
5265 }
5266
5267 fn i64_atomic_load_8u(
5268 &mut self,
5269 addr: Location,
5270 memarg: &MemArg,
5271 ret: Location,
5272 need_check: bool,
5273 imported_memories: bool,
5274 offset: i32,
5275 heap_access_oob: Label,
5276 unaligned_atomic: Label,
5277 ) -> Result<(), CompileError> {
5278 self.memory_op(
5279 addr,
5280 memarg,
5281 true,
5282 1,
5283 need_check,
5284 imported_memories,
5285 offset,
5286 heap_access_oob,
5287 unaligned_atomic,
5288 |this, addr| {
5289 this.emit_relaxed_zero_extension(
5290 Size::S8,
5291 Location::Memory(addr, 0),
5292 Size::S64,
5293 ret,
5294 )
5295 },
5296 )
5297 }
5298
5299 fn i64_atomic_load_16u(
5300 &mut self,
5301 addr: Location,
5302 memarg: &MemArg,
5303 ret: Location,
5304 need_check: bool,
5305 imported_memories: bool,
5306 offset: i32,
5307 heap_access_oob: Label,
5308 unaligned_atomic: Label,
5309 ) -> Result<(), CompileError> {
5310 self.memory_op(
5311 addr,
5312 memarg,
5313 true,
5314 2,
5315 need_check,
5316 imported_memories,
5317 offset,
5318 heap_access_oob,
5319 unaligned_atomic,
5320 |this, addr| {
5321 this.emit_relaxed_zero_extension(
5322 Size::S16,
5323 Location::Memory(addr, 0),
5324 Size::S64,
5325 ret,
5326 )
5327 },
5328 )
5329 }
5330
5331 fn i64_atomic_load_32u(
5332 &mut self,
5333 addr: Location,
5334 memarg: &MemArg,
5335 ret: Location,
5336 need_check: bool,
5337 imported_memories: bool,
5338 offset: i32,
5339 heap_access_oob: Label,
5340 unaligned_atomic: Label,
5341 ) -> Result<(), CompileError> {
5342 self.memory_op(
5343 addr,
5344 memarg,
5345 true,
5346 4,
5347 need_check,
5348 imported_memories,
5349 offset,
5350 heap_access_oob,
5351 unaligned_atomic,
5352 |this, addr| {
5353 match ret {
5354 Location::GPR(_) => {}
5355 Location::Memory(base, offset) => {
5356 this.move_location(
5357 Size::S32,
5358 Location::Imm32(0),
5359 Location::Memory(base, offset + 4),
5360 )?; }
5362 _ => {
5363 codegen_error!("singlepass i64_atomic_load_32u unreachable");
5364 }
5365 }
5366 this.emit_relaxed_zero_extension(
5367 Size::S32,
5368 Location::Memory(addr, 0),
5369 Size::S64,
5370 ret,
5371 )
5372 },
5373 )
5374 }
5375
5376 fn i64_save(
5377 &mut self,
5378 target_value: Location,
5379 memarg: &MemArg,
5380 target_addr: Location,
5381 need_check: bool,
5382 imported_memories: bool,
5383 offset: i32,
5384 heap_access_oob: Label,
5385 unaligned_atomic: Label,
5386 ) -> Result<(), CompileError> {
5387 self.memory_op(
5388 target_addr,
5389 memarg,
5390 false,
5391 8,
5392 need_check,
5393 imported_memories,
5394 offset,
5395 heap_access_oob,
5396 unaligned_atomic,
5397 |this, addr| {
5398 this.emit_relaxed_binop(
5399 AssemblerX64::emit_mov,
5400 Size::S64,
5401 target_value,
5402 Location::Memory(addr, 0),
5403 )
5404 },
5405 )
5406 }
5407
5408 fn i64_save_8(
5409 &mut self,
5410 target_value: Location,
5411 memarg: &MemArg,
5412 target_addr: Location,
5413 need_check: bool,
5414 imported_memories: bool,
5415 offset: i32,
5416 heap_access_oob: Label,
5417 unaligned_atomic: Label,
5418 ) -> Result<(), CompileError> {
5419 self.memory_op(
5420 target_addr,
5421 memarg,
5422 false,
5423 1,
5424 need_check,
5425 imported_memories,
5426 offset,
5427 heap_access_oob,
5428 unaligned_atomic,
5429 |this, addr| {
5430 this.emit_relaxed_binop(
5431 AssemblerX64::emit_mov,
5432 Size::S8,
5433 target_value,
5434 Location::Memory(addr, 0),
5435 )
5436 },
5437 )
5438 }
5439
5440 fn i64_save_16(
5441 &mut self,
5442 target_value: Location,
5443 memarg: &MemArg,
5444 target_addr: Location,
5445 need_check: bool,
5446 imported_memories: bool,
5447 offset: i32,
5448 heap_access_oob: Label,
5449 unaligned_atomic: Label,
5450 ) -> Result<(), CompileError> {
5451 self.memory_op(
5452 target_addr,
5453 memarg,
5454 false,
5455 2,
5456 need_check,
5457 imported_memories,
5458 offset,
5459 heap_access_oob,
5460 unaligned_atomic,
5461 |this, addr| {
5462 this.emit_relaxed_binop(
5463 AssemblerX64::emit_mov,
5464 Size::S16,
5465 target_value,
5466 Location::Memory(addr, 0),
5467 )
5468 },
5469 )
5470 }
5471
5472 fn i64_save_32(
5473 &mut self,
5474 target_value: Location,
5475 memarg: &MemArg,
5476 target_addr: Location,
5477 need_check: bool,
5478 imported_memories: bool,
5479 offset: i32,
5480 heap_access_oob: Label,
5481 unaligned_atomic: Label,
5482 ) -> Result<(), CompileError> {
5483 self.memory_op(
5484 target_addr,
5485 memarg,
5486 false,
5487 4,
5488 need_check,
5489 imported_memories,
5490 offset,
5491 heap_access_oob,
5492 unaligned_atomic,
5493 |this, addr| {
5494 this.emit_relaxed_binop(
5495 AssemblerX64::emit_mov,
5496 Size::S32,
5497 target_value,
5498 Location::Memory(addr, 0),
5499 )
5500 },
5501 )
5502 }
5503
5504 fn i64_atomic_save(
5505 &mut self,
5506 value: Location,
5507 memarg: &MemArg,
5508 target_addr: Location,
5509 need_check: bool,
5510 imported_memories: bool,
5511 offset: i32,
5512 heap_access_oob: Label,
5513 unaligned_atomic: Label,
5514 ) -> Result<(), CompileError> {
5515 self.memory_op(
5516 target_addr,
5517 memarg,
5518 true,
5519 8,
5520 need_check,
5521 imported_memories,
5522 offset,
5523 heap_access_oob,
5524 unaligned_atomic,
5525 |this, addr| this.emit_relaxed_atomic_xchg(Size::S64, value, Location::Memory(addr, 0)),
5526 )
5527 }
5528
5529 fn i64_atomic_save_8(
5530 &mut self,
5531 value: Location,
5532 memarg: &MemArg,
5533 target_addr: Location,
5534 need_check: bool,
5535 imported_memories: bool,
5536 offset: i32,
5537 heap_access_oob: Label,
5538 unaligned_atomic: Label,
5539 ) -> Result<(), CompileError> {
5540 self.memory_op(
5541 target_addr,
5542 memarg,
5543 true,
5544 1,
5545 need_check,
5546 imported_memories,
5547 offset,
5548 heap_access_oob,
5549 unaligned_atomic,
5550 |this, addr| this.emit_relaxed_atomic_xchg(Size::S8, value, Location::Memory(addr, 0)),
5551 )
5552 }
5553
5554 fn i64_atomic_save_16(
5555 &mut self,
5556 value: Location,
5557 memarg: &MemArg,
5558 target_addr: Location,
5559 need_check: bool,
5560 imported_memories: bool,
5561 offset: i32,
5562 heap_access_oob: Label,
5563 unaligned_atomic: Label,
5564 ) -> Result<(), CompileError> {
5565 self.memory_op(
5566 target_addr,
5567 memarg,
5568 true,
5569 2,
5570 need_check,
5571 imported_memories,
5572 offset,
5573 heap_access_oob,
5574 unaligned_atomic,
5575 |this, addr| this.emit_relaxed_atomic_xchg(Size::S16, value, Location::Memory(addr, 0)),
5576 )
5577 }
5578
5579 fn i64_atomic_save_32(
5580 &mut self,
5581 value: Location,
5582 memarg: &MemArg,
5583 target_addr: Location,
5584 need_check: bool,
5585 imported_memories: bool,
5586 offset: i32,
5587 heap_access_oob: Label,
5588 unaligned_atomic: Label,
5589 ) -> Result<(), CompileError> {
5590 self.memory_op(
5591 target_addr,
5592 memarg,
5593 true,
5594 4,
5595 need_check,
5596 imported_memories,
5597 offset,
5598 heap_access_oob,
5599 unaligned_atomic,
5600 |this, addr| this.emit_relaxed_atomic_xchg(Size::S32, value, Location::Memory(addr, 0)),
5601 )
5602 }
5603
5604 fn i64_atomic_add(
5605 &mut self,
5606 loc: Location,
5607 target: Location,
5608 memarg: &MemArg,
5609 ret: Location,
5610 need_check: bool,
5611 imported_memories: bool,
5612 offset: i32,
5613 heap_access_oob: Label,
5614 unaligned_atomic: Label,
5615 ) -> Result<(), CompileError> {
5616 let value = self.acquire_temp_gpr().ok_or_else(|| {
5617 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5618 })?;
5619 self.move_location(Size::S64, loc, Location::GPR(value))?;
5620 self.memory_op(
5621 target,
5622 memarg,
5623 true,
5624 8,
5625 need_check,
5626 imported_memories,
5627 offset,
5628 heap_access_oob,
5629 unaligned_atomic,
5630 |this, addr| {
5631 this.assembler.emit_lock_xadd(
5632 Size::S64,
5633 Location::GPR(value),
5634 Location::Memory(addr, 0),
5635 )
5636 },
5637 )?;
5638 self.move_location(Size::S64, Location::GPR(value), ret)?;
5639 self.release_gpr(value);
5640 Ok(())
5641 }
5642
5643 fn i64_atomic_add_8u(
5644 &mut self,
5645 loc: Location,
5646 target: Location,
5647 memarg: &MemArg,
5648 ret: Location,
5649 need_check: bool,
5650 imported_memories: bool,
5651 offset: i32,
5652 heap_access_oob: Label,
5653 unaligned_atomic: Label,
5654 ) -> Result<(), CompileError> {
5655 let value = self.acquire_temp_gpr().ok_or_else(|| {
5656 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5657 })?;
5658 self.move_location_extend(Size::S8, false, loc, Size::S64, Location::GPR(value))?;
5659 self.memory_op(
5660 target,
5661 memarg,
5662 true,
5663 1,
5664 need_check,
5665 imported_memories,
5666 offset,
5667 heap_access_oob,
5668 unaligned_atomic,
5669 |this, addr| {
5670 this.assembler.emit_lock_xadd(
5671 Size::S8,
5672 Location::GPR(value),
5673 Location::Memory(addr, 0),
5674 )
5675 },
5676 )?;
5677 self.move_location_extend(
5679 Size::S8,
5680 false,
5681 Location::GPR(value),
5682 Size::S64,
5683 Location::GPR(value),
5684 )?;
5685 self.move_location(Size::S64, Location::GPR(value), ret)?;
5686 self.release_gpr(value);
5687 Ok(())
5688 }
5689
5690 fn i64_atomic_add_16u(
5691 &mut self,
5692 loc: Location,
5693 target: Location,
5694 memarg: &MemArg,
5695 ret: Location,
5696 need_check: bool,
5697 imported_memories: bool,
5698 offset: i32,
5699 heap_access_oob: Label,
5700 unaligned_atomic: Label,
5701 ) -> Result<(), CompileError> {
5702 let value = self.acquire_temp_gpr().ok_or_else(|| {
5703 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5704 })?;
5705 self.move_location_extend(Size::S16, false, loc, Size::S64, Location::GPR(value))?;
5706 self.memory_op(
5707 target,
5708 memarg,
5709 true,
5710 2,
5711 need_check,
5712 imported_memories,
5713 offset,
5714 heap_access_oob,
5715 unaligned_atomic,
5716 |this, addr| {
5717 this.assembler.emit_lock_xadd(
5718 Size::S16,
5719 Location::GPR(value),
5720 Location::Memory(addr, 0),
5721 )
5722 },
5723 )?;
5724 self.move_location_extend(
5726 Size::S16,
5727 false,
5728 Location::GPR(value),
5729 Size::S64,
5730 Location::GPR(value),
5731 )?;
5732 self.move_location(Size::S64, Location::GPR(value), ret)?;
5733 self.release_gpr(value);
5734 Ok(())
5735 }
5736
5737 fn i64_atomic_add_32u(
5738 &mut self,
5739 loc: Location,
5740 target: Location,
5741 memarg: &MemArg,
5742 ret: Location,
5743 need_check: bool,
5744 imported_memories: bool,
5745 offset: i32,
5746 heap_access_oob: Label,
5747 unaligned_atomic: Label,
5748 ) -> Result<(), CompileError> {
5749 let value = self.acquire_temp_gpr().ok_or_else(|| {
5750 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5751 })?;
5752 self.move_location_extend(Size::S32, false, loc, Size::S64, Location::GPR(value))?;
5753 self.memory_op(
5754 target,
5755 memarg,
5756 true,
5757 4,
5758 need_check,
5759 imported_memories,
5760 offset,
5761 heap_access_oob,
5762 unaligned_atomic,
5763 |this, addr| {
5764 this.assembler.emit_lock_xadd(
5765 Size::S32,
5766 Location::GPR(value),
5767 Location::Memory(addr, 0),
5768 )
5769 },
5770 )?;
5771 self.move_location_extend(
5773 Size::S32,
5774 false,
5775 Location::GPR(value),
5776 Size::S64,
5777 Location::GPR(value),
5778 )?;
5779 self.move_location(Size::S64, Location::GPR(value), ret)?;
5780 self.release_gpr(value);
5781 Ok(())
5782 }
5783
5784 fn i64_atomic_sub(
5785 &mut self,
5786 loc: Location,
5787 target: Location,
5788 memarg: &MemArg,
5789 ret: Location,
5790 need_check: bool,
5791 imported_memories: bool,
5792 offset: i32,
5793 heap_access_oob: Label,
5794 unaligned_atomic: Label,
5795 ) -> Result<(), CompileError> {
5796 let value = self.acquire_temp_gpr().ok_or_else(|| {
5797 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5798 })?;
5799 self.location_neg(Size::S64, false, loc, Size::S64, Location::GPR(value))?;
5800 self.memory_op(
5801 target,
5802 memarg,
5803 true,
5804 8,
5805 need_check,
5806 imported_memories,
5807 offset,
5808 heap_access_oob,
5809 unaligned_atomic,
5810 |this, addr| {
5811 this.assembler.emit_lock_xadd(
5812 Size::S64,
5813 Location::GPR(value),
5814 Location::Memory(addr, 0),
5815 )
5816 },
5817 )?;
5818 self.move_location(Size::S64, Location::GPR(value), ret)?;
5819 self.release_gpr(value);
5820 Ok(())
5821 }
5822
5823 fn i64_atomic_sub_8u(
5824 &mut self,
5825 loc: Location,
5826 target: Location,
5827 memarg: &MemArg,
5828 ret: Location,
5829 need_check: bool,
5830 imported_memories: bool,
5831 offset: i32,
5832 heap_access_oob: Label,
5833 unaligned_atomic: Label,
5834 ) -> Result<(), CompileError> {
5835 let value = self.acquire_temp_gpr().ok_or_else(|| {
5836 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5837 })?;
5838 self.location_neg(Size::S8, false, loc, Size::S64, Location::GPR(value))?;
5839 self.memory_op(
5840 target,
5841 memarg,
5842 true,
5843 1,
5844 need_check,
5845 imported_memories,
5846 offset,
5847 heap_access_oob,
5848 unaligned_atomic,
5849 |this, addr| {
5850 this.assembler.emit_lock_xadd(
5851 Size::S8,
5852 Location::GPR(value),
5853 Location::Memory(addr, 0),
5854 )
5855 },
5856 )?;
5857 self.move_location_extend(
5859 Size::S8,
5860 false,
5861 Location::GPR(value),
5862 Size::S64,
5863 Location::GPR(value),
5864 )?;
5865 self.move_location(Size::S64, Location::GPR(value), ret)?;
5866 self.release_gpr(value);
5867 Ok(())
5868 }
5869
5870 fn i64_atomic_sub_16u(
5871 &mut self,
5872 loc: Location,
5873 target: Location,
5874 memarg: &MemArg,
5875 ret: Location,
5876 need_check: bool,
5877 imported_memories: bool,
5878 offset: i32,
5879 heap_access_oob: Label,
5880 unaligned_atomic: Label,
5881 ) -> Result<(), CompileError> {
5882 let value = self.acquire_temp_gpr().ok_or_else(|| {
5883 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5884 })?;
5885 self.location_neg(Size::S16, false, loc, Size::S64, Location::GPR(value))?;
5886 self.memory_op(
5887 target,
5888 memarg,
5889 true,
5890 2,
5891 need_check,
5892 imported_memories,
5893 offset,
5894 heap_access_oob,
5895 unaligned_atomic,
5896 |this, addr| {
5897 this.assembler.emit_lock_xadd(
5898 Size::S16,
5899 Location::GPR(value),
5900 Location::Memory(addr, 0),
5901 )
5902 },
5903 )?;
5904 self.move_location_extend(
5906 Size::S16,
5907 false,
5908 Location::GPR(value),
5909 Size::S64,
5910 Location::GPR(value),
5911 )?;
5912 self.move_location(Size::S64, Location::GPR(value), ret)?;
5913 self.release_gpr(value);
5914 Ok(())
5915 }
5916
5917 fn i64_atomic_sub_32u(
5918 &mut self,
5919 loc: Location,
5920 target: Location,
5921 memarg: &MemArg,
5922 ret: Location,
5923 need_check: bool,
5924 imported_memories: bool,
5925 offset: i32,
5926 heap_access_oob: Label,
5927 unaligned_atomic: Label,
5928 ) -> Result<(), CompileError> {
5929 let value = self.acquire_temp_gpr().ok_or_else(|| {
5930 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
5931 })?;
5932 self.location_neg(Size::S32, false, loc, Size::S64, Location::GPR(value))?;
5933 self.memory_op(
5934 target,
5935 memarg,
5936 true,
5937 4,
5938 need_check,
5939 imported_memories,
5940 offset,
5941 heap_access_oob,
5942 unaligned_atomic,
5943 |this, addr| {
5944 this.assembler.emit_lock_xadd(
5945 Size::S32,
5946 Location::GPR(value),
5947 Location::Memory(addr, 0),
5948 )
5949 },
5950 )?;
5951 self.move_location_extend(
5953 Size::S32,
5954 false,
5955 Location::GPR(value),
5956 Size::S64,
5957 Location::GPR(value),
5958 )?;
5959 self.move_location(Size::S64, Location::GPR(value), ret)?;
5960 self.release_gpr(value);
5961 Ok(())
5962 }
5963
5964 fn i64_atomic_and(
5965 &mut self,
5966 loc: Location,
5967 target: Location,
5968 memarg: &MemArg,
5969 ret: Location,
5970 need_check: bool,
5971 imported_memories: bool,
5972 offset: i32,
5973 heap_access_oob: Label,
5974 unaligned_atomic: Label,
5975 ) -> Result<(), CompileError> {
5976 self.emit_compare_and_swap(
5977 loc,
5978 target,
5979 ret,
5980 memarg,
5981 8,
5982 Size::S64,
5983 Size::S64,
5984 need_check,
5985 imported_memories,
5986 offset,
5987 heap_access_oob,
5988 unaligned_atomic,
5989 |this, src, dst| {
5990 this.assembler
5991 .emit_and(Size::S64, Location::GPR(src), Location::GPR(dst))
5992 },
5993 )
5994 }
5995
5996 fn i64_atomic_and_8u(
5997 &mut self,
5998 loc: Location,
5999 target: Location,
6000 memarg: &MemArg,
6001 ret: Location,
6002 need_check: bool,
6003 imported_memories: bool,
6004 offset: i32,
6005 heap_access_oob: Label,
6006 unaligned_atomic: Label,
6007 ) -> Result<(), CompileError> {
6008 self.emit_compare_and_swap(
6009 loc,
6010 target,
6011 ret,
6012 memarg,
6013 1,
6014 Size::S8,
6015 Size::S64,
6016 need_check,
6017 imported_memories,
6018 offset,
6019 heap_access_oob,
6020 unaligned_atomic,
6021 |this, src, dst| {
6022 this.assembler
6023 .emit_and(Size::S64, Location::GPR(src), Location::GPR(dst))
6024 },
6025 )
6026 }
6027
6028 fn i64_atomic_and_16u(
6029 &mut self,
6030 loc: Location,
6031 target: Location,
6032 memarg: &MemArg,
6033 ret: Location,
6034 need_check: bool,
6035 imported_memories: bool,
6036 offset: i32,
6037 heap_access_oob: Label,
6038 unaligned_atomic: Label,
6039 ) -> Result<(), CompileError> {
6040 self.emit_compare_and_swap(
6041 loc,
6042 target,
6043 ret,
6044 memarg,
6045 2,
6046 Size::S16,
6047 Size::S64,
6048 need_check,
6049 imported_memories,
6050 offset,
6051 heap_access_oob,
6052 unaligned_atomic,
6053 |this, src, dst| {
6054 this.assembler
6055 .emit_and(Size::S64, Location::GPR(src), Location::GPR(dst))
6056 },
6057 )
6058 }
6059
6060 fn i64_atomic_and_32u(
6061 &mut self,
6062 loc: Location,
6063 target: Location,
6064 memarg: &MemArg,
6065 ret: Location,
6066 need_check: bool,
6067 imported_memories: bool,
6068 offset: i32,
6069 heap_access_oob: Label,
6070 unaligned_atomic: Label,
6071 ) -> Result<(), CompileError> {
6072 self.emit_compare_and_swap(
6073 loc,
6074 target,
6075 ret,
6076 memarg,
6077 4,
6078 Size::S32,
6079 Size::S64,
6080 need_check,
6081 imported_memories,
6082 offset,
6083 heap_access_oob,
6084 unaligned_atomic,
6085 |this, src, dst| {
6086 this.assembler
6087 .emit_and(Size::S64, Location::GPR(src), Location::GPR(dst))
6088 },
6089 )
6090 }
6091
6092 fn i64_atomic_or(
6093 &mut self,
6094 loc: Location,
6095 target: Location,
6096 memarg: &MemArg,
6097 ret: Location,
6098 need_check: bool,
6099 imported_memories: bool,
6100 offset: i32,
6101 heap_access_oob: Label,
6102 unaligned_atomic: Label,
6103 ) -> Result<(), CompileError> {
6104 self.emit_compare_and_swap(
6105 loc,
6106 target,
6107 ret,
6108 memarg,
6109 8,
6110 Size::S64,
6111 Size::S64,
6112 need_check,
6113 imported_memories,
6114 offset,
6115 heap_access_oob,
6116 unaligned_atomic,
6117 |this, src, dst| {
6118 this.location_or(Size::S64, Location::GPR(src), Location::GPR(dst), false)
6119 },
6120 )
6121 }
6122
6123 fn i64_atomic_or_8u(
6124 &mut self,
6125 loc: Location,
6126 target: Location,
6127 memarg: &MemArg,
6128 ret: Location,
6129 need_check: bool,
6130 imported_memories: bool,
6131 offset: i32,
6132 heap_access_oob: Label,
6133 unaligned_atomic: Label,
6134 ) -> Result<(), CompileError> {
6135 self.emit_compare_and_swap(
6136 loc,
6137 target,
6138 ret,
6139 memarg,
6140 1,
6141 Size::S8,
6142 Size::S64,
6143 need_check,
6144 imported_memories,
6145 offset,
6146 heap_access_oob,
6147 unaligned_atomic,
6148 |this, src, dst| {
6149 this.location_or(Size::S64, Location::GPR(src), Location::GPR(dst), false)
6150 },
6151 )
6152 }
6153
6154 fn i64_atomic_or_16u(
6155 &mut self,
6156 loc: Location,
6157 target: Location,
6158 memarg: &MemArg,
6159 ret: Location,
6160 need_check: bool,
6161 imported_memories: bool,
6162 offset: i32,
6163 heap_access_oob: Label,
6164 unaligned_atomic: Label,
6165 ) -> Result<(), CompileError> {
6166 self.emit_compare_and_swap(
6167 loc,
6168 target,
6169 ret,
6170 memarg,
6171 2,
6172 Size::S16,
6173 Size::S64,
6174 need_check,
6175 imported_memories,
6176 offset,
6177 heap_access_oob,
6178 unaligned_atomic,
6179 |this, src, dst| {
6180 this.location_or(Size::S64, Location::GPR(src), Location::GPR(dst), false)
6181 },
6182 )
6183 }
6184
6185 fn i64_atomic_or_32u(
6186 &mut self,
6187 loc: Location,
6188 target: Location,
6189 memarg: &MemArg,
6190 ret: Location,
6191 need_check: bool,
6192 imported_memories: bool,
6193 offset: i32,
6194 heap_access_oob: Label,
6195 unaligned_atomic: Label,
6196 ) -> Result<(), CompileError> {
6197 self.emit_compare_and_swap(
6198 loc,
6199 target,
6200 ret,
6201 memarg,
6202 4,
6203 Size::S32,
6204 Size::S64,
6205 need_check,
6206 imported_memories,
6207 offset,
6208 heap_access_oob,
6209 unaligned_atomic,
6210 |this, src, dst| {
6211 this.location_or(Size::S64, Location::GPR(src), Location::GPR(dst), false)
6212 },
6213 )
6214 }
6215
6216 fn i64_atomic_xor(
6217 &mut self,
6218 loc: Location,
6219 target: Location,
6220 memarg: &MemArg,
6221 ret: Location,
6222 need_check: bool,
6223 imported_memories: bool,
6224 offset: i32,
6225 heap_access_oob: Label,
6226 unaligned_atomic: Label,
6227 ) -> Result<(), CompileError> {
6228 self.emit_compare_and_swap(
6229 loc,
6230 target,
6231 ret,
6232 memarg,
6233 8,
6234 Size::S64,
6235 Size::S64,
6236 need_check,
6237 imported_memories,
6238 offset,
6239 heap_access_oob,
6240 unaligned_atomic,
6241 |this, src, dst| {
6242 this.location_xor(Size::S64, Location::GPR(src), Location::GPR(dst), false)
6243 },
6244 )
6245 }
6246
6247 fn i64_atomic_xor_8u(
6248 &mut self,
6249 loc: Location,
6250 target: Location,
6251 memarg: &MemArg,
6252 ret: Location,
6253 need_check: bool,
6254 imported_memories: bool,
6255 offset: i32,
6256 heap_access_oob: Label,
6257 unaligned_atomic: Label,
6258 ) -> Result<(), CompileError> {
6259 self.emit_compare_and_swap(
6260 loc,
6261 target,
6262 ret,
6263 memarg,
6264 1,
6265 Size::S8,
6266 Size::S64,
6267 need_check,
6268 imported_memories,
6269 offset,
6270 heap_access_oob,
6271 unaligned_atomic,
6272 |this, src, dst| {
6273 this.location_xor(Size::S64, Location::GPR(src), Location::GPR(dst), false)
6274 },
6275 )
6276 }
6277
6278 fn i64_atomic_xor_16u(
6279 &mut self,
6280 loc: Location,
6281 target: Location,
6282 memarg: &MemArg,
6283 ret: Location,
6284 need_check: bool,
6285 imported_memories: bool,
6286 offset: i32,
6287 heap_access_oob: Label,
6288 unaligned_atomic: Label,
6289 ) -> Result<(), CompileError> {
6290 self.emit_compare_and_swap(
6291 loc,
6292 target,
6293 ret,
6294 memarg,
6295 2,
6296 Size::S16,
6297 Size::S64,
6298 need_check,
6299 imported_memories,
6300 offset,
6301 heap_access_oob,
6302 unaligned_atomic,
6303 |this, src, dst| {
6304 this.location_xor(Size::S64, Location::GPR(src), Location::GPR(dst), false)
6305 },
6306 )
6307 }
6308
6309 fn i64_atomic_xor_32u(
6310 &mut self,
6311 loc: Location,
6312 target: Location,
6313 memarg: &MemArg,
6314 ret: Location,
6315 need_check: bool,
6316 imported_memories: bool,
6317 offset: i32,
6318 heap_access_oob: Label,
6319 unaligned_atomic: Label,
6320 ) -> Result<(), CompileError> {
6321 self.emit_compare_and_swap(
6322 loc,
6323 target,
6324 ret,
6325 memarg,
6326 4,
6327 Size::S32,
6328 Size::S64,
6329 need_check,
6330 imported_memories,
6331 offset,
6332 heap_access_oob,
6333 unaligned_atomic,
6334 |this, src, dst| {
6335 this.location_xor(Size::S64, Location::GPR(src), Location::GPR(dst), false)
6336 },
6337 )
6338 }
6339
6340 fn i64_atomic_xchg(
6341 &mut self,
6342 loc: Location,
6343 target: Location,
6344 memarg: &MemArg,
6345 ret: Location,
6346 need_check: bool,
6347 imported_memories: bool,
6348 offset: i32,
6349 heap_access_oob: Label,
6350 unaligned_atomic: Label,
6351 ) -> Result<(), CompileError> {
6352 let value = self.acquire_temp_gpr().ok_or_else(|| {
6353 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6354 })?;
6355 self.move_location(Size::S64, loc, Location::GPR(value))?;
6356 self.memory_op(
6357 target,
6358 memarg,
6359 true,
6360 8,
6361 need_check,
6362 imported_memories,
6363 offset,
6364 heap_access_oob,
6365 unaligned_atomic,
6366 |this, addr| {
6367 this.assembler
6368 .emit_xchg(Size::S64, Location::GPR(value), Location::Memory(addr, 0))
6369 },
6370 )?;
6371 self.move_location(Size::S64, Location::GPR(value), ret)?;
6372 self.release_gpr(value);
6373 Ok(())
6374 }
6375
6376 fn i64_atomic_xchg_8u(
6377 &mut self,
6378 loc: Location,
6379 target: Location,
6380 memarg: &MemArg,
6381 ret: Location,
6382 need_check: bool,
6383 imported_memories: bool,
6384 offset: i32,
6385 heap_access_oob: Label,
6386 unaligned_atomic: Label,
6387 ) -> Result<(), CompileError> {
6388 let value = self.acquire_temp_gpr().ok_or_else(|| {
6389 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6390 })?;
6391 self.assembler
6392 .emit_movzx(Size::S8, loc, Size::S64, Location::GPR(value))?;
6393 self.memory_op(
6394 target,
6395 memarg,
6396 true,
6397 1,
6398 need_check,
6399 imported_memories,
6400 offset,
6401 heap_access_oob,
6402 unaligned_atomic,
6403 |this, addr| {
6404 this.assembler
6405 .emit_xchg(Size::S8, Location::GPR(value), Location::Memory(addr, 0))
6406 },
6407 )?;
6408 self.move_location_extend(
6410 Size::S8,
6411 false,
6412 Location::GPR(value),
6413 Size::S64,
6414 Location::GPR(value),
6415 )?;
6416 self.move_location(Size::S64, Location::GPR(value), ret)?;
6417 self.release_gpr(value);
6418 Ok(())
6419 }
6420
6421 fn i64_atomic_xchg_16u(
6422 &mut self,
6423 loc: Location,
6424 target: Location,
6425 memarg: &MemArg,
6426 ret: Location,
6427 need_check: bool,
6428 imported_memories: bool,
6429 offset: i32,
6430 heap_access_oob: Label,
6431 unaligned_atomic: Label,
6432 ) -> Result<(), CompileError> {
6433 let value = self.acquire_temp_gpr().ok_or_else(|| {
6434 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6435 })?;
6436 self.assembler
6437 .emit_movzx(Size::S16, loc, Size::S64, Location::GPR(value))?;
6438 self.memory_op(
6439 target,
6440 memarg,
6441 true,
6442 2,
6443 need_check,
6444 imported_memories,
6445 offset,
6446 heap_access_oob,
6447 unaligned_atomic,
6448 |this, addr| {
6449 this.assembler
6450 .emit_xchg(Size::S16, Location::GPR(value), Location::Memory(addr, 0))
6451 },
6452 )?;
6453 self.move_location_extend(
6455 Size::S16,
6456 false,
6457 Location::GPR(value),
6458 Size::S64,
6459 Location::GPR(value),
6460 )?;
6461 self.move_location(Size::S64, Location::GPR(value), ret)?;
6462 self.release_gpr(value);
6463 Ok(())
6464 }
6465
6466 fn i64_atomic_xchg_32u(
6467 &mut self,
6468 loc: Location,
6469 target: Location,
6470 memarg: &MemArg,
6471 ret: Location,
6472 need_check: bool,
6473 imported_memories: bool,
6474 offset: i32,
6475 heap_access_oob: Label,
6476 unaligned_atomic: Label,
6477 ) -> Result<(), CompileError> {
6478 let value = self.acquire_temp_gpr().ok_or_else(|| {
6479 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6480 })?;
6481 self.assembler
6482 .emit_movzx(Size::S32, loc, Size::S64, Location::GPR(value))?;
6483 self.memory_op(
6484 target,
6485 memarg,
6486 true,
6487 4,
6488 need_check,
6489 imported_memories,
6490 offset,
6491 heap_access_oob,
6492 unaligned_atomic,
6493 |this, addr| {
6494 this.assembler
6495 .emit_xchg(Size::S32, Location::GPR(value), Location::Memory(addr, 0))
6496 },
6497 )?;
6498 self.move_location_extend(
6500 Size::S32,
6501 false,
6502 Location::GPR(value),
6503 Size::S64,
6504 Location::GPR(value),
6505 )?;
6506 self.move_location(Size::S64, Location::GPR(value), ret)?;
6507 self.release_gpr(value);
6508 Ok(())
6509 }
6510
6511 fn i64_atomic_cmpxchg(
6512 &mut self,
6513 new: Location,
6514 cmp: Location,
6515 target: Location,
6516 memarg: &MemArg,
6517 ret: Location,
6518 need_check: bool,
6519 imported_memories: bool,
6520 offset: i32,
6521 heap_access_oob: Label,
6522 unaligned_atomic: Label,
6523 ) -> Result<(), CompileError> {
6524 let compare = self.reserve_unused_temp_gpr(GPR::RAX);
6525 let value = if cmp == Location::GPR(GPR::R14) {
6526 if new == Location::GPR(GPR::R13) {
6527 GPR::R12
6528 } else {
6529 GPR::R13
6530 }
6531 } else {
6532 GPR::R14
6533 };
6534 self.assembler.emit_push(Size::S64, Location::GPR(value))?;
6535 self.assembler
6536 .emit_mov(Size::S64, cmp, Location::GPR(compare))?;
6537 self.assembler
6538 .emit_mov(Size::S64, new, Location::GPR(value))?;
6539
6540 self.memory_op(
6541 target,
6542 memarg,
6543 true,
6544 8,
6545 need_check,
6546 imported_memories,
6547 offset,
6548 heap_access_oob,
6549 unaligned_atomic,
6550 |this, addr| {
6551 this.assembler.emit_lock_cmpxchg(
6552 Size::S64,
6553 Location::GPR(value),
6554 Location::Memory(addr, 0),
6555 )?;
6556 this.assembler
6557 .emit_mov(Size::S64, Location::GPR(compare), ret)
6558 },
6559 )?;
6560 self.assembler.emit_pop(Size::S64, Location::GPR(value))?;
6561 self.release_gpr(compare);
6562 Ok(())
6563 }
6564
6565 fn i64_atomic_cmpxchg_8u(
6566 &mut self,
6567 new: Location,
6568 cmp: Location,
6569 target: Location,
6570 memarg: &MemArg,
6571 ret: Location,
6572 need_check: bool,
6573 imported_memories: bool,
6574 offset: i32,
6575 heap_access_oob: Label,
6576 unaligned_atomic: Label,
6577 ) -> Result<(), CompileError> {
6578 let compare = self.reserve_unused_temp_gpr(GPR::RAX);
6579 let value = if cmp == Location::GPR(GPR::R14) {
6580 if new == Location::GPR(GPR::R13) {
6581 GPR::R12
6582 } else {
6583 GPR::R13
6584 }
6585 } else {
6586 GPR::R14
6587 };
6588 self.assembler.emit_push(Size::S64, Location::GPR(value))?;
6589 self.assembler
6590 .emit_mov(Size::S64, cmp, Location::GPR(compare))?;
6591 self.assembler
6592 .emit_mov(Size::S64, new, Location::GPR(value))?;
6593
6594 self.memory_op(
6595 target,
6596 memarg,
6597 true,
6598 1,
6599 need_check,
6600 imported_memories,
6601 offset,
6602 heap_access_oob,
6603 unaligned_atomic,
6604 |this, addr| {
6605 this.assembler.emit_lock_cmpxchg(
6606 Size::S8,
6607 Location::GPR(value),
6608 Location::Memory(addr, 0),
6609 )?;
6610 this.move_location_extend(
6611 Size::S8,
6612 false,
6613 Location::GPR(compare),
6614 Size::S64,
6615 Location::GPR(compare),
6616 )?;
6617 this.move_location(Size::S64, Location::GPR(compare), ret)
6618 },
6619 )?;
6620 self.assembler.emit_pop(Size::S64, Location::GPR(value))?;
6621 self.release_gpr(compare);
6622 Ok(())
6623 }
6624
6625 fn i64_atomic_cmpxchg_16u(
6626 &mut self,
6627 new: Location,
6628 cmp: Location,
6629 target: Location,
6630 memarg: &MemArg,
6631 ret: Location,
6632 need_check: bool,
6633 imported_memories: bool,
6634 offset: i32,
6635 heap_access_oob: Label,
6636 unaligned_atomic: Label,
6637 ) -> Result<(), CompileError> {
6638 let compare = self.reserve_unused_temp_gpr(GPR::RAX);
6639 let value = if cmp == Location::GPR(GPR::R14) {
6640 if new == Location::GPR(GPR::R13) {
6641 GPR::R12
6642 } else {
6643 GPR::R13
6644 }
6645 } else {
6646 GPR::R14
6647 };
6648 self.assembler.emit_push(Size::S64, Location::GPR(value))?;
6649 self.assembler
6650 .emit_mov(Size::S64, cmp, Location::GPR(compare))?;
6651 self.assembler
6652 .emit_mov(Size::S64, new, Location::GPR(value))?;
6653
6654 self.memory_op(
6655 target,
6656 memarg,
6657 true,
6658 2,
6659 need_check,
6660 imported_memories,
6661 offset,
6662 heap_access_oob,
6663 unaligned_atomic,
6664 |this, addr| {
6665 this.assembler.emit_lock_cmpxchg(
6666 Size::S16,
6667 Location::GPR(value),
6668 Location::Memory(addr, 0),
6669 )?;
6670 this.move_location_extend(
6671 Size::S16,
6672 false,
6673 Location::GPR(compare),
6674 Size::S64,
6675 Location::GPR(compare),
6676 )?;
6677 this.move_location(Size::S64, Location::GPR(compare), ret)
6678 },
6679 )?;
6680 self.assembler.emit_pop(Size::S64, Location::GPR(value))?;
6681 self.release_gpr(compare);
6682 Ok(())
6683 }
6684
6685 fn i64_atomic_cmpxchg_32u(
6686 &mut self,
6687 new: Location,
6688 cmp: Location,
6689 target: Location,
6690 memarg: &MemArg,
6691 ret: Location,
6692 need_check: bool,
6693 imported_memories: bool,
6694 offset: i32,
6695 heap_access_oob: Label,
6696 unaligned_atomic: Label,
6697 ) -> Result<(), CompileError> {
6698 let compare = self.reserve_unused_temp_gpr(GPR::RAX);
6699 let value = if cmp == Location::GPR(GPR::R14) {
6700 if new == Location::GPR(GPR::R13) {
6701 GPR::R12
6702 } else {
6703 GPR::R13
6704 }
6705 } else {
6706 GPR::R14
6707 };
6708 self.assembler.emit_push(Size::S64, Location::GPR(value))?;
6709 self.assembler
6710 .emit_mov(Size::S64, cmp, Location::GPR(compare))?;
6711 self.assembler
6712 .emit_mov(Size::S64, new, Location::GPR(value))?;
6713
6714 self.memory_op(
6715 target,
6716 memarg,
6717 true,
6718 4,
6719 need_check,
6720 imported_memories,
6721 offset,
6722 heap_access_oob,
6723 unaligned_atomic,
6724 |this, addr| {
6725 this.assembler.emit_lock_cmpxchg(
6726 Size::S32,
6727 Location::GPR(value),
6728 Location::Memory(addr, 0),
6729 )?;
6730 this.move_location_extend(
6731 Size::S32,
6732 false,
6733 Location::GPR(compare),
6734 Size::S64,
6735 Location::GPR(compare),
6736 )?;
6737 this.move_location(Size::S64, Location::GPR(compare), ret)
6738 },
6739 )?;
6740 self.assembler.emit_pop(Size::S64, Location::GPR(value))?;
6741 self.release_gpr(compare);
6742 Ok(())
6743 }
6744
6745 fn f32_load(
6746 &mut self,
6747 addr: Location,
6748 memarg: &MemArg,
6749 ret: Location,
6750 need_check: bool,
6751 imported_memories: bool,
6752 offset: i32,
6753 heap_access_oob: Label,
6754 unaligned_atomic: Label,
6755 ) -> Result<(), CompileError> {
6756 self.memory_op(
6757 addr,
6758 memarg,
6759 false,
6760 4,
6761 need_check,
6762 imported_memories,
6763 offset,
6764 heap_access_oob,
6765 unaligned_atomic,
6766 |this, addr| {
6767 this.emit_relaxed_binop(
6768 AssemblerX64::emit_mov,
6769 Size::S32,
6770 Location::Memory(addr, 0),
6771 ret,
6772 )
6773 },
6774 )
6775 }
6776
6777 fn f32_save(
6778 &mut self,
6779 target_value: Location,
6780 memarg: &MemArg,
6781 target_addr: Location,
6782 canonicalize: bool,
6783 need_check: bool,
6784 imported_memories: bool,
6785 offset: i32,
6786 heap_access_oob: Label,
6787 unaligned_atomic: Label,
6788 ) -> Result<(), CompileError> {
6789 self.memory_op(
6790 target_addr,
6791 memarg,
6792 false,
6793 4,
6794 need_check,
6795 imported_memories,
6796 offset,
6797 heap_access_oob,
6798 unaligned_atomic,
6799 |this, addr| {
6800 if !canonicalize {
6801 this.emit_relaxed_binop(
6802 AssemblerX64::emit_mov,
6803 Size::S32,
6804 target_value,
6805 Location::Memory(addr, 0),
6806 )
6807 } else {
6808 this.canonicalize_nan(Size::S32, target_value, Location::Memory(addr, 0))
6809 }
6810 },
6811 )
6812 }
6813
6814 fn f64_load(
6815 &mut self,
6816 addr: Location,
6817 memarg: &MemArg,
6818 ret: Location,
6819 need_check: bool,
6820 imported_memories: bool,
6821 offset: i32,
6822 heap_access_oob: Label,
6823 unaligned_atomic: Label,
6824 ) -> Result<(), CompileError> {
6825 self.memory_op(
6826 addr,
6827 memarg,
6828 false,
6829 8,
6830 need_check,
6831 imported_memories,
6832 offset,
6833 heap_access_oob,
6834 unaligned_atomic,
6835 |this, addr| {
6836 this.emit_relaxed_binop(
6837 AssemblerX64::emit_mov,
6838 Size::S64,
6839 Location::Memory(addr, 0),
6840 ret,
6841 )
6842 },
6843 )
6844 }
6845
6846 fn f64_save(
6847 &mut self,
6848 target_value: Location,
6849 memarg: &MemArg,
6850 target_addr: Location,
6851 canonicalize: bool,
6852 need_check: bool,
6853 imported_memories: bool,
6854 offset: i32,
6855 heap_access_oob: Label,
6856 unaligned_atomic: Label,
6857 ) -> Result<(), CompileError> {
6858 self.memory_op(
6859 target_addr,
6860 memarg,
6861 false,
6862 8,
6863 need_check,
6864 imported_memories,
6865 offset,
6866 heap_access_oob,
6867 unaligned_atomic,
6868 |this, addr| {
6869 if !canonicalize {
6870 this.emit_relaxed_binop(
6871 AssemblerX64::emit_mov,
6872 Size::S64,
6873 target_value,
6874 Location::Memory(addr, 0),
6875 )
6876 } else {
6877 this.canonicalize_nan(Size::S64, target_value, Location::Memory(addr, 0))
6878 }
6879 },
6880 )
6881 }
6882
6883 fn convert_f64_i64(
6884 &mut self,
6885 loc: Location,
6886 signed: bool,
6887 ret: Location,
6888 ) -> Result<(), CompileError> {
6889 let tmp_out = self.acquire_temp_simd().ok_or_else(|| {
6890 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
6891 })?;
6892 let tmp_in = self.acquire_temp_gpr().ok_or_else(|| {
6893 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6894 })?;
6895 if self.assembler.arch_has_fconverti() {
6896 self.emit_relaxed_mov(Size::S64, loc, Location::GPR(tmp_in))?;
6897 if signed {
6898 self.assembler.arch_emit_f64_convert_si64(tmp_in, tmp_out)?;
6899 } else {
6900 self.assembler.arch_emit_f64_convert_ui64(tmp_in, tmp_out)?;
6901 }
6902 self.emit_relaxed_mov(Size::S64, Location::SIMD(tmp_out), ret)?;
6903 } else if signed {
6904 self.assembler
6905 .emit_mov(Size::S64, loc, Location::GPR(tmp_in))?;
6906 self.assembler
6907 .emit_vcvtsi2sd_64(tmp_out, GPROrMemory::GPR(tmp_in), tmp_out)?;
6908 self.move_location(Size::S64, Location::SIMD(tmp_out), ret)?;
6909 } else {
6910 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
6911 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6912 })?;
6913
6914 let do_convert = self.assembler.get_label();
6915 let end_convert = self.assembler.get_label();
6916
6917 self.assembler
6918 .emit_mov(Size::S64, loc, Location::GPR(tmp_in))?;
6919 self.assembler.emit_test_gpr_64(tmp_in)?;
6920 self.assembler.emit_jmp(Condition::Signed, do_convert)?;
6921 self.assembler
6922 .emit_vcvtsi2sd_64(tmp_out, GPROrMemory::GPR(tmp_in), tmp_out)?;
6923 self.assembler.emit_jmp(Condition::None, end_convert)?;
6924 self.emit_label(do_convert)?;
6925 self.move_location(Size::S64, Location::GPR(tmp_in), Location::GPR(tmp))?;
6926 self.assembler
6927 .emit_and(Size::S64, Location::Imm32(1), Location::GPR(tmp))?;
6928 self.assembler
6929 .emit_shr(Size::S64, Location::Imm8(1), Location::GPR(tmp_in))?;
6930 self.assembler
6931 .emit_or(Size::S64, Location::GPR(tmp), Location::GPR(tmp_in))?;
6932 self.assembler
6933 .emit_vcvtsi2sd_64(tmp_out, GPROrMemory::GPR(tmp_in), tmp_out)?;
6934 self.assembler
6935 .emit_vaddsd(tmp_out, XMMOrMemory::XMM(tmp_out), tmp_out)?;
6936 self.emit_label(end_convert)?;
6937 self.move_location(Size::S64, Location::SIMD(tmp_out), ret)?;
6938
6939 self.release_gpr(tmp);
6940 }
6941 self.release_gpr(tmp_in);
6942 self.release_simd(tmp_out);
6943 Ok(())
6944 }
6945
6946 fn convert_f64_i32(
6947 &mut self,
6948 loc: Location,
6949 signed: bool,
6950 ret: Location,
6951 ) -> Result<(), CompileError> {
6952 let tmp_out = self.acquire_temp_simd().ok_or_else(|| {
6953 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
6954 })?;
6955 let tmp_in = self.acquire_temp_gpr().ok_or_else(|| {
6956 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6957 })?;
6958 if self.assembler.arch_has_fconverti() {
6959 self.emit_relaxed_mov(Size::S32, loc, Location::GPR(tmp_in))?;
6960 if signed {
6961 self.assembler.arch_emit_f64_convert_si32(tmp_in, tmp_out)?;
6962 } else {
6963 self.assembler.arch_emit_f64_convert_ui32(tmp_in, tmp_out)?;
6964 }
6965 self.emit_relaxed_mov(Size::S64, Location::SIMD(tmp_out), ret)?;
6966 } else {
6967 self.assembler
6968 .emit_mov(Size::S32, loc, Location::GPR(tmp_in))?;
6969 if signed {
6970 self.assembler
6971 .emit_vcvtsi2sd_32(tmp_out, GPROrMemory::GPR(tmp_in), tmp_out)?;
6972 } else {
6973 self.assembler
6974 .emit_vcvtsi2sd_64(tmp_out, GPROrMemory::GPR(tmp_in), tmp_out)?;
6975 }
6976 self.move_location(Size::S64, Location::SIMD(tmp_out), ret)?;
6977 }
6978 self.release_gpr(tmp_in);
6979 self.release_simd(tmp_out);
6980 Ok(())
6981 }
6982
6983 fn convert_f32_i64(
6984 &mut self,
6985 loc: Location,
6986 signed: bool,
6987 ret: Location,
6988 ) -> Result<(), CompileError> {
6989 let tmp_out = self.acquire_temp_simd().ok_or_else(|| {
6990 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
6991 })?;
6992 let tmp_in = self.acquire_temp_gpr().ok_or_else(|| {
6993 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
6994 })?;
6995 if self.assembler.arch_has_fconverti() {
6996 self.emit_relaxed_mov(Size::S64, loc, Location::GPR(tmp_in))?;
6997 if signed {
6998 self.assembler.arch_emit_f32_convert_si64(tmp_in, tmp_out)?;
6999 } else {
7000 self.assembler.arch_emit_f32_convert_ui64(tmp_in, tmp_out)?;
7001 }
7002 self.emit_relaxed_mov(Size::S32, Location::SIMD(tmp_out), ret)?;
7003 } else if signed {
7004 self.assembler
7005 .emit_mov(Size::S64, loc, Location::GPR(tmp_in))?;
7006 self.assembler
7007 .emit_vcvtsi2ss_64(tmp_out, GPROrMemory::GPR(tmp_in), tmp_out)?;
7008 self.move_location(Size::S32, Location::SIMD(tmp_out), ret)?;
7009 } else {
7010 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
7011 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7012 })?;
7013
7014 let do_convert = self.assembler.get_label();
7015 let end_convert = self.assembler.get_label();
7016
7017 self.assembler
7018 .emit_mov(Size::S64, loc, Location::GPR(tmp_in))?;
7019 self.assembler.emit_test_gpr_64(tmp_in)?;
7020 self.assembler.emit_jmp(Condition::Signed, do_convert)?;
7021 self.assembler
7022 .emit_vcvtsi2ss_64(tmp_out, GPROrMemory::GPR(tmp_in), tmp_out)?;
7023 self.assembler.emit_jmp(Condition::None, end_convert)?;
7024 self.emit_label(do_convert)?;
7025 self.move_location(Size::S64, Location::GPR(tmp_in), Location::GPR(tmp))?;
7026 self.assembler
7027 .emit_and(Size::S64, Location::Imm32(1), Location::GPR(tmp))?;
7028 self.assembler
7029 .emit_shr(Size::S64, Location::Imm8(1), Location::GPR(tmp_in))?;
7030 self.assembler
7031 .emit_or(Size::S64, Location::GPR(tmp), Location::GPR(tmp_in))?;
7032 self.assembler
7033 .emit_vcvtsi2ss_64(tmp_out, GPROrMemory::GPR(tmp_in), tmp_out)?;
7034 self.assembler
7035 .emit_vaddss(tmp_out, XMMOrMemory::XMM(tmp_out), tmp_out)?;
7036 self.emit_label(end_convert)?;
7037 self.move_location(Size::S32, Location::SIMD(tmp_out), ret)?;
7038
7039 self.release_gpr(tmp);
7040 }
7041 self.release_gpr(tmp_in);
7042 self.release_simd(tmp_out);
7043 Ok(())
7044 }
7045
7046 fn convert_f32_i32(
7047 &mut self,
7048 loc: Location,
7049 signed: bool,
7050 ret: Location,
7051 ) -> Result<(), CompileError> {
7052 let tmp_out = self.acquire_temp_simd().ok_or_else(|| {
7053 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
7054 })?;
7055 let tmp_in = self.acquire_temp_gpr().ok_or_else(|| {
7056 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7057 })?;
7058 if self.assembler.arch_has_fconverti() {
7059 self.emit_relaxed_mov(Size::S32, loc, Location::GPR(tmp_in))?;
7060 if signed {
7061 self.assembler.arch_emit_f32_convert_si32(tmp_in, tmp_out)?;
7062 } else {
7063 self.assembler.arch_emit_f32_convert_ui32(tmp_in, tmp_out)?;
7064 }
7065 self.emit_relaxed_mov(Size::S32, Location::SIMD(tmp_out), ret)?;
7066 } else {
7067 self.assembler
7068 .emit_mov(Size::S32, loc, Location::GPR(tmp_in))?;
7069 if signed {
7070 self.assembler
7071 .emit_vcvtsi2ss_32(tmp_out, GPROrMemory::GPR(tmp_in), tmp_out)?;
7072 } else {
7073 self.assembler
7074 .emit_vcvtsi2ss_64(tmp_out, GPROrMemory::GPR(tmp_in), tmp_out)?;
7075 }
7076 self.move_location(Size::S32, Location::SIMD(tmp_out), ret)?;
7077 }
7078 self.release_gpr(tmp_in);
7079 self.release_simd(tmp_out);
7080 Ok(())
7081 }
7082
7083 fn convert_i64_f64(
7084 &mut self,
7085 loc: Location,
7086 ret: Location,
7087 signed: bool,
7088 sat: bool,
7089 ) -> Result<(), CompileError> {
7090 match (signed, sat) {
7091 (false, true) => self.convert_i64_f64_u_s(loc, ret),
7092 (false, false) => self.convert_i64_f64_u_u(loc, ret),
7093 (true, true) => self.convert_i64_f64_s_s(loc, ret),
7094 (true, false) => self.convert_i64_f64_s_u(loc, ret),
7095 }
7096 }
7097
7098 fn convert_i32_f64(
7099 &mut self,
7100 loc: Location,
7101 ret: Location,
7102 signed: bool,
7103 sat: bool,
7104 ) -> Result<(), CompileError> {
7105 match (signed, sat) {
7106 (false, true) => self.convert_i32_f64_u_s(loc, ret),
7107 (false, false) => self.convert_i32_f64_u_u(loc, ret),
7108 (true, true) => self.convert_i32_f64_s_s(loc, ret),
7109 (true, false) => self.convert_i32_f64_s_u(loc, ret),
7110 }
7111 }
7112
7113 fn convert_i64_f32(
7114 &mut self,
7115 loc: Location,
7116 ret: Location,
7117 signed: bool,
7118 sat: bool,
7119 ) -> Result<(), CompileError> {
7120 match (signed, sat) {
7121 (false, true) => self.convert_i64_f32_u_s(loc, ret),
7122 (false, false) => self.convert_i64_f32_u_u(loc, ret),
7123 (true, true) => self.convert_i64_f32_s_s(loc, ret),
7124 (true, false) => self.convert_i64_f32_s_u(loc, ret),
7125 }
7126 }
7127
7128 fn convert_i32_f32(
7129 &mut self,
7130 loc: Location,
7131 ret: Location,
7132 signed: bool,
7133 sat: bool,
7134 ) -> Result<(), CompileError> {
7135 match (signed, sat) {
7136 (false, true) => self.convert_i32_f32_u_s(loc, ret),
7137 (false, false) => self.convert_i32_f32_u_u(loc, ret),
7138 (true, true) => self.convert_i32_f32_s_s(loc, ret),
7139 (true, false) => self.convert_i32_f32_s_u(loc, ret),
7140 }
7141 }
7142
7143 fn convert_f64_f32(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7144 self.emit_relaxed_avx(AssemblerX64::emit_vcvtss2sd, loc, loc, ret)
7145 }
7146
7147 fn convert_f32_f64(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7148 self.emit_relaxed_avx(AssemblerX64::emit_vcvtsd2ss, loc, loc, ret)
7149 }
7150
7151 fn f64_neg(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7152 if self.assembler.arch_has_fneg() {
7153 let tmp = self.acquire_temp_simd().ok_or_else(|| {
7154 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
7155 })?;
7156 self.emit_relaxed_mov(Size::S64, loc, Location::SIMD(tmp))?;
7157 self.assembler.arch_emit_f64_neg(tmp, tmp)?;
7158 self.emit_relaxed_mov(Size::S64, Location::SIMD(tmp), ret)?;
7159 self.release_simd(tmp);
7160 } else {
7161 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
7162 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7163 })?;
7164 self.move_location(Size::S64, loc, Location::GPR(tmp))?;
7165 self.assembler.emit_btc_gpr_imm8_64(63, tmp)?;
7166 self.move_location(Size::S64, Location::GPR(tmp), ret)?;
7167 self.release_gpr(tmp);
7168 }
7169 Ok(())
7170 }
7171
7172 fn f64_abs(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7173 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
7174 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7175 })?;
7176 let c = self.acquire_temp_gpr().ok_or_else(|| {
7177 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7178 })?;
7179
7180 self.move_location(Size::S64, loc, Location::GPR(tmp))?;
7181 self.move_location(
7182 Size::S64,
7183 Location::Imm64(0x7fffffffffffffffu64),
7184 Location::GPR(c),
7185 )?;
7186 self.assembler
7187 .emit_and(Size::S64, Location::GPR(c), Location::GPR(tmp))?;
7188 self.move_location(Size::S64, Location::GPR(tmp), ret)?;
7189
7190 self.release_gpr(c);
7191 self.release_gpr(tmp);
7192 Ok(())
7193 }
7194
7195 fn emit_i64_copysign(&mut self, tmp1: GPR, tmp2: GPR) -> Result<(), CompileError> {
7196 let c = self.acquire_temp_gpr().ok_or_else(|| {
7197 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7198 })?;
7199
7200 self.move_location(
7201 Size::S64,
7202 Location::Imm64(0x7fffffffffffffffu64),
7203 Location::GPR(c),
7204 )?;
7205 self.assembler
7206 .emit_and(Size::S64, Location::GPR(c), Location::GPR(tmp1))?;
7207
7208 self.move_location(
7209 Size::S64,
7210 Location::Imm64(0x8000000000000000u64),
7211 Location::GPR(c),
7212 )?;
7213 self.assembler
7214 .emit_and(Size::S64, Location::GPR(c), Location::GPR(tmp2))?;
7215
7216 self.assembler
7217 .emit_or(Size::S64, Location::GPR(tmp2), Location::GPR(tmp1))?;
7218
7219 self.release_gpr(c);
7220 Ok(())
7221 }
7222
7223 fn f64_sqrt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7224 self.emit_relaxed_avx(AssemblerX64::emit_vsqrtsd, loc, loc, ret)
7225 }
7226
7227 fn f64_trunc(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7228 self.emit_relaxed_avx(AssemblerX64::emit_vroundsd_trunc, loc, loc, ret)
7229 }
7230
7231 fn f64_ceil(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7232 self.emit_relaxed_avx(AssemblerX64::emit_vroundsd_ceil, loc, loc, ret)
7233 }
7234
7235 fn f64_floor(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7236 self.emit_relaxed_avx(AssemblerX64::emit_vroundsd_floor, loc, loc, ret)
7237 }
7238
7239 fn f64_nearest(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7240 self.emit_relaxed_avx(AssemblerX64::emit_vroundsd_nearest, loc, loc, ret)
7241 }
7242
7243 fn f64_cmp_ge(
7244 &mut self,
7245 loc_a: Location,
7246 loc_b: Location,
7247 ret: Location,
7248 ) -> Result<(), CompileError> {
7249 self.emit_relaxed_avx(AssemblerX64::emit_vcmpgesd, loc_a, loc_b, ret)?;
7250 self.assembler.emit_and(Size::S32, Location::Imm32(1), ret)
7251 }
7252
7253 fn f64_cmp_gt(
7254 &mut self,
7255 loc_a: Location,
7256 loc_b: Location,
7257 ret: Location,
7258 ) -> Result<(), CompileError> {
7259 self.emit_relaxed_avx(AssemblerX64::emit_vcmpgtsd, loc_a, loc_b, ret)?;
7260 self.assembler.emit_and(Size::S32, Location::Imm32(1), ret)
7261 }
7262
7263 fn f64_cmp_le(
7264 &mut self,
7265 loc_a: Location,
7266 loc_b: Location,
7267 ret: Location,
7268 ) -> Result<(), CompileError> {
7269 self.emit_relaxed_avx(AssemblerX64::emit_vcmplesd, loc_a, loc_b, ret)?;
7270 self.assembler.emit_and(Size::S32, Location::Imm32(1), ret)
7271 }
7272
7273 fn f64_cmp_lt(
7274 &mut self,
7275 loc_a: Location,
7276 loc_b: Location,
7277 ret: Location,
7278 ) -> Result<(), CompileError> {
7279 self.emit_relaxed_avx(AssemblerX64::emit_vcmpltsd, loc_a, loc_b, ret)?;
7280 self.assembler.emit_and(Size::S32, Location::Imm32(1), ret)
7281 }
7282
7283 fn f64_cmp_ne(
7284 &mut self,
7285 loc_a: Location,
7286 loc_b: Location,
7287 ret: Location,
7288 ) -> Result<(), CompileError> {
7289 self.emit_relaxed_avx(AssemblerX64::emit_vcmpneqsd, loc_a, loc_b, ret)?;
7290 self.assembler.emit_and(Size::S32, Location::Imm32(1), ret)
7291 }
7292
7293 fn f64_cmp_eq(
7294 &mut self,
7295 loc_a: Location,
7296 loc_b: Location,
7297 ret: Location,
7298 ) -> Result<(), CompileError> {
7299 self.emit_relaxed_avx(AssemblerX64::emit_vcmpeqsd, loc_a, loc_b, ret)?;
7300 self.assembler.emit_and(Size::S32, Location::Imm32(1), ret)
7301 }
7302
7303 fn f64_min(
7304 &mut self,
7305 loc_a: Location,
7306 loc_b: Location,
7307 ret: Location,
7308 ) -> Result<(), CompileError> {
7309 let tmp1 = self.acquire_temp_simd().ok_or_else(|| {
7311 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
7312 })?;
7313 let tmp2 = self.acquire_temp_simd().ok_or_else(|| {
7314 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
7315 })?;
7316 let tmpg1 = self.acquire_temp_gpr().ok_or_else(|| {
7317 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7318 })?;
7319 let tmpg2 = self.acquire_temp_gpr().ok_or_else(|| {
7320 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7321 })?;
7322
7323 let src1 = match loc_a {
7324 Location::SIMD(x) => x,
7325 Location::GPR(_) | Location::Memory(_, _) => {
7326 self.move_location(Size::S64, loc_a, Location::SIMD(tmp1))?;
7327 tmp1
7328 }
7329 Location::Imm32(_) => {
7330 self.move_location(Size::S32, loc_a, Location::GPR(tmpg1))?;
7331 self.move_location(Size::S32, Location::GPR(tmpg1), Location::SIMD(tmp1))?;
7332 tmp1
7333 }
7334 Location::Imm64(_) => {
7335 self.move_location(Size::S64, loc_a, Location::GPR(tmpg1))?;
7336 self.move_location(Size::S64, Location::GPR(tmpg1), Location::SIMD(tmp1))?;
7337 tmp1
7338 }
7339 _ => {
7340 codegen_error!("singlepass f64_min unreachable");
7341 }
7342 };
7343 let src2 = match loc_b {
7344 Location::SIMD(x) => x,
7345 Location::GPR(_) | Location::Memory(_, _) => {
7346 self.move_location(Size::S64, loc_b, Location::SIMD(tmp2))?;
7347 tmp2
7348 }
7349 Location::Imm32(_) => {
7350 self.move_location(Size::S32, loc_b, Location::GPR(tmpg1))?;
7351 self.move_location(Size::S32, Location::GPR(tmpg1), Location::SIMD(tmp2))?;
7352 tmp2
7353 }
7354 Location::Imm64(_) => {
7355 self.move_location(Size::S64, loc_b, Location::GPR(tmpg1))?;
7356 self.move_location(Size::S64, Location::GPR(tmpg1), Location::SIMD(tmp2))?;
7357 tmp2
7358 }
7359 _ => {
7360 codegen_error!("singlepass f64_min unreachable");
7361 }
7362 };
7363
7364 let tmp_xmm1 = XMM::XMM8;
7365 let tmp_xmm2 = XMM::XMM9;
7366 let tmp_xmm3 = XMM::XMM10;
7367
7368 self.move_location(Size::S64, Location::SIMD(src1), Location::GPR(tmpg1))?;
7369 self.move_location(Size::S64, Location::SIMD(src2), Location::GPR(tmpg2))?;
7370 self.emit_relaxed_cmp(Size::S64, Location::GPR(tmpg2), Location::GPR(tmpg1))?;
7371 self.assembler
7372 .emit_vminsd(src1, XMMOrMemory::XMM(src2), tmp_xmm1)?;
7373 let label1 = self.assembler.get_label();
7374 let label2 = self.assembler.get_label();
7375 self.assembler.emit_jmp(Condition::NotEqual, label1)?;
7376 self.assembler
7377 .emit_vmovapd(XMMOrMemory::XMM(tmp_xmm1), XMMOrMemory::XMM(tmp_xmm2))?;
7378 self.assembler.emit_jmp(Condition::None, label2)?;
7379 self.emit_label(label1)?;
7380 self.move_location(
7382 Size::S64,
7383 Location::Imm64(0x8000_0000_0000_0000), Location::GPR(tmpg1),
7385 )?;
7386 self.move_location(Size::S64, Location::GPR(tmpg1), Location::SIMD(tmp_xmm2))?;
7387 self.emit_label(label2)?;
7388 self.assembler
7389 .emit_vcmpeqsd(src1, XMMOrMemory::XMM(src2), tmp_xmm3)?;
7390 self.assembler
7391 .emit_vblendvpd(tmp_xmm3, XMMOrMemory::XMM(tmp_xmm2), tmp_xmm1, tmp_xmm1)?;
7392 self.assembler
7393 .emit_vcmpunordsd(src1, XMMOrMemory::XMM(src2), src1)?;
7394 self.move_location(
7396 Size::S64,
7397 Location::Imm64(0x7FF8_0000_0000_0000), Location::GPR(tmpg1),
7399 )?;
7400 self.move_location(Size::S64, Location::GPR(tmpg1), Location::SIMD(src2))?;
7401 self.assembler
7402 .emit_vblendvpd(src1, XMMOrMemory::XMM(src2), tmp_xmm1, src1)?;
7403 match ret {
7404 Location::SIMD(x) => {
7405 self.assembler
7406 .emit_vmovaps(XMMOrMemory::XMM(src1), XMMOrMemory::XMM(x))?;
7407 }
7408 Location::Memory(_, _) | Location::GPR(_) => {
7409 self.move_location(Size::S64, Location::SIMD(src1), ret)?;
7410 }
7411 _ => {
7412 codegen_error!("singlepass f64_min unreachable");
7413 }
7414 }
7415
7416 self.release_gpr(tmpg2);
7417 self.release_gpr(tmpg1);
7418 self.release_simd(tmp2);
7419 self.release_simd(tmp1);
7420 Ok(())
7421 }
7422
7423 fn f64_max(
7424 &mut self,
7425 loc_a: Location,
7426 loc_b: Location,
7427 ret: Location,
7428 ) -> Result<(), CompileError> {
7429 let tmp1 = self.acquire_temp_simd().ok_or_else(|| {
7431 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
7432 })?;
7433 let tmp2 = self.acquire_temp_simd().ok_or_else(|| {
7434 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
7435 })?;
7436 let tmpg1 = self.acquire_temp_gpr().ok_or_else(|| {
7437 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7438 })?;
7439 let tmpg2 = self.acquire_temp_gpr().ok_or_else(|| {
7440 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7441 })?;
7442
7443 let src1 = match loc_a {
7444 Location::SIMD(x) => x,
7445 Location::GPR(_) | Location::Memory(_, _) => {
7446 self.move_location(Size::S64, loc_a, Location::SIMD(tmp1))?;
7447 tmp1
7448 }
7449 Location::Imm32(_) => {
7450 self.move_location(Size::S32, loc_a, Location::GPR(tmpg1))?;
7451 self.move_location(Size::S32, Location::GPR(tmpg1), Location::SIMD(tmp1))?;
7452 tmp1
7453 }
7454 Location::Imm64(_) => {
7455 self.move_location(Size::S64, loc_a, Location::GPR(tmpg1))?;
7456 self.move_location(Size::S64, Location::GPR(tmpg1), Location::SIMD(tmp1))?;
7457 tmp1
7458 }
7459 _ => {
7460 codegen_error!("singlepass f64_max unreachable");
7461 }
7462 };
7463 let src2 = match loc_b {
7464 Location::SIMD(x) => x,
7465 Location::GPR(_) | Location::Memory(_, _) => {
7466 self.move_location(Size::S64, loc_b, Location::SIMD(tmp2))?;
7467 tmp2
7468 }
7469 Location::Imm32(_) => {
7470 self.move_location(Size::S32, loc_b, Location::GPR(tmpg1))?;
7471 self.move_location(Size::S32, Location::GPR(tmpg1), Location::SIMD(tmp2))?;
7472 tmp2
7473 }
7474 Location::Imm64(_) => {
7475 self.move_location(Size::S64, loc_b, Location::GPR(tmpg1))?;
7476 self.move_location(Size::S64, Location::GPR(tmpg1), Location::SIMD(tmp2))?;
7477 tmp2
7478 }
7479 _ => {
7480 codegen_error!("singlepass f64_max unreachable");
7481 }
7482 };
7483
7484 let tmp_xmm1 = XMM::XMM8;
7485 let tmp_xmm2 = XMM::XMM9;
7486 let tmp_xmm3 = XMM::XMM10;
7487
7488 self.move_location(Size::S64, Location::SIMD(src1), Location::GPR(tmpg1))?;
7489 self.move_location(Size::S64, Location::SIMD(src2), Location::GPR(tmpg2))?;
7490 self.emit_relaxed_cmp(Size::S64, Location::GPR(tmpg2), Location::GPR(tmpg1))?;
7491 self.assembler
7492 .emit_vmaxsd(src1, XMMOrMemory::XMM(src2), tmp_xmm1)?;
7493 let label1 = self.assembler.get_label();
7494 let label2 = self.assembler.get_label();
7495 self.assembler.emit_jmp(Condition::NotEqual, label1)?;
7496 self.assembler
7497 .emit_vmovapd(XMMOrMemory::XMM(tmp_xmm1), XMMOrMemory::XMM(tmp_xmm2))?;
7498 self.assembler.emit_jmp(Condition::None, label2)?;
7499 self.emit_label(label1)?;
7500 self.assembler
7501 .emit_vxorpd(tmp_xmm2, XMMOrMemory::XMM(tmp_xmm2), tmp_xmm2)?;
7502 self.emit_label(label2)?;
7503 self.assembler
7504 .emit_vcmpeqsd(src1, XMMOrMemory::XMM(src2), tmp_xmm3)?;
7505 self.assembler
7506 .emit_vblendvpd(tmp_xmm3, XMMOrMemory::XMM(tmp_xmm2), tmp_xmm1, tmp_xmm1)?;
7507 self.assembler
7508 .emit_vcmpunordsd(src1, XMMOrMemory::XMM(src2), src1)?;
7509 self.move_location(
7511 Size::S64,
7512 Location::Imm64(0x7FF8_0000_0000_0000), Location::GPR(tmpg1),
7514 )?;
7515 self.move_location(Size::S64, Location::GPR(tmpg1), Location::SIMD(src2))?;
7516 self.assembler
7517 .emit_vblendvpd(src1, XMMOrMemory::XMM(src2), tmp_xmm1, src1)?;
7518 match ret {
7519 Location::SIMD(x) => {
7520 self.assembler
7521 .emit_vmovapd(XMMOrMemory::XMM(src1), XMMOrMemory::XMM(x))?;
7522 }
7523 Location::Memory(_, _) | Location::GPR(_) => {
7524 self.move_location(Size::S64, Location::SIMD(src1), ret)?;
7525 }
7526 _ => {
7527 codegen_error!("singlepass f64_max unreachable");
7528 }
7529 }
7530
7531 self.release_gpr(tmpg2);
7532 self.release_gpr(tmpg1);
7533 self.release_simd(tmp2);
7534 self.release_simd(tmp1);
7535 Ok(())
7536 }
7537
7538 fn f64_add(
7539 &mut self,
7540 loc_a: Location,
7541 loc_b: Location,
7542 ret: Location,
7543 ) -> Result<(), CompileError> {
7544 self.emit_relaxed_avx(AssemblerX64::emit_vaddsd, loc_a, loc_b, ret)
7545 }
7546
7547 fn f64_sub(
7548 &mut self,
7549 loc_a: Location,
7550 loc_b: Location,
7551 ret: Location,
7552 ) -> Result<(), CompileError> {
7553 self.emit_relaxed_avx(AssemblerX64::emit_vsubsd, loc_a, loc_b, ret)
7554 }
7555
7556 fn f64_mul(
7557 &mut self,
7558 loc_a: Location,
7559 loc_b: Location,
7560 ret: Location,
7561 ) -> Result<(), CompileError> {
7562 self.emit_relaxed_avx(AssemblerX64::emit_vmulsd, loc_a, loc_b, ret)
7563 }
7564
7565 fn f64_div(
7566 &mut self,
7567 loc_a: Location,
7568 loc_b: Location,
7569 ret: Location,
7570 ) -> Result<(), CompileError> {
7571 self.emit_relaxed_avx(AssemblerX64::emit_vdivsd, loc_a, loc_b, ret)
7572 }
7573
7574 fn f32_neg(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7575 if self.assembler.arch_has_fneg() {
7576 let tmp = self.acquire_temp_simd().ok_or_else(|| {
7577 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
7578 })?;
7579 self.emit_relaxed_mov(Size::S32, loc, Location::SIMD(tmp))?;
7580 self.assembler.arch_emit_f32_neg(tmp, tmp)?;
7581 self.emit_relaxed_mov(Size::S32, Location::SIMD(tmp), ret)?;
7582 self.release_simd(tmp);
7583 } else {
7584 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
7585 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7586 })?;
7587 self.move_location(Size::S32, loc, Location::GPR(tmp))?;
7588 self.assembler.emit_btc_gpr_imm8_32(31, tmp)?;
7589 self.move_location(Size::S32, Location::GPR(tmp), ret)?;
7590 self.release_gpr(tmp);
7591 }
7592 Ok(())
7593 }
7594
7595 fn f32_abs(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7596 let tmp = self.acquire_temp_gpr().ok_or_else(|| {
7597 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7598 })?;
7599 self.move_location(Size::S32, loc, Location::GPR(tmp))?;
7600 self.assembler.emit_and(
7601 Size::S32,
7602 Location::Imm32(0x7fffffffu32),
7603 Location::GPR(tmp),
7604 )?;
7605 self.move_location(Size::S32, Location::GPR(tmp), ret)?;
7606 self.release_gpr(tmp);
7607 Ok(())
7608 }
7609
7610 fn emit_i32_copysign(&mut self, tmp1: GPR, tmp2: GPR) -> Result<(), CompileError> {
7611 self.assembler.emit_and(
7612 Size::S32,
7613 Location::Imm32(0x7fffffffu32),
7614 Location::GPR(tmp1),
7615 )?;
7616 self.assembler.emit_and(
7617 Size::S32,
7618 Location::Imm32(0x80000000u32),
7619 Location::GPR(tmp2),
7620 )?;
7621 self.assembler
7622 .emit_or(Size::S32, Location::GPR(tmp2), Location::GPR(tmp1))
7623 }
7624
7625 fn f32_sqrt(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7626 self.emit_relaxed_avx(AssemblerX64::emit_vsqrtss, loc, loc, ret)
7627 }
7628
7629 fn f32_trunc(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7630 self.emit_relaxed_avx(AssemblerX64::emit_vroundss_trunc, loc, loc, ret)
7631 }
7632
7633 fn f32_ceil(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7634 self.emit_relaxed_avx(AssemblerX64::emit_vroundss_ceil, loc, loc, ret)
7635 }
7636
7637 fn f32_floor(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7638 self.emit_relaxed_avx(AssemblerX64::emit_vroundss_floor, loc, loc, ret)
7639 }
7640
7641 fn f32_nearest(&mut self, loc: Location, ret: Location) -> Result<(), CompileError> {
7642 self.emit_relaxed_avx(AssemblerX64::emit_vroundss_nearest, loc, loc, ret)
7643 }
7644
7645 fn f32_cmp_ge(
7646 &mut self,
7647 loc_a: Location,
7648 loc_b: Location,
7649 ret: Location,
7650 ) -> Result<(), CompileError> {
7651 self.emit_relaxed_avx(AssemblerX64::emit_vcmpgess, loc_a, loc_b, ret)?;
7652 self.assembler.emit_and(Size::S32, Location::Imm32(1), ret)
7653 }
7654
7655 fn f32_cmp_gt(
7656 &mut self,
7657 loc_a: Location,
7658 loc_b: Location,
7659 ret: Location,
7660 ) -> Result<(), CompileError> {
7661 self.emit_relaxed_avx(AssemblerX64::emit_vcmpgtss, loc_a, loc_b, ret)?;
7662 self.assembler.emit_and(Size::S32, Location::Imm32(1), ret)
7663 }
7664
7665 fn f32_cmp_le(
7666 &mut self,
7667 loc_a: Location,
7668 loc_b: Location,
7669 ret: Location,
7670 ) -> Result<(), CompileError> {
7671 self.emit_relaxed_avx(AssemblerX64::emit_vcmpless, loc_a, loc_b, ret)?;
7672 self.assembler.emit_and(Size::S32, Location::Imm32(1), ret)
7673 }
7674
7675 fn f32_cmp_lt(
7676 &mut self,
7677 loc_a: Location,
7678 loc_b: Location,
7679 ret: Location,
7680 ) -> Result<(), CompileError> {
7681 self.emit_relaxed_avx(AssemblerX64::emit_vcmpltss, loc_a, loc_b, ret)?;
7682 self.assembler.emit_and(Size::S32, Location::Imm32(1), ret)
7683 }
7684
7685 fn f32_cmp_ne(
7686 &mut self,
7687 loc_a: Location,
7688 loc_b: Location,
7689 ret: Location,
7690 ) -> Result<(), CompileError> {
7691 self.emit_relaxed_avx(AssemblerX64::emit_vcmpneqss, loc_a, loc_b, ret)?;
7692 self.assembler.emit_and(Size::S32, Location::Imm32(1), ret)
7693 }
7694
7695 fn f32_cmp_eq(
7696 &mut self,
7697 loc_a: Location,
7698 loc_b: Location,
7699 ret: Location,
7700 ) -> Result<(), CompileError> {
7701 self.emit_relaxed_avx(AssemblerX64::emit_vcmpeqss, loc_a, loc_b, ret)?;
7702 self.assembler.emit_and(Size::S32, Location::Imm32(1), ret)
7703 }
7704
7705 fn f32_min(
7706 &mut self,
7707 loc_a: Location,
7708 loc_b: Location,
7709 ret: Location,
7710 ) -> Result<(), CompileError> {
7711 let tmp1 = self.acquire_temp_simd().ok_or_else(|| {
7713 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
7714 })?;
7715 let tmp2 = self.acquire_temp_simd().ok_or_else(|| {
7716 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
7717 })?;
7718 let tmpg1 = self.acquire_temp_gpr().ok_or_else(|| {
7719 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7720 })?;
7721 let tmpg2 = self.acquire_temp_gpr().ok_or_else(|| {
7722 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7723 })?;
7724
7725 let src1 = match loc_a {
7726 Location::SIMD(x) => x,
7727 Location::GPR(_) | Location::Memory(_, _) => {
7728 self.move_location(Size::S64, loc_a, Location::SIMD(tmp1))?;
7729 tmp1
7730 }
7731 Location::Imm32(_) => {
7732 self.move_location(Size::S32, loc_a, Location::GPR(tmpg1))?;
7733 self.move_location(Size::S32, Location::GPR(tmpg1), Location::SIMD(tmp1))?;
7734 tmp1
7735 }
7736 Location::Imm64(_) => {
7737 self.move_location(Size::S64, loc_a, Location::GPR(tmpg1))?;
7738 self.move_location(Size::S64, Location::GPR(tmpg1), Location::SIMD(tmp1))?;
7739 tmp1
7740 }
7741 _ => {
7742 codegen_error!("singlepass f32_min unreachable");
7743 }
7744 };
7745 let src2 = match loc_b {
7746 Location::SIMD(x) => x,
7747 Location::GPR(_) | Location::Memory(_, _) => {
7748 self.move_location(Size::S64, loc_b, Location::SIMD(tmp2))?;
7749 tmp2
7750 }
7751 Location::Imm32(_) => {
7752 self.move_location(Size::S32, loc_b, Location::GPR(tmpg1))?;
7753 self.move_location(Size::S32, Location::GPR(tmpg1), Location::SIMD(tmp2))?;
7754 tmp2
7755 }
7756 Location::Imm64(_) => {
7757 self.move_location(Size::S64, loc_b, Location::GPR(tmpg1))?;
7758 self.move_location(Size::S64, Location::GPR(tmpg1), Location::SIMD(tmp2))?;
7759 tmp2
7760 }
7761 _ => {
7762 codegen_error!("singlepass f32_min unreachable");
7763 }
7764 };
7765
7766 let tmp_xmm1 = XMM::XMM8;
7767 let tmp_xmm2 = XMM::XMM9;
7768 let tmp_xmm3 = XMM::XMM10;
7769
7770 self.move_location(Size::S32, Location::SIMD(src1), Location::GPR(tmpg1))?;
7771 self.move_location(Size::S32, Location::SIMD(src2), Location::GPR(tmpg2))?;
7772 self.emit_relaxed_cmp(Size::S32, Location::GPR(tmpg2), Location::GPR(tmpg1))?;
7773 self.assembler
7774 .emit_vminss(src1, XMMOrMemory::XMM(src2), tmp_xmm1)?;
7775 let label1 = self.assembler.get_label();
7776 let label2 = self.assembler.get_label();
7777 self.assembler.emit_jmp(Condition::NotEqual, label1)?;
7778 self.assembler
7779 .emit_vmovaps(XMMOrMemory::XMM(tmp_xmm1), XMMOrMemory::XMM(tmp_xmm2))?;
7780 self.assembler.emit_jmp(Condition::None, label2)?;
7781 self.emit_label(label1)?;
7782 self.move_location(
7784 Size::S64,
7785 Location::Imm32(0x8000_0000), Location::GPR(tmpg1),
7787 )?;
7788 self.move_location(Size::S64, Location::GPR(tmpg1), Location::SIMD(tmp_xmm2))?;
7789 self.emit_label(label2)?;
7790 self.assembler
7791 .emit_vcmpeqss(src1, XMMOrMemory::XMM(src2), tmp_xmm3)?;
7792 self.assembler
7793 .emit_vblendvps(tmp_xmm3, XMMOrMemory::XMM(tmp_xmm2), tmp_xmm1, tmp_xmm1)?;
7794 self.assembler
7795 .emit_vcmpunordss(src1, XMMOrMemory::XMM(src2), src1)?;
7796 self.move_location(
7798 Size::S64,
7799 Location::Imm32(0x7FC0_0000), Location::GPR(tmpg1),
7801 )?;
7802 self.move_location(Size::S64, Location::GPR(tmpg1), Location::SIMD(src2))?;
7803 self.assembler
7804 .emit_vblendvps(src1, XMMOrMemory::XMM(src2), tmp_xmm1, src1)?;
7805 match ret {
7806 Location::SIMD(x) => {
7807 self.assembler
7808 .emit_vmovaps(XMMOrMemory::XMM(src1), XMMOrMemory::XMM(x))?;
7809 }
7810 Location::Memory(_, _) | Location::GPR(_) => {
7811 self.move_location(Size::S64, Location::SIMD(src1), ret)?;
7812 }
7813 _ => {
7814 codegen_error!("singlepass f32_min unreachable");
7815 }
7816 }
7817
7818 self.release_gpr(tmpg2);
7819 self.release_gpr(tmpg1);
7820 self.release_simd(tmp2);
7821 self.release_simd(tmp1);
7822 Ok(())
7823 }
7824
7825 fn f32_max(
7826 &mut self,
7827 loc_a: Location,
7828 loc_b: Location,
7829 ret: Location,
7830 ) -> Result<(), CompileError> {
7831 let tmp1 = self.acquire_temp_simd().ok_or_else(|| {
7833 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
7834 })?;
7835 let tmp2 = self.acquire_temp_simd().ok_or_else(|| {
7836 CompileError::Codegen("singlepass cannot acquire temp simd".to_owned())
7837 })?;
7838 let tmpg1 = self.acquire_temp_gpr().ok_or_else(|| {
7839 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7840 })?;
7841 let tmpg2 = self.acquire_temp_gpr().ok_or_else(|| {
7842 CompileError::Codegen("singlepass cannot acquire temp gpr".to_owned())
7843 })?;
7844
7845 let src1 = match loc_a {
7846 Location::SIMD(x) => x,
7847 Location::GPR(_) | Location::Memory(_, _) => {
7848 self.move_location(Size::S64, loc_a, Location::SIMD(tmp1))?;
7849 tmp1
7850 }
7851 Location::Imm32(_) => {
7852 self.move_location(Size::S32, loc_a, Location::GPR(tmpg1))?;
7853 self.move_location(Size::S32, Location::GPR(tmpg1), Location::SIMD(tmp1))?;
7854 tmp1
7855 }
7856 Location::Imm64(_) => {
7857 self.move_location(Size::S64, loc_a, Location::GPR(tmpg1))?;
7858 self.move_location(Size::S64, Location::GPR(tmpg1), Location::SIMD(tmp1))?;
7859 tmp1
7860 }
7861 _ => {
7862 codegen_error!("singlepass f32_max unreachable");
7863 }
7864 };
7865 let src2 = match loc_b {
7866 Location::SIMD(x) => x,
7867 Location::GPR(_) | Location::Memory(_, _) => {
7868 self.move_location(Size::S64, loc_b, Location::SIMD(tmp2))?;
7869 tmp2
7870 }
7871 Location::Imm32(_) => {
7872 self.move_location(Size::S32, loc_b, Location::GPR(tmpg1))?;
7873 self.move_location(Size::S32, Location::GPR(tmpg1), Location::SIMD(tmp2))?;
7874 tmp2
7875 }
7876 Location::Imm64(_) => {
7877 self.move_location(Size::S64, loc_b, Location::GPR(tmpg1))?;
7878 self.move_location(Size::S64, Location::GPR(tmpg1), Location::SIMD(tmp2))?;
7879 tmp2
7880 }
7881 _ => {
7882 codegen_error!("singlepass f32_max unreachable");
7883 }
7884 };
7885
7886 let tmp_xmm1 = XMM::XMM8;
7887 let tmp_xmm2 = XMM::XMM9;
7888 let tmp_xmm3 = XMM::XMM10;
7889
7890 self.move_location(Size::S32, Location::SIMD(src1), Location::GPR(tmpg1))?;
7891 self.move_location(Size::S32, Location::SIMD(src2), Location::GPR(tmpg2))?;
7892 self.emit_relaxed_cmp(Size::S32, Location::GPR(tmpg2), Location::GPR(tmpg1))?;
7893 self.assembler
7894 .emit_vmaxss(src1, XMMOrMemory::XMM(src2), tmp_xmm1)?;
7895 let label1 = self.assembler.get_label();
7896 let label2 = self.assembler.get_label();
7897 self.assembler.emit_jmp(Condition::NotEqual, label1)?;
7898 self.assembler
7899 .emit_vmovaps(XMMOrMemory::XMM(tmp_xmm1), XMMOrMemory::XMM(tmp_xmm2))?;
7900 self.assembler.emit_jmp(Condition::None, label2)?;
7901 self.emit_label(label1)?;
7902 self.assembler
7903 .emit_vxorps(tmp_xmm2, XMMOrMemory::XMM(tmp_xmm2), tmp_xmm2)?;
7904 self.emit_label(label2)?;
7905 self.assembler
7906 .emit_vcmpeqss(src1, XMMOrMemory::XMM(src2), tmp_xmm3)?;
7907 self.assembler
7908 .emit_vblendvps(tmp_xmm3, XMMOrMemory::XMM(tmp_xmm2), tmp_xmm1, tmp_xmm1)?;
7909 self.assembler
7910 .emit_vcmpunordss(src1, XMMOrMemory::XMM(src2), src1)?;
7911 self.move_location(
7913 Size::S64,
7914 Location::Imm32(0x7FC0_0000), Location::GPR(tmpg1),
7916 )?;
7917 self.move_location(Size::S64, Location::GPR(tmpg1), Location::SIMD(src2))?;
7918 self.assembler
7919 .emit_vblendvps(src1, XMMOrMemory::XMM(src2), tmp_xmm1, src1)?;
7920 match ret {
7921 Location::SIMD(x) => {
7922 self.assembler
7923 .emit_vmovaps(XMMOrMemory::XMM(src1), XMMOrMemory::XMM(x))?;
7924 }
7925 Location::Memory(_, _) | Location::GPR(_) => {
7926 self.move_location(Size::S64, Location::SIMD(src1), ret)?;
7927 }
7928 _ => {
7929 codegen_error!("singlepass f32_max unreachable");
7930 }
7931 }
7932
7933 self.release_gpr(tmpg2);
7934 self.release_gpr(tmpg1);
7935 self.release_simd(tmp2);
7936 self.release_simd(tmp1);
7937 Ok(())
7938 }
7939
7940 fn f32_add(
7941 &mut self,
7942 loc_a: Location,
7943 loc_b: Location,
7944 ret: Location,
7945 ) -> Result<(), CompileError> {
7946 self.emit_relaxed_avx(AssemblerX64::emit_vaddss, loc_a, loc_b, ret)
7947 }
7948
7949 fn f32_sub(
7950 &mut self,
7951 loc_a: Location,
7952 loc_b: Location,
7953 ret: Location,
7954 ) -> Result<(), CompileError> {
7955 self.emit_relaxed_avx(AssemblerX64::emit_vsubss, loc_a, loc_b, ret)
7956 }
7957
7958 fn f32_mul(
7959 &mut self,
7960 loc_a: Location,
7961 loc_b: Location,
7962 ret: Location,
7963 ) -> Result<(), CompileError> {
7964 self.emit_relaxed_avx(AssemblerX64::emit_vmulss, loc_a, loc_b, ret)
7965 }
7966
7967 fn f32_div(
7968 &mut self,
7969 loc_a: Location,
7970 loc_b: Location,
7971 ret: Location,
7972 ) -> Result<(), CompileError> {
7973 self.emit_relaxed_avx(AssemblerX64::emit_vdivss, loc_a, loc_b, ret)
7974 }
7975
7976 fn gen_std_trampoline(
7977 &self,
7978 sig: &FunctionType,
7979 _calling_convention: CallingConvention,
7980 progress_callback: Option<&CompilationProgressCallback>,
7981 ) -> Result<FunctionBody, CompileError> {
7982 let mut a = AssemblerX64::new(0, None)?;
7984 let mut output_reporter = ChunkedOutputReporter::new(progress_callback);
7985
7986 let stack_params = (0..sig.params().len() + 1)
7988 .filter(|&i| self.get_param_registers().get(i).is_none())
7989 .count();
7990 let stack_return_slots = sig
7991 .results()
7992 .len()
7993 .saturating_sub(X86_64_RETURN_VALUE_REGISTERS.len());
7994
7995 let mut stack_offset = 8 * (stack_params + stack_return_slots) as u32;
7997
7998 if stack_offset % 16 != 8 {
8000 stack_offset += 8;
8001 }
8002
8003 a.emit_push(Size::S64, Location::GPR(GPR::R15))?;
8005 a.emit_push(Size::S64, Location::GPR(GPR::R14))?;
8006
8007 a.emit_sub(
8009 Size::S64,
8010 Location::Imm32(stack_offset),
8011 Location::GPR(GPR::RSP),
8012 )?;
8013
8014 a.emit_mov(
8016 Size::S64,
8017 Location::GPR(self.get_simple_param_location(1)),
8018 Location::GPR(GPR::R15),
8019 )?; a.emit_mov(
8021 Size::S64,
8022 Location::GPR(self.get_simple_param_location(2)),
8023 Location::GPR(GPR::R14),
8024 )?; {
8029 let mut n_stack_args = 0u32;
8030 for (i, _param) in sig.params().iter().enumerate() {
8031 let src_loc = Location::Memory(GPR::R14, (i * 16) as _); let dst_loc = self.get_param_registers().get(1 + i);
8033
8034 match dst_loc {
8035 Some(&gpr) => {
8036 a.emit_mov(Size::S64, src_loc, Location::GPR(gpr))?;
8037 }
8038 None => {
8039 a.emit_mov(Size::S64, src_loc, Location::GPR(GPR::RAX))?;
8042 a.emit_mov(
8043 Size::S64,
8044 Location::GPR(GPR::RAX),
8045 Location::Memory(
8046 GPR::RSP,
8047 ((n_stack_args + stack_return_slots as u32) * 8) as _,
8048 ),
8049 )?;
8050 n_stack_args += 1;
8051 }
8052 }
8053 output_reporter.check(a.get_offset().0)?;
8054 }
8055 }
8056
8057 a.emit_call_location(Location::GPR(GPR::R15))?;
8059
8060 let mut n_stack_return_slots: usize = 0;
8062 for i in 0..sig.results().len() {
8063 let src = if let Some(®) = X86_64_RETURN_VALUE_REGISTERS.get(i) {
8064 Location::GPR(reg)
8065 } else {
8066 let loc = Location::GPR(GPR::R15);
8067 a.emit_mov(
8068 Size::S64,
8069 Location::Memory(GPR::RSP, (n_stack_return_slots as u32 * 8) as _),
8070 loc,
8071 )?;
8072 n_stack_return_slots += 1;
8073 loc
8074 };
8075 a.emit_mov(Size::S64, src, Location::Memory(GPR::R14, (i * 16) as _))?;
8076 output_reporter.check(a.get_offset().0)?;
8077 }
8078
8079 a.emit_add(
8081 Size::S64,
8082 Location::Imm32(stack_offset),
8083 Location::GPR(GPR::RSP),
8084 )?;
8085
8086 a.emit_pop(Size::S64, Location::GPR(GPR::R14))?;
8088 a.emit_pop(Size::S64, Location::GPR(GPR::R15))?;
8089
8090 a.emit_ret()?;
8091
8092 let mut body = a.finalize().unwrap();
8093 body.shrink_to_fit();
8094 output_reporter.finish(body.len())?;
8095
8096 Ok(FunctionBody {
8097 body,
8098 unwind_info: None,
8099 })
8100 }
8101 fn gen_std_dynamic_import_trampoline(
8104 &self,
8105 vmoffsets: &VMOffsets,
8106 sig: &FunctionType,
8107 calling_convention: CallingConvention,
8108 progress_callback: Option<&CompilationProgressCallback>,
8109 ) -> Result<FunctionBody, CompileError> {
8110 let mut a = AssemblerX64::new(0, None)?;
8112 let mut output_reporter = ChunkedOutputReporter::new(progress_callback);
8113
8114 let stack_offset: usize = 16 * std::cmp::max(sig.params().len(), sig.results().len()) + 8; a.emit_sub(
8117 Size::S64,
8118 Location::Imm32(stack_offset as _),
8119 Location::GPR(GPR::RSP),
8120 )?;
8121
8122 if !sig.params().is_empty() {
8124 let mut argalloc = ArgumentRegisterAllocator::default();
8125 argalloc.next(Type::I64, calling_convention).unwrap(); let mut stack_param_count: usize = 0;
8128
8129 for (i, ty) in sig.params().iter().enumerate() {
8130 let source_loc = match argalloc.next(*ty, calling_convention)? {
8131 Some(X64Register::GPR(gpr)) => Location::GPR(gpr),
8132 Some(X64Register::XMM(xmm)) => Location::SIMD(xmm),
8133 None => {
8134 a.emit_mov(
8135 Size::S64,
8136 Location::Memory(
8137 GPR::RSP,
8138 (stack_offset + 8 + stack_param_count * 8) as _,
8139 ),
8140 Location::GPR(GPR::RAX),
8141 )?;
8142 stack_param_count += 1;
8143 Location::GPR(GPR::RAX)
8144 }
8145 };
8146 a.emit_mov(
8147 Size::S64,
8148 source_loc,
8149 Location::Memory(GPR::RSP, (i * 16) as _),
8150 )?;
8151
8152 a.emit_mov(
8154 Size::S64,
8155 Location::Imm32(0),
8156 Location::Memory(GPR::RSP, (i * 16 + 8) as _),
8157 )?;
8158 output_reporter.check(a.get_offset().0)?;
8159 }
8160 }
8161
8162 a.emit_mov(
8164 Size::S64,
8165 Location::Memory(
8166 GPR::RDI,
8167 vmoffsets.vmdynamicfunction_import_context_address() as i32,
8168 ),
8169 Location::GPR(GPR::RAX),
8170 )?;
8171 a.emit_mov(Size::S64, Location::GPR(GPR::RSP), Location::GPR(GPR::RSI))?;
8173
8174 a.emit_call_location(Location::GPR(GPR::RAX))?;
8176
8177 if !sig.results().is_empty() {
8179 assert_eq!(sig.results().len(), 1);
8180 a.emit_mov(
8181 Size::S64,
8182 Location::Memory(GPR::RSP, 0),
8183 Location::GPR(GPR::RAX),
8184 )?;
8185 }
8186
8187 a.emit_add(
8189 Size::S64,
8190 Location::Imm32((stack_offset) as _),
8191 Location::GPR(GPR::RSP),
8192 )?;
8193
8194 a.emit_ret()?;
8196
8197 let mut body = a.finalize().unwrap();
8198 body.shrink_to_fit();
8199 output_reporter.finish(body.len())?;
8200 Ok(FunctionBody {
8201 body,
8202 unwind_info: None,
8203 })
8204 }
8205 fn gen_import_call_trampoline(
8208 &self,
8209 vmoffsets: &VMOffsets,
8210 index: FunctionIndex,
8211 sig: &FunctionType,
8212 calling_convention: CallingConvention,
8213 progress_callback: Option<&CompilationProgressCallback>,
8214 ) -> Result<CustomSection, CompileError> {
8215 let mut a = AssemblerX64::new(0, None)?;
8217 let mut output_reporter = ChunkedOutputReporter::new(progress_callback);
8218
8219 if sig
8229 .params()
8230 .iter()
8231 .any(|&x| x == Type::F32 || x == Type::F64)
8232 {
8233 let mut param_locations = vec![];
8234
8235 let stack_offset: i32 = if sig.params().len() > 5 {
8237 5 * 8
8238 } else {
8239 (sig.params().len() as i32) * 8
8240 };
8241 if stack_offset > 0 {
8242 a.emit_sub(
8243 Size::S64,
8244 Location::Imm32(stack_offset as u32),
8245 Location::GPR(GPR::RSP),
8246 )?;
8247 }
8248
8249 static PARAM_REGS: &[GPR] = &[GPR::RSI, GPR::RDX, GPR::RCX, GPR::R8, GPR::R9];
8251 #[allow(clippy::needless_range_loop)]
8252 for i in 0..sig.params().len() {
8253 let loc = match i {
8254 0..=4 => {
8255 let loc = Location::Memory(GPR::RSP, (i * 8) as i32);
8256 a.emit_mov(Size::S64, Location::GPR(PARAM_REGS[i]), loc)?;
8257 loc
8258 }
8259 _ => Location::Memory(GPR::RSP, stack_offset + 8 + ((i - 5) * 8) as i32),
8260 };
8261 param_locations.push(loc);
8262 output_reporter.check(a.get_offset().0)?;
8263 }
8264
8265 let mut argalloc = ArgumentRegisterAllocator::default();
8267 argalloc.next(Type::I64, calling_convention)?.unwrap(); let mut caller_stack_offset: i32 = 0;
8269 for (i, ty) in sig.params().iter().enumerate() {
8270 let prev_loc = param_locations[i];
8271 let targ = match argalloc.next(*ty, calling_convention)? {
8272 Some(X64Register::GPR(gpr)) => Location::GPR(gpr),
8273 Some(X64Register::XMM(xmm)) => Location::SIMD(xmm),
8274 None => {
8275 a.emit_mov(Size::S64, prev_loc, Location::GPR(GPR::RAX))?;
8281 a.emit_mov(
8282 Size::S64,
8283 Location::GPR(GPR::RAX),
8284 Location::Memory(GPR::RSP, stack_offset + 8 + caller_stack_offset),
8285 )?;
8286 caller_stack_offset += 8;
8287 output_reporter.check(a.get_offset().0)?;
8288 continue;
8289 }
8290 };
8291 a.emit_mov(Size::S64, prev_loc, targ)?;
8292 output_reporter.check(a.get_offset().0)?;
8293 }
8294
8295 if stack_offset > 0 {
8297 a.emit_add(
8298 Size::S64,
8299 Location::Imm32(stack_offset as u32),
8300 Location::GPR(GPR::RSP),
8301 )?;
8302 }
8303 }
8304
8305 let offset = vmoffsets.vmctx_vmfunction_import(index);
8309 let ptr_arg_offset = vmctx_offset(offset)?;
8310 let vmctx_arg_offset = vmctx_offset(offset.saturating_add(8))?;
8311
8312 a.emit_mov(
8313 Size::S64,
8314 Location::Memory(GPR::RDI, ptr_arg_offset), Location::GPR(GPR::RAX),
8316 )?;
8317 a.emit_mov(
8318 Size::S64,
8319 Location::Memory(GPR::RDI, vmctx_arg_offset), Location::GPR(GPR::RDI),
8321 )?;
8322 a.emit_host_redirection(GPR::RAX)?;
8323
8324 let mut contents = a.finalize().unwrap();
8325 contents.shrink_to_fit();
8326 output_reporter.finish(contents.len())?;
8327 let section_body = SectionBody::new_with_vec(contents);
8328
8329 Ok(CustomSection {
8330 protection: CustomSectionProtection::ReadExecute,
8331 alignment: None,
8332 bytes: section_body,
8333 relocations: vec![],
8334 })
8335 }
8336
8337 #[cfg(feature = "unwind")]
8338 fn gen_dwarf_unwind_info(&mut self, code_len: usize) -> Option<UnwindInstructions> {
8339 let mut instructions = vec![];
8340 for &(instruction_offset, ref inst) in &self.unwind_ops {
8341 let instruction_offset = instruction_offset as u32;
8342 match *inst {
8343 UnwindOps::PushFP { up_to_sp } => {
8344 instructions.push((
8345 instruction_offset,
8346 CallFrameInstruction::CfaOffset(up_to_sp as i32),
8347 ));
8348 instructions.push((
8349 instruction_offset,
8350 CallFrameInstruction::Offset(X86_64::RBP, -(up_to_sp as i32)),
8351 ));
8352 }
8353 UnwindOps::DefineNewFrame => {
8354 instructions.push((
8355 instruction_offset,
8356 CallFrameInstruction::CfaRegister(X86_64::RBP),
8357 ));
8358 }
8359 UnwindOps::SaveRegister { reg, bp_neg_offset } => instructions.push((
8360 instruction_offset,
8361 CallFrameInstruction::Offset(reg.dwarf_index(), -bp_neg_offset),
8362 )),
8363 UnwindOps::Push2Regs { .. } | UnwindOps::SubtractFP { .. } => unimplemented!(),
8364 }
8365 }
8366 Some(UnwindInstructions {
8367 instructions,
8368 len: code_len as u32,
8369 })
8370 }
8371 #[cfg(not(feature = "unwind"))]
8372
8373 fn gen_dwarf_unwind_info(&mut self, _code_len: usize) -> Option<UnwindInstructions> {
8374 None
8375 }
8376
8377 #[cfg(feature = "unwind")]
8378 fn gen_windows_unwind_info(&mut self, _code_len: usize) -> Option<Vec<u8>> {
8379 let unwind_info = create_unwind_info_from_insts(&self.unwind_ops);
8380 if let Some(unwind) = unwind_info {
8381 let sz = unwind.emit_size();
8382 let mut tbl = vec![0; sz];
8383 unwind.emit(&mut tbl);
8384 Some(tbl)
8385 } else {
8386 None
8387 }
8388 }
8389
8390 #[cfg(not(feature = "unwind"))]
8391
8392 fn gen_windows_unwind_info(&mut self, _code_len: usize) -> Option<Vec<u8>> {
8393 None
8394 }
8395}
8396
8397#[cfg(test)]
8398mod test {
8399 use super::*;
8400 use enumset::enum_set;
8401 use std::str::FromStr;
8402 use wasmer_types::target::{CpuFeature, Target, Triple};
8403
8404 fn test_move_location(machine: &mut MachineX86_64) -> Result<(), CompileError> {
8405 machine.move_location_for_native(
8406 Size::S64,
8407 Location::GPR(GPR::RAX),
8408 Location::GPR(GPR::RCX),
8409 )?;
8410 machine.move_location_for_native(
8411 Size::S64,
8412 Location::GPR(GPR::RAX),
8413 Location::Memory(GPR::RDX, 10),
8414 )?;
8415 machine.move_location_for_native(
8416 Size::S64,
8417 Location::GPR(GPR::RAX),
8418 Location::Memory(GPR::RDX, -10),
8419 )?;
8420 machine.move_location_for_native(
8421 Size::S64,
8422 Location::Memory(GPR::RDX, 10),
8423 Location::GPR(GPR::RAX),
8424 )?;
8425 machine.move_location_for_native(
8426 Size::S64,
8427 Location::Imm64(50),
8428 Location::GPR(GPR::RAX),
8429 )?;
8430 machine.move_location_for_native(
8431 Size::S64,
8432 Location::Imm32(50),
8433 Location::GPR(GPR::RAX),
8434 )?;
8435 machine.move_location_for_native(Size::S64, Location::Imm8(50), Location::GPR(GPR::RAX))?;
8436
8437 machine.move_location_for_native(
8438 Size::S32,
8439 Location::GPR(GPR::RAX),
8440 Location::GPR(GPR::RCX),
8441 )?;
8442 machine.move_location_for_native(
8443 Size::S32,
8444 Location::GPR(GPR::RAX),
8445 Location::Memory(GPR::RDX, 10),
8446 )?;
8447 machine.move_location_for_native(
8448 Size::S32,
8449 Location::GPR(GPR::RAX),
8450 Location::Memory(GPR::RDX, -10),
8451 )?;
8452 machine.move_location_for_native(
8453 Size::S32,
8454 Location::Memory(GPR::RDX, 10),
8455 Location::GPR(GPR::RAX),
8456 )?;
8457 machine.move_location_for_native(
8458 Size::S32,
8459 Location::Imm32(50),
8460 Location::GPR(GPR::RAX),
8461 )?;
8462 machine.move_location_for_native(Size::S32, Location::Imm8(50), Location::GPR(GPR::RAX))?;
8463
8464 machine.move_location_for_native(
8465 Size::S16,
8466 Location::GPR(GPR::RAX),
8467 Location::GPR(GPR::RCX),
8468 )?;
8469 machine.move_location_for_native(
8470 Size::S16,
8471 Location::GPR(GPR::RAX),
8472 Location::Memory(GPR::RDX, 10),
8473 )?;
8474 machine.move_location_for_native(
8475 Size::S16,
8476 Location::GPR(GPR::RAX),
8477 Location::Memory(GPR::RDX, -10),
8478 )?;
8479 machine.move_location_for_native(
8480 Size::S16,
8481 Location::Memory(GPR::RDX, 10),
8482 Location::GPR(GPR::RAX),
8483 )?;
8484 machine.move_location_for_native(Size::S16, Location::Imm8(50), Location::GPR(GPR::RAX))?;
8485
8486 machine.move_location_for_native(
8487 Size::S8,
8488 Location::GPR(GPR::RAX),
8489 Location::GPR(GPR::RCX),
8490 )?;
8491 machine.move_location_for_native(
8492 Size::S8,
8493 Location::GPR(GPR::RAX),
8494 Location::Memory(GPR::RDX, 10),
8495 )?;
8496 machine.move_location_for_native(
8497 Size::S8,
8498 Location::GPR(GPR::RAX),
8499 Location::Memory(GPR::RDX, -10),
8500 )?;
8501 machine.move_location_for_native(
8502 Size::S8,
8503 Location::Memory(GPR::RDX, 10),
8504 Location::GPR(GPR::RAX),
8505 )?;
8506 machine.move_location_for_native(Size::S8, Location::Imm8(50), Location::GPR(GPR::RAX))?;
8507
8508 machine.move_location_for_native(
8509 Size::S64,
8510 Location::SIMD(XMM::XMM0),
8511 Location::GPR(GPR::RAX),
8512 )?;
8513 machine.move_location_for_native(
8514 Size::S64,
8515 Location::SIMD(XMM::XMM0),
8516 Location::Memory(GPR::RDX, -10),
8517 )?;
8518 machine.move_location_for_native(
8519 Size::S64,
8520 Location::GPR(GPR::RAX),
8521 Location::SIMD(XMM::XMM0),
8522 )?;
8523 machine.move_location_for_native(
8524 Size::S64,
8525 Location::Memory(GPR::RDX, -10),
8526 Location::SIMD(XMM::XMM0),
8527 )?;
8528
8529 Ok(())
8530 }
8531
8532 fn test_move_location_extended(
8533 machine: &mut MachineX86_64,
8534 signed: bool,
8535 sized: Size,
8536 ) -> Result<(), CompileError> {
8537 machine.move_location_extend(
8538 sized,
8539 signed,
8540 Location::GPR(GPR::RAX),
8541 Size::S64,
8542 Location::GPR(GPR::RCX),
8543 )?;
8544 machine.move_location_extend(
8545 sized,
8546 signed,
8547 Location::GPR(GPR::RAX),
8548 Size::S64,
8549 Location::Memory(GPR::RCX, 10),
8550 )?;
8551 machine.move_location_extend(
8552 sized,
8553 signed,
8554 Location::Memory(GPR::RAX, 10),
8555 Size::S64,
8556 Location::GPR(GPR::RCX),
8557 )?;
8558 if sized != Size::S32 {
8559 machine.move_location_extend(
8560 sized,
8561 signed,
8562 Location::GPR(GPR::RAX),
8563 Size::S32,
8564 Location::GPR(GPR::RCX),
8565 )?;
8566 machine.move_location_extend(
8567 sized,
8568 signed,
8569 Location::GPR(GPR::RAX),
8570 Size::S32,
8571 Location::Memory(GPR::RCX, 10),
8572 )?;
8573 machine.move_location_extend(
8574 sized,
8575 signed,
8576 Location::Memory(GPR::RAX, 10),
8577 Size::S32,
8578 Location::GPR(GPR::RCX),
8579 )?;
8580 }
8581
8582 Ok(())
8583 }
8584
8585 fn test_binop_op(
8586 machine: &mut MachineX86_64,
8587 op: fn(&mut MachineX86_64, Location, Location, Location) -> Result<(), CompileError>,
8588 ) -> Result<(), CompileError> {
8589 op(
8590 machine,
8591 Location::GPR(GPR::RDX),
8592 Location::GPR(GPR::RDX),
8593 Location::GPR(GPR::RAX),
8594 )?;
8595 op(
8596 machine,
8597 Location::GPR(GPR::RDX),
8598 Location::Imm32(10),
8599 Location::GPR(GPR::RAX),
8600 )?;
8601 op(
8602 machine,
8603 Location::GPR(GPR::RAX),
8604 Location::GPR(GPR::RAX),
8605 Location::GPR(GPR::RAX),
8606 )?;
8607 op(
8608 machine,
8609 Location::Imm32(10),
8610 Location::GPR(GPR::RDX),
8611 Location::GPR(GPR::RAX),
8612 )?;
8613 op(
8614 machine,
8615 Location::GPR(GPR::RAX),
8616 Location::GPR(GPR::RDX),
8617 Location::Memory(GPR::RAX, 10),
8618 )?;
8619 op(
8620 machine,
8621 Location::GPR(GPR::RAX),
8622 Location::Memory(GPR::RDX, 16),
8623 Location::Memory(GPR::RAX, 10),
8624 )?;
8625 op(
8626 machine,
8627 Location::Memory(GPR::RAX, 0),
8628 Location::Memory(GPR::RDX, 16),
8629 Location::Memory(GPR::RAX, 10),
8630 )?;
8631
8632 Ok(())
8633 }
8634
8635 fn test_float_binop_op(
8636 machine: &mut MachineX86_64,
8637 op: fn(&mut MachineX86_64, Location, Location, Location) -> Result<(), CompileError>,
8638 ) -> Result<(), CompileError> {
8639 op(
8640 machine,
8641 Location::SIMD(XMM::XMM3),
8642 Location::SIMD(XMM::XMM2),
8643 Location::SIMD(XMM::XMM0),
8644 )?;
8645 op(
8646 machine,
8647 Location::SIMD(XMM::XMM0),
8648 Location::SIMD(XMM::XMM2),
8649 Location::SIMD(XMM::XMM0),
8650 )?;
8651 op(
8652 machine,
8653 Location::SIMD(XMM::XMM0),
8654 Location::SIMD(XMM::XMM0),
8655 Location::SIMD(XMM::XMM0),
8656 )?;
8657 op(
8658 machine,
8659 Location::Memory(GPR::RBP, 0),
8660 Location::SIMD(XMM::XMM2),
8661 Location::SIMD(XMM::XMM0),
8662 )?;
8663 op(
8664 machine,
8665 Location::Memory(GPR::RBP, 0),
8666 Location::Memory(GPR::RDX, 10),
8667 Location::SIMD(XMM::XMM0),
8668 )?;
8669 op(
8670 machine,
8671 Location::Memory(GPR::RBP, 0),
8672 Location::Memory(GPR::RDX, 16),
8673 Location::Memory(GPR::RAX, 32),
8674 )?;
8675 op(
8676 machine,
8677 Location::SIMD(XMM::XMM0),
8678 Location::Memory(GPR::RDX, 16),
8679 Location::Memory(GPR::RAX, 32),
8680 )?;
8681 op(
8682 machine,
8683 Location::SIMD(XMM::XMM0),
8684 Location::SIMD(XMM::XMM1),
8685 Location::Memory(GPR::RAX, 32),
8686 )?;
8687
8688 Ok(())
8689 }
8690
8691 fn test_float_cmp_op(
8692 machine: &mut MachineX86_64,
8693 op: fn(&mut MachineX86_64, Location, Location, Location) -> Result<(), CompileError>,
8694 ) -> Result<(), CompileError> {
8695 op(
8696 machine,
8697 Location::SIMD(XMM::XMM3),
8698 Location::SIMD(XMM::XMM2),
8699 Location::GPR(GPR::RAX),
8700 )?;
8701 op(
8702 machine,
8703 Location::SIMD(XMM::XMM0),
8704 Location::SIMD(XMM::XMM0),
8705 Location::GPR(GPR::RAX),
8706 )?;
8707 op(
8708 machine,
8709 Location::Memory(GPR::RBP, 0),
8710 Location::SIMD(XMM::XMM2),
8711 Location::GPR(GPR::RAX),
8712 )?;
8713 op(
8714 machine,
8715 Location::Memory(GPR::RBP, 0),
8716 Location::Memory(GPR::RDX, 10),
8717 Location::GPR(GPR::RAX),
8718 )?;
8719 op(
8720 machine,
8721 Location::Memory(GPR::RBP, 0),
8722 Location::Memory(GPR::RDX, 16),
8723 Location::Memory(GPR::RAX, 32),
8724 )?;
8725 op(
8726 machine,
8727 Location::SIMD(XMM::XMM0),
8728 Location::Memory(GPR::RDX, 16),
8729 Location::Memory(GPR::RAX, 32),
8730 )?;
8731 op(
8732 machine,
8733 Location::SIMD(XMM::XMM0),
8734 Location::SIMD(XMM::XMM1),
8735 Location::Memory(GPR::RAX, 32),
8736 )?;
8737
8738 Ok(())
8739 }
8740
8741 #[test]
8742 fn tests_avx() -> Result<(), CompileError> {
8743 let set = enum_set!(CpuFeature::AVX);
8744 let target = Target::new(Triple::from_str("x86_64-linux-gnu").unwrap(), set);
8745 let mut machine = MachineX86_64::new(Some(target))?;
8746
8747 test_move_location(&mut machine)?;
8748 test_move_location_extended(&mut machine, false, Size::S8)?;
8749 test_move_location_extended(&mut machine, false, Size::S16)?;
8750 test_move_location_extended(&mut machine, false, Size::S32)?;
8751 test_move_location_extended(&mut machine, true, Size::S8)?;
8752 test_move_location_extended(&mut machine, true, Size::S16)?;
8753 test_move_location_extended(&mut machine, true, Size::S32)?;
8754 test_binop_op(&mut machine, MachineX86_64::emit_binop_add32)?;
8755 test_binop_op(&mut machine, MachineX86_64::emit_binop_add64)?;
8756 test_binop_op(&mut machine, MachineX86_64::emit_binop_sub32)?;
8757 test_binop_op(&mut machine, MachineX86_64::emit_binop_sub64)?;
8758 test_binop_op(&mut machine, MachineX86_64::emit_binop_and32)?;
8759 test_binop_op(&mut machine, MachineX86_64::emit_binop_and64)?;
8760 test_binop_op(&mut machine, MachineX86_64::emit_binop_xor32)?;
8761 test_binop_op(&mut machine, MachineX86_64::emit_binop_xor64)?;
8762 test_binop_op(&mut machine, MachineX86_64::emit_binop_or32)?;
8763 test_binop_op(&mut machine, MachineX86_64::emit_binop_or64)?;
8764 test_binop_op(&mut machine, MachineX86_64::emit_binop_mul32)?;
8765 test_binop_op(&mut machine, MachineX86_64::emit_binop_mul64)?;
8766 test_float_binop_op(&mut machine, MachineX86_64::f32_add)?;
8767 test_float_binop_op(&mut machine, MachineX86_64::f32_sub)?;
8768 test_float_binop_op(&mut machine, MachineX86_64::f32_mul)?;
8769 test_float_binop_op(&mut machine, MachineX86_64::f32_div)?;
8770 test_float_cmp_op(&mut machine, MachineX86_64::f32_cmp_eq)?;
8771 test_float_cmp_op(&mut machine, MachineX86_64::f32_cmp_lt)?;
8772 test_float_cmp_op(&mut machine, MachineX86_64::f32_cmp_le)?;
8773
8774 Ok(())
8775 }
8776
8777 #[test]
8778 fn tests_sse42() -> Result<(), CompileError> {
8779 let set = enum_set!(CpuFeature::SSE42);
8780 let target = Target::new(Triple::from_str("x86_64-linux-gnu").unwrap(), set);
8781 let mut machine = MachineX86_64::new(Some(target))?;
8782
8783 test_move_location(&mut machine)?;
8784 test_move_location_extended(&mut machine, false, Size::S8)?;
8785 test_move_location_extended(&mut machine, false, Size::S16)?;
8786 test_move_location_extended(&mut machine, false, Size::S32)?;
8787 test_move_location_extended(&mut machine, true, Size::S8)?;
8788 test_move_location_extended(&mut machine, true, Size::S16)?;
8789 test_move_location_extended(&mut machine, true, Size::S32)?;
8790 test_binop_op(&mut machine, MachineX86_64::emit_binop_add32)?;
8791 test_binop_op(&mut machine, MachineX86_64::emit_binop_add64)?;
8792 test_binop_op(&mut machine, MachineX86_64::emit_binop_sub32)?;
8793 test_binop_op(&mut machine, MachineX86_64::emit_binop_sub64)?;
8794 test_binop_op(&mut machine, MachineX86_64::emit_binop_and32)?;
8795 test_binop_op(&mut machine, MachineX86_64::emit_binop_and64)?;
8796 test_binop_op(&mut machine, MachineX86_64::emit_binop_xor32)?;
8797 test_binop_op(&mut machine, MachineX86_64::emit_binop_xor64)?;
8798 test_binop_op(&mut machine, MachineX86_64::emit_binop_or32)?;
8799 test_binop_op(&mut machine, MachineX86_64::emit_binop_or64)?;
8800 test_binop_op(&mut machine, MachineX86_64::emit_binop_mul32)?;
8801 test_binop_op(&mut machine, MachineX86_64::emit_binop_mul64)?;
8802 test_float_binop_op(&mut machine, MachineX86_64::f32_add)?;
8803 test_float_binop_op(&mut machine, MachineX86_64::f32_sub)?;
8804 test_float_binop_op(&mut machine, MachineX86_64::f32_mul)?;
8805 test_float_binop_op(&mut machine, MachineX86_64::f32_div)?;
8806 test_float_cmp_op(&mut machine, MachineX86_64::f32_cmp_eq)?;
8807 test_float_cmp_op(&mut machine, MachineX86_64::f32_cmp_lt)?;
8808 test_float_cmp_op(&mut machine, MachineX86_64::f32_cmp_le)?;
8809
8810 Ok(())
8811 }
8812}